just add some logs for CI debugging for https://github.com/vercel/next.js/pull/86266/

Cargo.toml

@@ -328,7 +328,7 @@ turbopack-env = { path = "turbopack/crates/turbopack-env" }
 turbopack-image = { path = "turbopack/crates/turbopack-image" }
 turbopack-json = { path = "turbopack/crates/turbopack-json" }
 turbopack-mdx = { path = "turbopack/crates/turbopack-mdx" }
-turbopack-node = { path = "turbopack/crates/turbopack-node" }
+turbopack-node = { path = "turbopack/crates/turbopack-node", default-features = false }
 turbopack-resolve = { path = "turbopack/crates/turbopack-resolve" }
 turbopack-static = { path = "turbopack/crates/turbopack-static" }
 turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" }
@@ -422,6 +422,8 @@ napi = { version = "2", default-features = false, features = [
   "napi5",
   "compat-mode",
 ] }
+napi-derive = "2"
+napi-build = "2"
 notify = "8.1.0"
 once_cell = "1.17.1"
 owo-colors = "4.2.2"

crates/napi/Cargo.toml

@@ -46,12 +46,12 @@ workspace = true
 
 [package.metadata.cargo-shear]
 ignored = [
-  # we need to set features on these packages when building for WASM, but we don't directly use them
-  "getrandom",
-  "iana-time-zone",
-  # the plugins feature needs to set a feature on this transitively depended-on package, we never
-  # directly import it
-  "turbopack-ecmascript-plugins",
+    # we need to set features on these packages when building for WASM, but we don't directly use them
+    "getrandom",
+    "iana-time-zone",
+    # the plugins feature needs to set a feature on this transitively depended-on package, we never
+    # directly import it
+    "turbopack-ecmascript-plugins",
 ]
 
 [dependencies]
@@ -64,7 +64,7 @@ flate2 = { workspace = true }
 futures-util = { workspace = true }
 owo-colors = { workspace = true }
 napi = { workspace = true }
-napi-derive = "2"
+napi-derive = { workspace = true }
 next-custom-transforms = { workspace = true }
 next-taskless = { workspace = true }
 rand = { workspace = true }
@@ -116,14 +116,15 @@ next-core = { workspace = true }
 mdxjs = { workspace = true, features = ["serializable"] }
 
 turbo-tasks-malloc = { workspace = true, default-features = false, features = [
-    "custom_allocator"
+    "custom_allocator",
 ] }
 
 turbopack-core = { workspace = true }
 turbopack-ecmascript-hmr-protocol = { workspace = true }
 turbopack-trace-utils = { workspace = true }
 turbopack-trace-server = { workspace = true }
 turbopack-ecmascript-plugins = { workspace = true, optional = true }
+turbopack-node = { workspace = true, default-features = false, features = ["worker_pool"] }
 
 [target.'cfg(windows)'.dependencies]
 windows-sys = "0.60"
@@ -144,8 +145,7 @@ tokio = { workspace = true, features = ["full"] }
 
 [build-dependencies]
 anyhow = { workspace = true }
-napi-build = "2"
+napi-build = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 vergen-gitcl = { workspace = true }
-

crates/next-api/Cargo.toml

@@ -40,7 +40,7 @@ turbopack-browser = { workspace = true }
 turbopack-core = { workspace = true }
 turbopack-css = { workspace = true }
 turbopack-ecmascript = { workspace = true }
-turbopack-node = { workspace = true }
+turbopack-node = { workspace = true, default-features = false }
 turbopack-nodejs = { workspace = true }
 turbopack-resolve = { workspace = true }
 turbopack-wasm = { workspace = true }

crates/next-build-test/Cargo.toml

@@ -27,4 +27,4 @@ turbo-tasks = { workspace = true }
 turbo-tasks-backend = { workspace = true }
 turbo-tasks-malloc = { workspace = true }
 turbopack-trace-utils = { workspace = true }
-
+turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]}

crates/next-core/Cargo.toml

@@ -79,7 +79,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [
 ] }
 turbopack-ecmascript-runtime = { workspace = true }
 turbopack-image = { workspace = true }
-turbopack-node = { workspace = true }
+turbopack-node = { workspace = true, default-features = false }
 turbopack-nodejs = { workspace = true }
 turbopack-resolve = { workspace = true }
 turbopack-static = { workspace = true }

packages/next/src/build/swc/generated-native.d.ts

@@ -34,6 +34,28 @@ export declare class ExternalObject<T> {
     [K: symbol]: T
   }
 }
+export interface NapiPoolOptions {
+  filename: RcStr
+  concurrency: number
+  env: Record<string, string>
+  cwd: RcStr
+}
+export interface WorkerTermination {
+  filename: RcStr
+  workerId: number
+}
+export declare function recvPoolRequest(): Promise<NapiPoolOptions>
+export declare function recvWorkerTermination(): Promise<WorkerTermination>
+export declare function recvWorkerRequest(filename: RcStr): Promise<number>
+export declare function recvMessageInWorker(workerId: number): Promise<string>
+export declare function notifyWorkerAck(
+  taskId: number,
+  workerId: number
+): Promise<void>
+export declare function sendTaskMessage(
+  taskId: number,
+  message: string
+): Promise<void>
 export declare function lockfileTryAcquireSync(
   path: string
 ): { __napiType: 'Lockfile' } | null

packages/next/src/build/swc/index.ts

@@ -42,6 +42,7 @@ import type {
   WrittenEndpoint,
 } from './types'
 import { throwTurbopackInternalError } from '../../shared/lib/turbopack/internal-error'
+import { runLoaderWorkerPool } from './loaderWorkerPool'
 
 type RawBindings = typeof import('./generated-native')
 type RawWasmBindings = typeof import('./generated-wasm') & {
@@ -494,6 +495,7 @@ const normalizePathOnWindows = (p: string) =>
 // TODO(sokra) Support wasm option.
 function bindingToApi(
   binding: RawBindings,
+  bindingPath: string,
   _wasm: boolean
 ): Binding['turbo']['createProject'] {
   type NativeFunction<T> = (
@@ -674,6 +676,13 @@ function bindingToApi(
 
     constructor(nativeProject: { __napiType: 'Project' }) {
       this._nativeProject = nativeProject
+
+      if (
+        typeof binding.recvPoolRequest === 'function' &&
+        typeof binding.recvWorkerTermination === 'function'
+      ) {
+        runLoaderWorkerPool(binding, bindingPath)
+      }
     }
 
     async update(options: PartialProjectOptions) {
@@ -1361,30 +1370,34 @@ function loadNative(importPath?: string) {
     throw new Error('cannot run loadNative when `NEXT_TEST_WASM` is set')
   }
 
+  const customBindingsPath = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
+    ? require.resolve(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS)
+    : null
   const customBindings: RawBindings = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
     ? require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS)
     : null
   let bindings: RawBindings = customBindings
+  let bindingsPath = customBindingsPath
   let attempts: any[] = []
 
   const NEXT_TEST_NATIVE_DIR = process.env.NEXT_TEST_NATIVE_DIR
   for (const triple of triples) {
     if (NEXT_TEST_NATIVE_DIR) {
       try {
+        const bindingForTest = `${NEXT_TEST_NATIVE_DIR}/next-swc.${triple.platformArchABI}.node`
         // Use the binary directly to skip `pnpm pack` for testing as it's slow because of the large native binary.
-        bindings = require(
-          `${NEXT_TEST_NATIVE_DIR}/next-swc.${triple.platformArchABI}.node`
-        )
+        bindings = require(bindingForTest)
+        bindingsPath = require.resolve(bindingForTest)
         infoLog(
           'next-swc build: local built @next/swc from NEXT_TEST_NATIVE_DIR'
         )
         break
       } catch (e) {}
     } else {
       try {
-        bindings = require(
-          `@next/swc/native/next-swc.${triple.platformArchABI}.node`
-        )
+        const normalBinding = `@next/swc/native/next-swc.${triple.platformArchABI}.node`
+        bindings = require(normalBinding)
+        bindingsPath = require.resolve(normalBinding)
         infoLog('next-swc build: local built @next/swc')
         break
       } catch (e) {}
@@ -1402,6 +1415,7 @@ function loadNative(importPath?: string) {
         : `@next/swc-${triple.platformArchABI}`
       try {
         bindings = require(pkg)
+        bindingsPath = require.resolve(pkg)
         if (!importPath) {
           checkVersionMismatch(require(`${pkg}/package.json`))
         }
@@ -1480,7 +1494,11 @@ function loadNative(importPath?: string) {
       initCustomTraceSubscriber: bindings.initCustomTraceSubscriber,
       teardownTraceSubscriber: bindings.teardownTraceSubscriber,
       turbo: {
-        createProject: bindingToApi(customBindings ?? bindings, false),
+        createProject: bindingToApi(
+          customBindings ?? bindings,
+          customBindingsPath ?? bindingsPath!,
+          false
+        ),
         startTurbopackTraceServer(traceFilePath, port) {
           Log.warn(
             `Turbopack trace server started. View trace at https://trace.nextjs.org${port != null ? `?port=${port}` : ''}`

packages/next/src/build/swc/loaderWorkerPool.ts

@@ -0,0 +1,96 @@
+import { Worker } from 'worker_threads'
+
+const loaderWorkers: Record<string, Array<Worker>> = {}
+
+const KillMsg = '__kill__'
+
+export async function runLoaderWorkerPool(
+  binding: typeof import('./generated-native'),
+  bindingPath: string
+) {
+  await Promise.all([
+    runPoolScaler(binding, bindingPath),
+    runWorkerTerminator(binding),
+  ])
+}
+
+async function runPoolScaler(
+  binding: typeof import('./generated-native'),
+  bindingPath: string
+) {
+  while (true) {
+    try {
+      console.log('[loaderWorkerPool] waiting for pool request')
+      let poolOptions = await binding.recvPoolRequest()
+      console.log('[loaderWorkerPool] received pool request', {
+        ...poolOptions,
+        env: 'omitted',
+      })
+      const { filename, concurrency, env, cwd } = poolOptions
+      // Wildcard of "*" meaning to scale all of pools even with different poolId
+      const workers =
+        filename === '*'
+          ? Object.values(loaderWorkers).flat()
+          : loaderWorkers[filename] || (loaderWorkers[filename] = [])
+      if (workers.length < concurrency) {
+        for (let i = workers.length; i < concurrency; i++) {
+          const worker = new Worker(filename, {
+            workerData: {
+              poolId: filename,
+              bindingPath,
+              cwd,
+            },
+            env,
+          })
+          workers.push(worker)
+        }
+      } else if (workers.length > concurrency) {
+        const workersToKill = workers.splice(0, workers.length - concurrency)
+        workersToKill.forEach(terminateWorker)
+      }
+    } catch (_) {
+      // rust channel closed, do nothing
+      return
+    }
+  }
+}
+
+async function runWorkerTerminator(
+  binding: typeof import('./generated-native')
+) {
+  while (true) {
+    try {
+      console.log('[loaderWorkerPool] waiting for worker termination')
+      const { filename, workerId } = await binding.recvWorkerTermination()
+      console.log('[loaderWorkerPool] received worker termination', {
+        filename,
+        workerId,
+      })
+      const workers = loaderWorkers[filename]
+      const workerIdx = workers.findIndex(
+        (worker) => worker.threadId === workerId
+      )
+      if (workerIdx > -1) {
+        const workersToKill = workers.splice(workerIdx, 1)
+        workersToKill.forEach(terminateWorker)
+      }
+    } catch (_) {
+      // rust channel closed, do nothing
+      return
+    }
+  }
+}
+
+async function terminateWorker(worker: Worker) {
+  await new Promise<void>((resolve) => {
+    const onMessage = (msg: any) => {
+      if (msg === KillMsg) {
+        worker.off('message', onMessage)
+        resolve()
+      }
+    }
+    worker.on('message', onMessage)
+    worker.postMessage(KillMsg)
+  })
+  await worker.terminate()
+}

turbopack/crates/turbo-rcstr/src/lib.rs

@@ -240,6 +240,12 @@ impl AsRef<[u8]> for RcStr {
     }
 }
 
+impl AsRef<str> for RcStr {
+    fn as_ref(&self) -> &str {
+        self.as_str()
+    }
+}
+
 impl From<RcStr> for BytesStr {
     fn from(value: RcStr) -> Self {
         Self::from_str_slice(value.as_str())