feat(loader-runner): Allow limiting worker pool size for parallel loaders (#12277)

* feat(loader-runner): Make tinypool thread count configurable

* Addressing comments

Author: Pablinho

packages/rspack/etc/core.api.md

@@ -4510,7 +4510,9 @@ class LoaderObject {
     // (undocumented)
     options?: string | object;
     // (undocumented)
-    parallel?: boolean;
+    parallel?: boolean | {
+        maxWorkers?: number;
+    };
     // (undocumented)
     path: string;
     // (undocumented)
@@ -7049,7 +7051,9 @@ export type RuleSetLoaderOptions = string | Record<string, any>;
 export type RuleSetLoaderWithOptions = {
     ident?: string;
     loader: RuleSetLoader;
-    parallel?: boolean;
+    parallel?: boolean | {
+        maxWorkers?: number;
+    };
     options?: RuleSetLoaderOptions;
 };
 

packages/rspack/src/config/types.ts

@@ -882,7 +882,7 @@ export type RuleSetLoaderWithOptions = {
 
 	loader: RuleSetLoader;
 
-	parallel?: boolean;
+	parallel?: boolean | { maxWorkers?: number };
 
 	options?: RuleSetLoaderOptions;
 };

packages/rspack/src/loader-runner/index.ts

@@ -144,7 +144,7 @@ export class LoaderObject {
 	pitch?: Function;
 	raw?: boolean;
 	type?: "module" | "commonjs";
-	parallel?: boolean;
+	parallel?: boolean | { maxWorkers?: number };
 	/**
 	 * @internal This field is rspack internal. Do not edit.
 	 */
@@ -995,7 +995,10 @@ export async function runLoaders(
 						loaderState,
 						args
 					},
-					getWorkerLoaderHandlers()
+					getWorkerLoaderHandlers(),
+					typeof currentLoaderObject?.parallel === "object"
+						? currentLoaderObject.parallel
+						: undefined
 				)) || [];
 		} else {
 			if (loaderState === JsLoaderState.Normal)

packages/rspack/src/loader-runner/service.ts

@@ -3,20 +3,25 @@ import path from "node:path";
 import type { Tinypool } from "tinypool" with { "resolution-mode": "import" };
 
 let pool: Promise<Tinypool> | undefined;
-const ensureLoaderWorkerPool = async () => {
+const ensureLoaderWorkerPool = async (workerOptions?: {
+	maxWorkers?: number;
+}) => {
 	if (pool) {
 		return pool;
 	}
 	return (pool = import("tinypool").then(({ Tinypool }) => {
 		const cpus = require("node:os").cpus().length;
 		const availableThreads = Math.max(cpus - 1, 1);
+		const maxWorkers = workerOptions?.maxWorkers
+			? Math.max(workerOptions.maxWorkers, 1)
+			: undefined;
 
 		const pool = new Tinypool({
 			filename: path.resolve(__dirname, "worker.js"),
 			useAtomics: false,
 
-			maxThreads: availableThreads,
-			minThreads: availableThreads,
+			maxThreads: maxWorkers || availableThreads,
+			minThreads: maxWorkers || availableThreads,
 			concurrentTasksPerWorker: 1
 		});
 
@@ -199,9 +204,10 @@ export const run = async (
 	task: any,
 	options: RunOptions & {
 		handleIncomingRequest: HandleIncomingRequest;
-	}
+	},
+	workerOptions?: { maxWorkers?: number }
 ) =>
-	ensureLoaderWorkerPool().then(async pool => {
+	ensureLoaderWorkerPool(workerOptions).then(async pool => {
 		const { MessageChannel } = await import("node:worker_threads");
 		const { port1: mainPort, port2: workerPort } = new MessageChannel();
 		// Create message channel for processing sync API requests from worker

tests/rspack-test/configCases/loader-parallel/parallel-object/index.js

@@ -0,0 +1,3 @@
+it("should run loader in parallel", () => {
+	expect(require("./lib.js")).toBe(true)
+})

tests/rspack-test/configCases/loader-parallel/parallel-object/lib.js

@@ -0,0 +1,3 @@
+module.exports = function () {
+	return `module.exports = ${this.parallel}`;
+}

tests/rspack-test/configCases/loader-parallel/parallel-object/loader-in-worker.js

@@ -0,0 +1,26 @@
+module.exports = {
+	context: __dirname,
+	module: {
+		rules: [
+			{
+				test: /lib\.js/,
+				use: [
+					{
+						loader: "./unclonable.js",
+						options: {
+							notclonable() {}
+						}
+					},
+					{
+						loader: "./loader-in-worker.js",
+						parallel: { maxWorkers: 2 },
+						options: {}
+					}
+				]
+			}
+		]
+	},
+	experiments: {
+		parallelLoader: true
+	}
+};

tests/rspack-test/configCases/loader-parallel/parallel-object/rspack.config.js

@@ -0,0 +1,3 @@
+module.exports = function(content) {
+	return content
+}