From f5858765815f0c8600d59e4e712e56fd98441c43 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 24 Sep 2025 17:49:44 +0200 Subject: [PATCH] module: refactor and clarify async loader hook customizations - This updates the comments that assume loader hooks must be async - Differentiate the sync/async loader hook paths in naming `#customizations` is now `#asyncLoaderHooks` to make it clear it's from the async APIs. - Differentiate the paths running on the loader hook thread (affects the loading of async other loader hooks and are async) v.s. paths on the main thread calling out to code on the loader hook thread (do not handle loading of other async loader hooks, and can be sync by blocking). - `Hooks` is now `AsyncLoaderHooksOnLoaderHookWorker` - `CustomizedModuleLoader` is now `AsyncLoaderHooksProxiedToLoaderHookWorker` and moved into `lib/internal/modules/esm/hooks.js` as it implements the same interface as `AsyncLoaderHooksOnLoaderHookWorker` - `HooksProxy` is now `AsyncLoaderHookWorker` - Adjust the JSDoc accordingly - Clarify the "loader worker" as the "async loader hook worker" i.e. when there's no _async_ loader hook registered, there won't be this worker, to avoid the misconception that this worker is spawned unconditionally. - The code run on the loader hook worker to process `--experimental-loader` is moved into `lib/internal/modules/esm/worker.js` for clarity. - The initialization configuration `forceDefaultLoader` is split into `shouldSpawnLoaderHookWorker` and `shouldPreloadModules` as those can be separate. - `--experimental-vm-modules` is now processed during pre-execution and no longer part of the initialization of the built-in ESM loader, as it only exposes the vm APIs of ESM, and is unrelated to built-in ESM loading. --- lib/internal/main/worker_thread.js | 22 +- lib/internal/modules/esm/hooks.js | 148 ++++++++++- lib/internal/modules/esm/loader.js | 279 +++++++------------- lib/internal/modules/esm/utils.js | 62 +---- lib/internal/modules/esm/worker.js | 64 ++++- lib/internal/modules/run_main.js | 2 +- lib/internal/process/pre_execution.js | 75 +++--- test/es-module/test-esm-loader-modulemap.js | 2 +- test/parallel/test-bootstrap-modules.js | 10 +- 9 files changed, 355 insertions(+), 309 deletions(-) diff --git a/lib/internal/main/worker_thread.js b/lib/internal/main/worker_thread.js index 62dc44cf34b739..1e28a3452220e9 100644 --- a/lib/internal/main/worker_thread.js +++ b/lib/internal/main/worker_thread.js @@ -11,7 +11,6 @@ const { ObjectDefineProperty, PromisePrototypeThen, RegExpPrototypeExec, - SafeWeakMap, globalThis: { SharedArrayBuffer, }, @@ -19,7 +18,7 @@ const { const { prepareWorkerThreadExecution, - setupUserModules, + initializeModuleLoaders, markBootstrapComplete, } = require('internal/process/pre_execution'); @@ -138,11 +137,13 @@ port.on('message', (message) => { workerIo.sharedCwdCounter = cwdCounter; } - const isLoaderWorker = - doEval === 'internal' && - filename === require('internal/modules/esm/utils').loaderWorkerId; - // Disable custom loaders in loader worker. - setupUserModules(isLoaderWorker); + const isLoaderHookWorker = (filename === 'internal/modules/esm/worker' && doEval === 'internal'); + if (!isLoaderHookWorker) { + // If we are in the loader hook worker, delay the module loader initializations until + // initializeAsyncLoaderHooksOnLoaderHookWorker() which needs to run preloads + // after the asynchronous loader hooks are registered. + initializeModuleLoaders({ shouldSpawnLoaderHookWorker: true, shouldPreloadModules: true }); + } if (!hasStdin) process.stdin.push(null); @@ -152,9 +153,10 @@ port.on('message', (message) => { port.postMessage({ type: UP_AND_RUNNING }); switch (doEval) { case 'internal': { - // Create this WeakMap in js-land because V8 has no C++ API for WeakMap. - internalBinding('module_wrap').callbackMap = new SafeWeakMap(); - require(filename)(workerData, publicPort); + // Currently the only user of internal eval is the async loader hook thread. + assert(isLoaderHookWorker, `Unexpected internal eval ${filename}`); + const setupModuleWorker = require('internal/modules/esm/worker'); + setupModuleWorker(workerData, publicPort); break; } diff --git a/lib/internal/modules/esm/hooks.js b/lib/internal/modules/esm/hooks.js index e3ba5fa86252f5..cc66d47a43b704 100644 --- a/lib/internal/modules/esm/hooks.js +++ b/lib/internal/modules/esm/hooks.js @@ -54,7 +54,6 @@ const { } = require('internal/modules/esm/resolve'); const { getDefaultConditions, - loaderWorkerId, } = require('internal/modules/esm/utils'); const { deserializeError } = require('internal/error_serdes'); const { @@ -105,7 +104,39 @@ function defineImportAssertionAlias(context) { // [2] `validate...()`s throw the wrong error -class Hooks { +/** + * @typedef {{ format: ModuleFormat, source: ModuleSource }} LoadResult + */ + +/** + * @typedef {{ format: ModuleFormat, url: string, importAttributes: Record }} ResolveResult + */ + +/** + * Interface for classes that implement asynchronous loader hooks that can be attached to the ModuleLoader + * via `ModuleLoader.#setAsyncLoaderHooks()`. + * @typedef {object} AsyncLoaderHooks + * @property {boolean} allowImportMetaResolve Whether to allow the use of `import.meta.resolve`. + * @property {(url: string, context: object, defaultLoad: Function) => Promise} load + * Calling the asynchronous `load` hook asynchronously. + * @property {(url: string, context: object, defaultLoad: Function) => LoadResult} loadSync + * Calling the asynchronous `load` hook synchronously. + * @property {(originalSpecifier: string, parentURL: string, + * importAttributes: Record) => Promise} resolve + * Calling the asynchronous `resolve` hook asynchronously. + * @property {(originalSpecifier: string, parentURL: string, + * importAttributes: Record) => ResolveResult} resolveSync + * Calling the asynchronous `resolve` hook synchronously. + * @property {(specifier: string, parentURL: string) => any} register Register asynchronous loader hooks + * @property {() => void} waitForLoaderHookInitialization Force loading of hooks. + */ + +/** + * @implements {AsyncLoaderHooks} + * Instances of this class run directly on the loader hook worker thread and customize the module + * loading of the hooks worker itself. + */ +class AsyncLoaderHooksOnLoaderHookWorker { #chains = { /** * Phase 1 of 2 in ESM loading. @@ -452,7 +483,7 @@ class Hooks { }; } - forceLoadHooks() { + waitForLoaderHookInitialization() { // No-op } @@ -462,14 +493,20 @@ class Hooks { return meta; } } -ObjectSetPrototypeOf(Hooks.prototype, null); +ObjectSetPrototypeOf(AsyncLoaderHooksOnLoaderHookWorker.prototype, null); /** - * There may be multiple instances of Hooks/HooksProxy, but there is only 1 Internal worker, so - * there is only 1 MessageChannel. + * There is only one loader hook thread for each non-loader-hook worker thread + * (i.e. the non-loader-hook thread and any worker threads that are not loader hook workers themselves), + * so there is only 1 MessageChannel. */ let MessageChannel; -class HooksProxy { + +/** + * Abstraction over a worker thread that runs the asynchronous module loader hooks. + * Instances of this class run on the non-loader-hook thread and communicate with the loader hooks worker thread. + */ +class AsyncLoaderHookWorker { /** * Shared memory. Always use Atomics method to read or write to it. * @type {Int32Array} @@ -503,7 +540,7 @@ class HooksProxy { const lock = new SharedArrayBuffer(SHARED_MEMORY_BYTE_LENGTH); this.#lock = new Int32Array(lock); - this.#worker = new InternalWorker(loaderWorkerId, { + this.#worker = new InternalWorker('internal/modules/esm/worker', { stderr: false, stdin: false, stdout: false, @@ -644,7 +681,7 @@ class HooksProxy { this.#importMetaInitializer(meta, context, loader); } } -ObjectSetPrototypeOf(HooksProxy.prototype, null); +ObjectSetPrototypeOf(AsyncLoaderHookWorker.prototype, null); // TODO(JakobJingleheimer): Remove this when loaders go "stable". let globalPreloadWarningWasEmitted = false; @@ -757,6 +794,95 @@ function nextHookFactory(current, meta, { validateArgs, validateOutput }) { ); } +/** + * @type {AsyncLoaderHookWorker} + * Worker instance used to run async loader hooks in a separate thread. This is a singleton for each + * non-loader-hook worker thread (i.e. the main thread and any worker threads that are not + * loader hook workers themselves). + */ +let asyncLoaderHookWorker; +/** + * Get the AsyncLoaderHookWorker instance. If it is not defined, then create a new one. + * @returns {AsyncLoaderHookWorker} + */ +function getAsyncLoaderHookWorker() { + asyncLoaderHookWorker ??= new AsyncLoaderHookWorker(); + return asyncLoaderHookWorker; +} + +/** + * @implements {AsyncLoaderHooks} + * Instances of this class are created in the non-loader-hook thread and communicate with the worker thread + * spawned to run the async loader hooks. + */ +class AsyncLoaderHooksProxiedToLoaderHookWorker { + + allowImportMetaResolve = true; + + /** + * Instantiate a module loader that uses user-provided custom loader hooks. + */ + constructor() { + getAsyncLoaderHookWorker(); + } + + /** + * Register some loader specifier. + * @param {string} originalSpecifier The specified URL path of the loader to + * be registered. + * @param {string} parentURL The parent URL from where the loader will be + * registered if using it package name as specifier + * @param {any} [data] Arbitrary data to be passed from the custom loader + * (user-land) to the worker. + * @param {any[]} [transferList] Objects in `data` that are changing ownership + * @param {boolean} [isInternal] For internal loaders that should not be publicly exposed. + * @returns {{ format: string, url: URL['href'] }} + */ + register(originalSpecifier, parentURL, data, transferList, isInternal) { + return asyncLoaderHookWorker.makeSyncRequest('register', transferList, originalSpecifier, parentURL, + data, isInternal); + } + + /** + * Resolve the location of the module. + * @param {string} originalSpecifier The specified URL path of the module to + * be resolved. + * @param {string} [parentURL] The URL path of the module's parent. + * @param {ImportAttributes} importAttributes Attributes from the import + * statement or expression. + * @returns {{ format: string, url: URL['href'] }} + */ + resolve(originalSpecifier, parentURL, importAttributes) { + return asyncLoaderHookWorker.makeAsyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); + } + + resolveSync(originalSpecifier, parentURL, importAttributes) { + // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec. + return asyncLoaderHookWorker.makeSyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); + } + + /** + * Provide source that is understood by one of Node's translators. + * @param {URL['href']} url The URL/path of the module to be loaded + * @param {object} [context] Metadata about the module + * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>} + */ + load(url, context) { + return asyncLoaderHookWorker.makeAsyncRequest('load', undefined, url, context); + } + loadSync(url, context) { + return asyncLoaderHookWorker.makeSyncRequest('load', undefined, url, context); + } + + importMetaInitialize(meta, context, loader) { + asyncLoaderHookWorker.importMetaInitialize(meta, context, loader); + } + + waitForLoaderHookInitialization() { + asyncLoaderHookWorker.waitForWorker(); + } +} -exports.Hooks = Hooks; -exports.HooksProxy = HooksProxy; +exports.AsyncLoaderHooksProxiedToLoaderHookWorker = AsyncLoaderHooksProxiedToLoaderHookWorker; +exports.AsyncLoaderHooksOnLoaderHookWorker = AsyncLoaderHooksOnLoaderHookWorker; +exports.AsyncLoaderHookWorker = AsyncLoaderHookWorker; diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 9cb02f05c8d28a..d536d8215c93ab 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -34,7 +34,7 @@ const { kEmptyObject } = require('internal/util'); const { compileSourceTextModule, getDefaultConditions, - forceDefaultLoader, + shouldSpawnLoaderHookWorker, } = require('internal/modules/esm/utils'); const { kImplicitTypeAttribute } = require('internal/modules/esm/assert'); const { @@ -51,10 +51,10 @@ const { urlToFilename, } = require('internal/modules/helpers'); const { - resolveHooks, - resolveWithHooks, - loadHooks, - loadWithHooks, + resolveHooks: syncResolveHooks, + resolveWithHooks: resolveWithSyncHooks, + loadHooks: syncLoadHooks, + loadWithHooks: loadWithSyncHooks, validateLoadSloppy, } = require('internal/modules/customization_hooks'); let defaultResolve, defaultLoadSync, importMetaInitializer; @@ -69,7 +69,7 @@ let debug = require('internal/util/debuglog').debuglog('esm', (fn) => { const { isPromise } = require('internal/util/types'); /** - * @typedef {import('./hooks.js').HooksProxy} HooksProxy + * @typedef {import('./hooks.js').AsyncLoaderHookWorker} AsyncLoaderHookWorker * @typedef {import('./module_job.js').ModuleJobBase} ModuleJobBase * @typedef {import('url').URL} URL */ @@ -123,14 +123,6 @@ function getRaceMessage(filename, parentFilename) { return raceMessage; } -/** - * @type {HooksProxy} - * Multiple loader instances exist for various, specific reasons (see code comments at site). - * In order to maintain consistency, we use a single worker (sandbox), which must sit apart of an - * individual loader instance. - */ -let hooksProxy; - /** * @typedef {import('../cjs/loader.js').Module} CJSModule */ @@ -152,9 +144,17 @@ let hooksProxy; */ /** - * This class covers the base machinery of module loading. To add custom - * behavior you can pass a customizations object and this object will be - * used to do the loading/resolving/registration process. + * @typedef {import('./hooks.js').AsyncLoaderHooks} AsyncLoaderHooks + * @typedef {import('./hooks.js').AsyncLoaderHooksOnLoaderHookWorker} AsyncLoaderHooksOnLoaderHookWorker + * @typedef {import('./hooks.js').AsyncLoaderHooksProxiedToLoaderHookWorker} AsyncLoaderHooksProxiedToLoaderHookWorker + */ + +/** + * This class covers the base machinery of module loading. There are two types of loader hooks: + * 1. Asynchronous loader hooks, which are run in a separate loader hook worker thread. + * This is configured in #asyncLoaderHooks. + * 2. Synchronous loader hooks, which are run in-thread. This is shared with the CJS loader and is + * stored in the cross-module syncResolveHooks and syncLoadHooks arrays. */ class ModuleLoader { /** @@ -185,73 +185,44 @@ class ModuleLoader { allowImportMetaResolve; /** - * Customizations to pass requests to. - * @type {import('./hooks.js').Hooks} - * Note that this value _MUST_ be set with `setCustomizations` - * because it needs to copy `customizations.allowImportMetaResolve` + * Asynchronous loader hooks to pass requests to. + * + * Note that this value _MUST_ be set with `#setAsyncLoaderHooks` + * because it needs to copy `#asyncLoaderHooks.allowImportMetaResolve` * to this property and failure to do so will cause undefined * behavior when invoking `import.meta.resolve`. - * @see {ModuleLoader.setCustomizations} - * @type {CustomizedModuleLoader} + * + * When the ModuleLoader is created on a loader hook thread, this is + * {@link AsyncLoaderHooksOnLoaderHookWorker}, and its methods directly call out + * to loader methods. Otherwise, this is {@link AsyncLoaderHooksProxiedToLoaderHookWorker}, + * and its methods post messages to the loader thread and possibly block on it. + * @see {ModuleLoader.#setAsyncLoaderHooks} + * @type {AsyncLoaderHooks} */ - #customizations; + #asyncLoaderHooks; - constructor(customizations) { - this.setCustomizations(customizations); + constructor(asyncLoaderHooks) { + this.#setAsyncLoaderHooks(asyncLoaderHooks); } /** - * Change the currently activate customizations for this module - * loader to be the provided `customizations`. + * Change the currently activate async loader hooks for this module + * loader to be the provided `AsyncLoaderHooks`. * * If present, this class customizes its core functionality to the - * `customizations` object, including registration, loading, and resolving. + * `AsyncLoaderHooks` object, including registration, loading, and resolving. * There are some responsibilities that this class _always_ takes - * care of, like validating outputs, so that the customizations object + * care of, like validating outputs, so that the AsyncLoaderHooks object * does not have to do so. * - * The customizations object has the shape: - * - * ```ts - * interface LoadResult { - * format: ModuleFormat; - * source: ModuleSource; - * } - * - * interface ResolveResult { - * format: string; - * url: URL['href']; - * } - * - * interface Customizations { - * allowImportMetaResolve: boolean; - * load(url: string, context: object): Promise - * resolve( - * originalSpecifier: - * string, parentURL: string, - * importAttributes: Record - * ): Promise - * resolveSync( - * originalSpecifier: - * string, parentURL: string, - * importAttributes: Record - * ) ResolveResult; - * register(specifier: string, parentURL: string): any; - * forceLoadHooks(): void; - * } - * ``` - * - * Note that this class _also_ implements the `Customizations` - * interface, as does `CustomizedModuleLoader` and `Hooks`. - * * Calling this function alters how modules are loaded and should be * invoked with care. - * @param {CustomizedModuleLoader} customizations + * @param {AsyncLoaderHooks} asyncLoaderHooks */ - setCustomizations(customizations) { - this.#customizations = customizations; - if (customizations) { - this.allowImportMetaResolve = customizations.allowImportMetaResolve; + #setAsyncLoaderHooks(asyncLoaderHooks) { + this.#asyncLoaderHooks = asyncLoaderHooks; + if (asyncLoaderHooks) { + this.allowImportMetaResolve = asyncLoaderHooks.allowImportMetaResolve; } else { this.allowImportMetaResolve = true; } @@ -707,18 +678,18 @@ class ModuleLoader { } /** - * @see {@link CustomizedModuleLoader.register} + * @see {@link AsyncLoaderHooks.register} * @returns {any} */ register(specifier, parentURL, data, transferList, isInternal) { - if (!this.#customizations) { - // `CustomizedModuleLoader` is defined at the bottom of this file and - // available well before this line is ever invoked. This is here in - // order to preserve the git diff instead of moving the class. - // eslint-disable-next-line no-use-before-define - this.setCustomizations(new CustomizedModuleLoader()); + if (!this.#asyncLoaderHooks) { + // On the loader hook worker thread, the #asyncLoaderHooks must already have been initialized + // to be an instance of AsyncLoaderHooksOnLoaderHookWorker, so this branch can only ever + // be hit on a non-loader-hook thread that will talk to the loader hook worker thread. + const { AsyncLoaderHooksProxiedToLoaderHookWorker } = require('internal/modules/esm/hooks'); + this.#setAsyncLoaderHooks(new AsyncLoaderHooksProxiedToLoaderHookWorker()); } - return this.#customizations.register(`${specifier}`, `${parentURL}`, data, transferList, isInternal); + return this.#asyncLoaderHooks.register(`${specifier}`, `${parentURL}`, data, transferList, isInternal); } /** @@ -733,12 +704,12 @@ class ModuleLoader { */ resolve(specifier, parentURL, importAttributes) { specifier = `${specifier}`; - if (resolveHooks.length) { + if (syncResolveHooks.length) { // Has module.registerHooks() hooks, use the synchronous variant that can handle both hooks. return this.resolveSync(specifier, parentURL, importAttributes); } - if (this.#customizations) { // Only has module.register hooks. - return this.#customizations.resolve(specifier, parentURL, importAttributes); + if (this.#asyncLoaderHooks) { // Only has module.register hooks. + return this.#asyncLoaderHooks.resolve(specifier, parentURL, importAttributes); } return this.#cachedDefaultResolve(specifier, { __proto__: null, @@ -796,8 +767,8 @@ class ModuleLoader { * @returns {{ format: string, url: string }} */ #resolveAndMaybeBlockOnLoaderThread(specifier, context) { - if (this.#customizations) { - return this.#customizations.resolveSync(specifier, context.parentURL, context.importAttributes); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.resolveSync(specifier, context.parentURL, context.importAttributes); } return this.#cachedDefaultResolve(specifier, context); } @@ -817,10 +788,10 @@ class ModuleLoader { */ resolveSync(specifier, parentURL, importAttributes = { __proto__: null }) { specifier = `${specifier}`; - if (resolveHooks.length) { + if (syncResolveHooks.length) { // Has module.registerHooks() hooks, chain the asynchronous hooks in the default step. - return resolveWithHooks(specifier, parentURL, importAttributes, this.#defaultConditions, - this.#resolveAndMaybeBlockOnLoaderThread.bind(this)); + return resolveWithSyncHooks(specifier, parentURL, importAttributes, this.#defaultConditions, + this.#resolveAndMaybeBlockOnLoaderThread.bind(this)); } return this.#resolveAndMaybeBlockOnLoaderThread(specifier, { __proto__: null, @@ -838,12 +809,12 @@ class ModuleLoader { * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }> | { format: ModuleFormat, source: ModuleSource }} */ load(url, context) { - if (loadHooks.length) { + if (syncLoadHooks.length) { // Has module.registerHooks() hooks, use the synchronous variant that can handle both hooks. return this.#loadSync(url, context); } - if (this.#customizations) { - return this.#customizations.load(url, context); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.load(url, context); } defaultLoadSync ??= require('internal/modules/esm/load').defaultLoadSync; @@ -858,8 +829,8 @@ class ModuleLoader { * @returns {{ format: ModuleFormat, source: ModuleSource }} */ #loadAndMaybeBlockOnLoaderThread(url, context) { - if (this.#customizations) { - return this.#customizations.loadSync(url, context); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.loadSync(url, context); } defaultLoadSync ??= require('internal/modules/esm/load').defaultLoadSync; return defaultLoadSync(url, context); @@ -877,12 +848,12 @@ class ModuleLoader { * @returns {{ format: ModuleFormat, source: ModuleSource }} */ #loadSync(url, context) { - if (loadHooks.length) { + if (syncLoadHooks.length) { // Has module.registerHooks() hooks, chain the asynchronous hooks in the default step. // TODO(joyeecheung): construct the ModuleLoadContext in the loaders directly instead // of converting them from plain objects in the hooks. - return loadWithHooks(url, context.format, context.importAttributes, this.#defaultConditions, - this.#loadAndMaybeBlockOnLoaderThread.bind(this), validateLoadSloppy); + return loadWithSyncHooks(url, context.format, context.importAttributes, this.#defaultConditions, + this.#loadAndMaybeBlockOnLoaderThread.bind(this), validateLoadSloppy); } return this.#loadAndMaybeBlockOnLoaderThread(url, context); } @@ -894,8 +865,8 @@ class ModuleLoader { } importMetaInitialize(meta, context) { - if (this.#customizations) { - return this.#customizations.importMetaInitialize(meta, context, this); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.importMetaInitialize(meta, context, this); } importMetaInitializer ??= require('internal/modules/esm/initialize_import_meta').initializeImportMeta; meta = importMetaInitializer(meta, context, this); @@ -903,94 +874,29 @@ class ModuleLoader { } /** + * Block until the async loader hooks have been initialized. + * * No-op when no hooks have been supplied. */ - forceLoadHooks() { - this.#customizations?.forceLoadHooks(); + waitForAsyncLoaderHookInitialization() { + this.#asyncLoaderHooks?.waitForLoaderHookInitialization(); } } ObjectSetPrototypeOf(ModuleLoader.prototype, null); -class CustomizedModuleLoader { - - allowImportMetaResolve = true; - - /** - * Instantiate a module loader that uses user-provided custom loader hooks. - */ - constructor() { - getHooksProxy(); - } - - /** - * Register some loader specifier. - * @param {string} originalSpecifier The specified URL path of the loader to - * be registered. - * @param {string} parentURL The parent URL from where the loader will be - * registered if using it package name as specifier - * @param {any} [data] Arbitrary data to be passed from the custom loader - * (user-land) to the worker. - * @param {any[]} [transferList] Objects in `data` that are changing ownership - * @param {boolean} [isInternal] For internal loaders that should not be publicly exposed. - * @returns {{ format: string, url: URL['href'] }} - */ - register(originalSpecifier, parentURL, data, transferList, isInternal) { - return hooksProxy.makeSyncRequest('register', transferList, originalSpecifier, parentURL, data, isInternal); - } - - /** - * Resolve the location of the module. - * @param {string} originalSpecifier The specified URL path of the module to - * be resolved. - * @param {string} [parentURL] The URL path of the module's parent. - * @param {ImportAttributes} importAttributes Attributes from the import - * statement or expression. - * @returns {{ format: string, url: URL['href'] }} - */ - resolve(originalSpecifier, parentURL, importAttributes) { - return hooksProxy.makeAsyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); - } - - resolveSync(originalSpecifier, parentURL, importAttributes) { - // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec. - return hooksProxy.makeSyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); - } - - /** - * Provide source that is understood by one of Node's translators. - * @param {URL['href']} url The URL/path of the module to be loaded - * @param {object} [context] Metadata about the module - * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>} - */ - load(url, context) { - return hooksProxy.makeAsyncRequest('load', undefined, url, context); - } - loadSync(url, context) { - return hooksProxy.makeSyncRequest('load', undefined, url, context); - } - - importMetaInitialize(meta, context, loader) { - hooksProxy.importMetaInitialize(meta, context, loader); - } - - forceLoadHooks() { - hooksProxy.waitForWorker(); - } -} - let emittedLoaderFlagWarning = false; /** * A loader instance is used as the main entry point for loading ES modules. Currently, this is a singleton; there is * only one used for loading the main module and everything in its dependency graph, though separate instances of this * class might be instantiated as part of bootstrap for other purposes. + * @param {AsyncLoaderHooksOnLoaderHookWorker|undefined} [asyncLoaderHooks] + * Only provided when run on the loader hook thread. * @returns {ModuleLoader} */ -function createModuleLoader() { - let customizations = null; - // Don't spawn a new worker if custom loaders are disabled. For instance, if - // we're already in a worker thread created by instantiating - // CustomizedModuleLoader; doing so would cause an infinite loop. - if (!forceDefaultLoader()) { +function createModuleLoader(asyncLoaderHooks) { + // Don't spawn a new loader hook worker if we are already in a loader hook worker to avoid infinite recursion. + if (shouldSpawnLoaderHookWorker()) { + assert(asyncLoaderHooks === undefined, 'asyncLoaderHooks should only be provided on the loader hook thread itself'); const userLoaderPaths = getOptionValue('--experimental-loader'); if (userLoaderPaths.length > 0) { if (!emittedLoaderFlagWarning) { @@ -1012,44 +918,37 @@ function createModuleLoader() { ); emittedLoaderFlagWarning = true; } - customizations = new CustomizedModuleLoader(); + const { AsyncLoaderHooksProxiedToLoaderHookWorker } = require('internal/modules/esm/hooks'); + asyncLoaderHooks = new AsyncLoaderHooksProxiedToLoaderHookWorker(); } } - return new ModuleLoader(customizations); -} - - -/** - * Get the HooksProxy instance. If it is not defined, then create a new one. - * @returns {HooksProxy} - */ -function getHooksProxy() { - if (!hooksProxy) { - const { HooksProxy } = require('internal/modules/esm/hooks'); - hooksProxy = new HooksProxy(); - } - - return hooksProxy; + return new ModuleLoader(asyncLoaderHooks); } let cascadedLoader; /** * This is a singleton ESM loader that integrates the loader hooks, if any. - * It it used by other internal built-ins when they need to load ESM code + * It it used by other internal built-ins when they need to load user-land ESM code * while also respecting hooks. * When built-ins need access to this loader, they should do * require('internal/module/esm/loader').getOrInitializeCascadedLoader() * lazily only right before the loader is actually needed, and don't do it * in the top-level, to avoid circular dependencies. + * @param {AsyncLoaderHooksOnLoaderHookWorker|undefined} [asyncLoaderHooks] + * Only provided when run on the loader hook thread. * @returns {ModuleLoader} */ -function getOrInitializeCascadedLoader() { - cascadedLoader ??= createModuleLoader(); +function getOrInitializeCascadedLoader(asyncLoaderHooks) { + cascadedLoader ??= createModuleLoader(asyncLoaderHooks); return cascadedLoader; } +function isCascadedLoaderInitialized() { + return cascadedLoader !== undefined; +} + /** * Register a single loader programmatically. * @param {string|URL} specifier @@ -1094,7 +993,7 @@ function register(specifier, parentURL = undefined, options) { module.exports = { createModuleLoader, - getHooksProxy, getOrInitializeCascadedLoader, + isCascadedLoaderInitialized, register, }; diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js index 4a4279459341e8..a9076a7ae94128 100644 --- a/lib/internal/modules/esm/utils.js +++ b/lib/internal/modules/esm/utils.js @@ -35,13 +35,8 @@ const { ERR_INVALID_ARG_VALUE, } = require('internal/errors').codes; const { getOptionValue } = require('internal/options'); -const { - loadPreloadModules, - initializeFrozenIntrinsics, -} = require('internal/process/pre_execution'); const { emitExperimentalWarning, - getCWDURL, kEmptyObject, } = require('internal/util'); const assert = require('internal/assert'); @@ -283,15 +278,14 @@ async function importModuleDynamicallyCallback(referrerSymbol, specifier, phase, throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING(); } -let _forceDefaultLoader = false; +let _shouldSpawnLoaderHookWorker = true; /** * Initializes handling of ES modules. - * This is configured during pre-execution. Specifically it's set to true for - * the loader worker in internal/main/worker_thread.js. - * @param {boolean} [forceDefaultLoader] - A boolean indicating disabling custom loaders. + * @param {boolean} [shouldSpawnLoaderHookWorker] Whether the custom loader worker + * should be spawned later. */ -function initializeESM(forceDefaultLoader = false) { - _forceDefaultLoader = forceDefaultLoader; +function initializeESM(shouldSpawnLoaderHookWorker = true) { + _shouldSpawnLoaderHookWorker = shouldSpawnLoaderHookWorker; initializeDefaultConditions(); // Setup per-realm callbacks that locate data or callbacks that we keep // track of for different ESM modules. @@ -300,46 +294,12 @@ function initializeESM(forceDefaultLoader = false) { } /** - * Determine whether custom loaders are disabled and it is forced to use the - * default loader. + * Determine whether the custom loader worker should be spawned when initializing + * the singleton ESM loader. * @returns {boolean} */ -function forceDefaultLoader() { - return _forceDefaultLoader; -} - -/** - * Register module customization hooks. - * @returns {Promise} - */ -async function initializeHooks() { - const customLoaderURLs = getOptionValue('--experimental-loader'); - - const { Hooks } = require('internal/modules/esm/hooks'); - const cascadedLoader = require('internal/modules/esm/loader').getOrInitializeCascadedLoader(); - - const hooks = new Hooks(); - cascadedLoader.setCustomizations(hooks); - - // We need the loader customizations to be set _before_ we start invoking - // `--require`, otherwise loops can happen because a `--require` script - // might call `register(...)` before we've installed ourselves. These - // global values are magically set in `setupUserModules` just for us and - // we call them in the correct order. - // N.B. This block appears here specifically in order to ensure that - // `--require` calls occur before `--loader` ones do. - loadPreloadModules(); - initializeFrozenIntrinsics(); - - const parentURL = getCWDURL().href; - for (let i = 0; i < customLoaderURLs.length; i++) { - await hooks.register( - customLoaderURLs[i], - parentURL, - ); - } - - return hooks; +function shouldSpawnLoaderHookWorker() { + return _shouldSpawnLoaderHookWorker; } /** @@ -375,10 +335,8 @@ function compileSourceTextModule(url, source, cascadedLoader, context = kEmptyOb module.exports = { registerModule, initializeESM, - initializeHooks, getDefaultConditions, getConditionsSet, - loaderWorkerId: 'internal/modules/esm/worker', - forceDefaultLoader, + shouldSpawnLoaderHookWorker, compileSourceTextModule, }; diff --git a/lib/internal/modules/esm/worker.js b/lib/internal/modules/esm/worker.js index 6624b740d0a98b..fa5fa87d20efb1 100644 --- a/lib/internal/modules/esm/worker.js +++ b/lib/internal/modules/esm/worker.js @@ -21,12 +21,63 @@ const { isTypedArray, } = require('util/types'); +const { getOptionValue } = require('internal/options'); +const { + loadPreloadModules, + initializeModuleLoaders, + initializeFrozenIntrinsics, +} = require('internal/process/pre_execution'); const { receiveMessageOnPort } = require('internal/worker/io'); const { WORKER_TO_MAIN_THREAD_NOTIFICATION, } = require('internal/modules/esm/shared_constants'); -const { initializeHooks } = require('internal/modules/esm/utils'); const { isMarkedAsUntransferable } = require('internal/buffer'); +const { getCWDURL } = require('internal/util'); +const { isCascadedLoaderInitialized, getOrInitializeCascadedLoader } = require('internal/modules/esm/loader'); +const { AsyncLoaderHooksOnLoaderHookWorker } = require('internal/modules/esm/hooks'); + +/** + * Register asynchronus module loader customization hooks. This should only be run in the loader + * hooks worker. In a non-loader-hooks thread, if any asynchronous loader hook is registered, the + * ModuleLoader#asyncLoaderHooks are initialized to be AsyncLoaderHooksProxiedToLoaderHookWorker + * which posts the messages to the async loader hook worker thread. + * When no asynchronous loader hook is registered, the loader hook worker is not spawned and module + * loading is entiredly done in-thread. + * @returns {Promise} + */ +async function initializeAsyncLoaderHooksOnLoaderHookWorker() { + const customLoaderURLs = getOptionValue('--experimental-loader'); + + // The worker thread spawned for handling asynchronous loader hooks should not + // further spawn other hook threads or there will be an infinite recursion. + const shouldSpawnLoaderHookWorker = false; + // The worker thread for async loader hooks will preload user modules itself in + // initializeAsyncLoaderHooksOnLoaderHookWorker(). + const shouldPreloadModules = false; + initializeModuleLoaders({ shouldSpawnLoaderHookWorker, shouldPreloadModules }); + + assert(!isCascadedLoaderInitialized(), + 'ModuleLoader should be initialized in initializeAsyncLoaderHooksOnLoaderHookWorker()'); + const asyncLoaderHooks = new AsyncLoaderHooksOnLoaderHookWorker(); + getOrInitializeCascadedLoader(asyncLoaderHooks); + + // We need the async loader hooks to be set _before_ we start invoking + // `--require`, otherwise loops can happen because a `--require` script + // might call `register(...)` before we've installed ourselves. These + // global values are magically set in `initializeModuleLoaders` just for us and + // we call them in the correct order. + // N.B. This block appears here specifically in order to ensure that + // `--require` calls occur before `--loader` ones do. + loadPreloadModules(); + initializeFrozenIntrinsics(); + + const parentURL = getCWDURL().href; + for (let i = 0; i < customLoaderURLs.length; i++) { + await asyncLoaderHooks.register(customLoaderURLs[i], parentURL); + } + + return asyncLoaderHooks; +} /** * Transfers an ArrayBuffer, TypedArray, or DataView to a worker thread. @@ -82,7 +133,7 @@ function wrapMessage(status, body) { } /** - * Initializes a worker thread for a customized module loader. + * Initializes the loader hooks worker thread with customized asynchronous module loading hooks. * @param {SharedArrayBuffer} lock - The lock used to synchronize communication between the worker and the main thread. * @param {MessagePort} syncCommPort - The message port used for synchronous communication between the worker and the * main thread. @@ -90,7 +141,7 @@ function wrapMessage(status, body) { * @returns {Promise} A promise that resolves when the worker thread has been initialized. */ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { - let hooks; + let asyncLoaderHooks; let initializationError; let hasInitializationError = false; @@ -106,9 +157,8 @@ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { }; } - try { - hooks = await initializeHooks(); + asyncLoaderHooks = await initializeAsyncLoaderHooksOnLoaderHookWorker(); } catch (exception) { // If there was an error while parsing and executing a user loader, for example if because a // loader contained a syntax error, then we need to send the error to the main thread so it can @@ -178,7 +228,7 @@ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { // the main thread. let hasError = false; let shouldRemoveGlobalErrorHandler = false; - assert(typeof hooks[method] === 'function'); + assert(typeof asyncLoaderHooks[method] === 'function', `${method} is not implemented in the loader worker`); if (port == null && !hasUncaughtExceptionCaptureCallback()) { // When receiving sync messages, we want to unlock the main thread when there's an exception. process.on('uncaughtException', errorHandler); @@ -198,7 +248,7 @@ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { let response; try { - response = await ReflectApply(hooks[method], hooks, args); + response = await ReflectApply(asyncLoaderHooks[method], asyncLoaderHooks, args); } catch (exception) { hasError = true; response = exception; diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 2a9ef56d156808..d337eeca801b9c 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -96,7 +96,7 @@ async function asyncRunEntryPointWithESMLoader(callback) { await cascadedLoader.import(userImports[i], parentURL, kEmptyObject); } } else { - cascadedLoader.forceLoadHooks(); + cascadedLoader.waitForAsyncLoaderHookInitialization(); } await callback(cascadedLoader); } catch (err) { diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index f93b7dae1bca9b..76ec7d821cb4cc 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -47,6 +47,8 @@ function prepareMainThreadExecution(expandArgv1 = false, initializeModules = tru expandArgv1, initializeModules, isMainThread: true, + shouldSpawnLoaderHookWorker: initializeModules, + shouldPreloadModules: initializeModules, }); } @@ -55,15 +57,20 @@ function prepareTestRunnerMainExecution(loadUserModules = true) { expandArgv1: false, initializeModules: true, isMainThread: true, - forceDefaultLoader: !loadUserModules, + shouldSpawnLoaderHookWorker: loadUserModules, + shouldPreloadModules: loadUserModules, }); } function prepareWorkerThreadExecution() { prepareExecution({ expandArgv1: false, - initializeModules: false, isMainThread: false, + // Module loader initialization in workers are delayed until the worker thread + // is ready for execution. + initializeModules: false, + shouldSpawnLoaderHookWorker: false, + shouldPreloadModules: false, }); } @@ -74,7 +81,7 @@ function prepareShadowRealmExecution() { setupDebugEnv(); // Disable custom loaders in ShadowRealm. - setupUserModules(true); + initializeModuleLoaders({ shouldSpawnLoaderHookWorker: false, shouldPreloadModules: false }); const { privateSymbols: { host_defined_option_symbol, @@ -96,7 +103,7 @@ function prepareShadowRealmExecution() { } function prepareExecution(options) { - const { expandArgv1, initializeModules, isMainThread, forceDefaultLoader } = options; + const { expandArgv1, initializeModules, isMainThread, shouldSpawnLoaderHookWorker, shouldPreloadModules } = options; refreshRuntimeOptions(); @@ -154,8 +161,9 @@ function prepareExecution(options) { assert(!initializeModules); } + setupVmModules(); if (initializeModules) { - setupUserModules(forceDefaultLoader); + initializeModuleLoaders({ shouldSpawnLoaderHookWorker, shouldPreloadModules }); } // This has to be done after the user module loader is initialized, @@ -165,6 +173,21 @@ function prepareExecution(options) { return mainEntry; } +function setupVmModules() { + // Patch the vm module when --experimental-vm-modules is on. + // Please update the comments in vm.js when this block changes. + // TODO(joyeecheung): move this to vm.js? + if (getOptionValue('--experimental-vm-modules')) { + const { + Module, SourceTextModule, SyntheticModule, + } = require('internal/vm/module'); + const vm = require('vm'); + vm.Module = Module; + vm.SourceTextModule = SourceTextModule; + vm.SyntheticModule = SyntheticModule; + } +} + function setupHttpProxy() { // This normalized from both --use-env-proxy and NODE_USE_ENV_PROXY settings. if (!getOptionValue('--use-env-proxy')) { @@ -186,22 +209,32 @@ function setupHttpProxy() { // existing libraries that sets the global dispatcher or monkey patches the global agent. } -function setupUserModules(forceDefaultLoader = false) { - initializeCJSLoader(); - initializeESMLoader(forceDefaultLoader); +function initializeModuleLoaders(options) { + const { shouldSpawnLoaderHookWorker, shouldPreloadModules } = options; + // Initialize certain special module.Module properties and the CJS conditions. + const { initializeCJS } = require('internal/modules/cjs/loader'); + initializeCJS(); + // Initialize the ESM loader and a few module callbacks. + // If shouldSpawnLoaderHookWorker is true, later when the ESM loader is instantiated on-demand, + // it will spawn a loader worker thread to handle async custom loader hooks. + const { initializeESM } = require('internal/modules/esm/utils'); + initializeESM(shouldSpawnLoaderHookWorker); + const { hasStartedUserCJSExecution, hasStartedUserESMExecution, } = require('internal/modules/helpers'); + // At this point, no user module has been executed yet. assert(!hasStartedUserCJSExecution()); assert(!hasStartedUserESMExecution()); + if (getEmbedderOptions().hasEmbedderPreload) { runEmbedderPreload(); } // Do not enable preload modules if custom loaders are disabled. // For example, loader workers are responsible for doing this themselves. // And preload modules are not supported in ShadowRealm as well. - if (!forceDefaultLoader) { + if (shouldPreloadModules) { loadPreloadModules(); } // Need to be done after --require setup. @@ -638,28 +671,6 @@ function initializePermission() { } } -function initializeCJSLoader() { - const { initializeCJS } = require('internal/modules/cjs/loader'); - initializeCJS(); -} - -function initializeESMLoader(forceDefaultLoader) { - const { initializeESM } = require('internal/modules/esm/utils'); - initializeESM(forceDefaultLoader); - - // Patch the vm module when --experimental-vm-modules is on. - // Please update the comments in vm.js when this block changes. - if (getOptionValue('--experimental-vm-modules')) { - const { - Module, SourceTextModule, SyntheticModule, - } = require('internal/vm/module'); - const vm = require('vm'); - vm.Module = Module; - vm.SourceTextModule = SourceTextModule; - vm.SyntheticModule = SyntheticModule; - } -} - function initializeSourceMapsHandlers() { const { setSourceMapsSupport, @@ -729,7 +740,7 @@ function getHeapSnapshotFilename(diagnosticDir) { } module.exports = { - setupUserModules, + initializeModuleLoaders, prepareMainThreadExecution, prepareWorkerThreadExecution, prepareShadowRealmExecution, diff --git a/test/es-module/test-esm-loader-modulemap.js b/test/es-module/test-esm-loader-modulemap.js index 83125fce738139..f581c7f507640e 100644 --- a/test/es-module/test-esm-loader-modulemap.js +++ b/test/es-module/test-esm-loader-modulemap.js @@ -16,7 +16,7 @@ const jsonModuleDataUrl = 'data:application/json,""'; const stubJsModule = createDynamicModule([], ['default'], jsModuleDataUrl); const stubJsonModule = createDynamicModule([], ['default'], jsonModuleDataUrl); -const loader = createModuleLoader(false); +const loader = createModuleLoader(); const jsModuleJob = new ModuleJob(loader, stubJsModule.module, undefined, () => new Promise(() => {})); const jsonModuleJob = new ModuleJob(loader, stubJsonModule.module, diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index a66a3ec9ce360f..b050f5bffde04a 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -107,15 +107,15 @@ expected.beforePreExec = new Set([ 'NativeModule internal/modules/typescript', 'NativeModule internal/data_url', 'NativeModule internal/mime', -]); - -expected.atRunTime = new Set([ + 'NativeModule internal/modules/esm/utils', 'Internal Binding worker', 'NativeModule internal/modules/run_main', 'NativeModule internal/net', 'NativeModule internal/dns/utils', +]); + +expected.atRunTime = new Set([ 'NativeModule internal/process/pre_execution', - 'NativeModule internal/modules/esm/utils', ]); const { isMainThread } = require('worker_threads'); @@ -173,7 +173,7 @@ if (common.hasIntl) { if (process.features.inspector) { expected.beforePreExec.add('Internal Binding inspector'); expected.beforePreExec.add('NativeModule internal/util/inspector'); - expected.atRunTime.add('NativeModule internal/inspector_async_hook'); + expected.beforePreExec.add('NativeModule internal/inspector_async_hook'); } // This is loaded if the test is run with NODE_V8_COVERAGE.