diff --git a/lib/internal/main/worker_thread.js b/lib/internal/main/worker_thread.js index 62dc44cf34b739..1e28a3452220e9 100644 --- a/lib/internal/main/worker_thread.js +++ b/lib/internal/main/worker_thread.js @@ -11,7 +11,6 @@ const { ObjectDefineProperty, PromisePrototypeThen, RegExpPrototypeExec, - SafeWeakMap, globalThis: { SharedArrayBuffer, }, @@ -19,7 +18,7 @@ const { const { prepareWorkerThreadExecution, - setupUserModules, + initializeModuleLoaders, markBootstrapComplete, } = require('internal/process/pre_execution'); @@ -138,11 +137,13 @@ port.on('message', (message) => { workerIo.sharedCwdCounter = cwdCounter; } - const isLoaderWorker = - doEval === 'internal' && - filename === require('internal/modules/esm/utils').loaderWorkerId; - // Disable custom loaders in loader worker. - setupUserModules(isLoaderWorker); + const isLoaderHookWorker = (filename === 'internal/modules/esm/worker' && doEval === 'internal'); + if (!isLoaderHookWorker) { + // If we are in the loader hook worker, delay the module loader initializations until + // initializeAsyncLoaderHooksOnLoaderHookWorker() which needs to run preloads + // after the asynchronous loader hooks are registered. + initializeModuleLoaders({ shouldSpawnLoaderHookWorker: true, shouldPreloadModules: true }); + } if (!hasStdin) process.stdin.push(null); @@ -152,9 +153,10 @@ port.on('message', (message) => { port.postMessage({ type: UP_AND_RUNNING }); switch (doEval) { case 'internal': { - // Create this WeakMap in js-land because V8 has no C++ API for WeakMap. - internalBinding('module_wrap').callbackMap = new SafeWeakMap(); - require(filename)(workerData, publicPort); + // Currently the only user of internal eval is the async loader hook thread. + assert(isLoaderHookWorker, `Unexpected internal eval ${filename}`); + const setupModuleWorker = require('internal/modules/esm/worker'); + setupModuleWorker(workerData, publicPort); break; } diff --git a/lib/internal/modules/esm/hooks.js b/lib/internal/modules/esm/hooks.js index e3ba5fa86252f5..cc66d47a43b704 100644 --- a/lib/internal/modules/esm/hooks.js +++ b/lib/internal/modules/esm/hooks.js @@ -54,7 +54,6 @@ const { } = require('internal/modules/esm/resolve'); const { getDefaultConditions, - loaderWorkerId, } = require('internal/modules/esm/utils'); const { deserializeError } = require('internal/error_serdes'); const { @@ -105,7 +104,39 @@ function defineImportAssertionAlias(context) { // [2] `validate...()`s throw the wrong error -class Hooks { +/** + * @typedef {{ format: ModuleFormat, source: ModuleSource }} LoadResult + */ + +/** + * @typedef {{ format: ModuleFormat, url: string, importAttributes: Record }} ResolveResult + */ + +/** + * Interface for classes that implement asynchronous loader hooks that can be attached to the ModuleLoader + * via `ModuleLoader.#setAsyncLoaderHooks()`. + * @typedef {object} AsyncLoaderHooks + * @property {boolean} allowImportMetaResolve Whether to allow the use of `import.meta.resolve`. + * @property {(url: string, context: object, defaultLoad: Function) => Promise} load + * Calling the asynchronous `load` hook asynchronously. + * @property {(url: string, context: object, defaultLoad: Function) => LoadResult} loadSync + * Calling the asynchronous `load` hook synchronously. + * @property {(originalSpecifier: string, parentURL: string, + * importAttributes: Record) => Promise} resolve + * Calling the asynchronous `resolve` hook asynchronously. + * @property {(originalSpecifier: string, parentURL: string, + * importAttributes: Record) => ResolveResult} resolveSync + * Calling the asynchronous `resolve` hook synchronously. + * @property {(specifier: string, parentURL: string) => any} register Register asynchronous loader hooks + * @property {() => void} waitForLoaderHookInitialization Force loading of hooks. + */ + +/** + * @implements {AsyncLoaderHooks} + * Instances of this class run directly on the loader hook worker thread and customize the module + * loading of the hooks worker itself. + */ +class AsyncLoaderHooksOnLoaderHookWorker { #chains = { /** * Phase 1 of 2 in ESM loading. @@ -452,7 +483,7 @@ class Hooks { }; } - forceLoadHooks() { + waitForLoaderHookInitialization() { // No-op } @@ -462,14 +493,20 @@ class Hooks { return meta; } } -ObjectSetPrototypeOf(Hooks.prototype, null); +ObjectSetPrototypeOf(AsyncLoaderHooksOnLoaderHookWorker.prototype, null); /** - * There may be multiple instances of Hooks/HooksProxy, but there is only 1 Internal worker, so - * there is only 1 MessageChannel. + * There is only one loader hook thread for each non-loader-hook worker thread + * (i.e. the non-loader-hook thread and any worker threads that are not loader hook workers themselves), + * so there is only 1 MessageChannel. */ let MessageChannel; -class HooksProxy { + +/** + * Abstraction over a worker thread that runs the asynchronous module loader hooks. + * Instances of this class run on the non-loader-hook thread and communicate with the loader hooks worker thread. + */ +class AsyncLoaderHookWorker { /** * Shared memory. Always use Atomics method to read or write to it. * @type {Int32Array} @@ -503,7 +540,7 @@ class HooksProxy { const lock = new SharedArrayBuffer(SHARED_MEMORY_BYTE_LENGTH); this.#lock = new Int32Array(lock); - this.#worker = new InternalWorker(loaderWorkerId, { + this.#worker = new InternalWorker('internal/modules/esm/worker', { stderr: false, stdin: false, stdout: false, @@ -644,7 +681,7 @@ class HooksProxy { this.#importMetaInitializer(meta, context, loader); } } -ObjectSetPrototypeOf(HooksProxy.prototype, null); +ObjectSetPrototypeOf(AsyncLoaderHookWorker.prototype, null); // TODO(JakobJingleheimer): Remove this when loaders go "stable". let globalPreloadWarningWasEmitted = false; @@ -757,6 +794,95 @@ function nextHookFactory(current, meta, { validateArgs, validateOutput }) { ); } +/** + * @type {AsyncLoaderHookWorker} + * Worker instance used to run async loader hooks in a separate thread. This is a singleton for each + * non-loader-hook worker thread (i.e. the main thread and any worker threads that are not + * loader hook workers themselves). + */ +let asyncLoaderHookWorker; +/** + * Get the AsyncLoaderHookWorker instance. If it is not defined, then create a new one. + * @returns {AsyncLoaderHookWorker} + */ +function getAsyncLoaderHookWorker() { + asyncLoaderHookWorker ??= new AsyncLoaderHookWorker(); + return asyncLoaderHookWorker; +} + +/** + * @implements {AsyncLoaderHooks} + * Instances of this class are created in the non-loader-hook thread and communicate with the worker thread + * spawned to run the async loader hooks. + */ +class AsyncLoaderHooksProxiedToLoaderHookWorker { + + allowImportMetaResolve = true; + + /** + * Instantiate a module loader that uses user-provided custom loader hooks. + */ + constructor() { + getAsyncLoaderHookWorker(); + } + + /** + * Register some loader specifier. + * @param {string} originalSpecifier The specified URL path of the loader to + * be registered. + * @param {string} parentURL The parent URL from where the loader will be + * registered if using it package name as specifier + * @param {any} [data] Arbitrary data to be passed from the custom loader + * (user-land) to the worker. + * @param {any[]} [transferList] Objects in `data` that are changing ownership + * @param {boolean} [isInternal] For internal loaders that should not be publicly exposed. + * @returns {{ format: string, url: URL['href'] }} + */ + register(originalSpecifier, parentURL, data, transferList, isInternal) { + return asyncLoaderHookWorker.makeSyncRequest('register', transferList, originalSpecifier, parentURL, + data, isInternal); + } + + /** + * Resolve the location of the module. + * @param {string} originalSpecifier The specified URL path of the module to + * be resolved. + * @param {string} [parentURL] The URL path of the module's parent. + * @param {ImportAttributes} importAttributes Attributes from the import + * statement or expression. + * @returns {{ format: string, url: URL['href'] }} + */ + resolve(originalSpecifier, parentURL, importAttributes) { + return asyncLoaderHookWorker.makeAsyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); + } + + resolveSync(originalSpecifier, parentURL, importAttributes) { + // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec. + return asyncLoaderHookWorker.makeSyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); + } + + /** + * Provide source that is understood by one of Node's translators. + * @param {URL['href']} url The URL/path of the module to be loaded + * @param {object} [context] Metadata about the module + * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>} + */ + load(url, context) { + return asyncLoaderHookWorker.makeAsyncRequest('load', undefined, url, context); + } + loadSync(url, context) { + return asyncLoaderHookWorker.makeSyncRequest('load', undefined, url, context); + } + + importMetaInitialize(meta, context, loader) { + asyncLoaderHookWorker.importMetaInitialize(meta, context, loader); + } + + waitForLoaderHookInitialization() { + asyncLoaderHookWorker.waitForWorker(); + } +} -exports.Hooks = Hooks; -exports.HooksProxy = HooksProxy; +exports.AsyncLoaderHooksProxiedToLoaderHookWorker = AsyncLoaderHooksProxiedToLoaderHookWorker; +exports.AsyncLoaderHooksOnLoaderHookWorker = AsyncLoaderHooksOnLoaderHookWorker; +exports.AsyncLoaderHookWorker = AsyncLoaderHookWorker; diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 9cb02f05c8d28a..d536d8215c93ab 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -34,7 +34,7 @@ const { kEmptyObject } = require('internal/util'); const { compileSourceTextModule, getDefaultConditions, - forceDefaultLoader, + shouldSpawnLoaderHookWorker, } = require('internal/modules/esm/utils'); const { kImplicitTypeAttribute } = require('internal/modules/esm/assert'); const { @@ -51,10 +51,10 @@ const { urlToFilename, } = require('internal/modules/helpers'); const { - resolveHooks, - resolveWithHooks, - loadHooks, - loadWithHooks, + resolveHooks: syncResolveHooks, + resolveWithHooks: resolveWithSyncHooks, + loadHooks: syncLoadHooks, + loadWithHooks: loadWithSyncHooks, validateLoadSloppy, } = require('internal/modules/customization_hooks'); let defaultResolve, defaultLoadSync, importMetaInitializer; @@ -69,7 +69,7 @@ let debug = require('internal/util/debuglog').debuglog('esm', (fn) => { const { isPromise } = require('internal/util/types'); /** - * @typedef {import('./hooks.js').HooksProxy} HooksProxy + * @typedef {import('./hooks.js').AsyncLoaderHookWorker} AsyncLoaderHookWorker * @typedef {import('./module_job.js').ModuleJobBase} ModuleJobBase * @typedef {import('url').URL} URL */ @@ -123,14 +123,6 @@ function getRaceMessage(filename, parentFilename) { return raceMessage; } -/** - * @type {HooksProxy} - * Multiple loader instances exist for various, specific reasons (see code comments at site). - * In order to maintain consistency, we use a single worker (sandbox), which must sit apart of an - * individual loader instance. - */ -let hooksProxy; - /** * @typedef {import('../cjs/loader.js').Module} CJSModule */ @@ -152,9 +144,17 @@ let hooksProxy; */ /** - * This class covers the base machinery of module loading. To add custom - * behavior you can pass a customizations object and this object will be - * used to do the loading/resolving/registration process. + * @typedef {import('./hooks.js').AsyncLoaderHooks} AsyncLoaderHooks + * @typedef {import('./hooks.js').AsyncLoaderHooksOnLoaderHookWorker} AsyncLoaderHooksOnLoaderHookWorker + * @typedef {import('./hooks.js').AsyncLoaderHooksProxiedToLoaderHookWorker} AsyncLoaderHooksProxiedToLoaderHookWorker + */ + +/** + * This class covers the base machinery of module loading. There are two types of loader hooks: + * 1. Asynchronous loader hooks, which are run in a separate loader hook worker thread. + * This is configured in #asyncLoaderHooks. + * 2. Synchronous loader hooks, which are run in-thread. This is shared with the CJS loader and is + * stored in the cross-module syncResolveHooks and syncLoadHooks arrays. */ class ModuleLoader { /** @@ -185,73 +185,44 @@ class ModuleLoader { allowImportMetaResolve; /** - * Customizations to pass requests to. - * @type {import('./hooks.js').Hooks} - * Note that this value _MUST_ be set with `setCustomizations` - * because it needs to copy `customizations.allowImportMetaResolve` + * Asynchronous loader hooks to pass requests to. + * + * Note that this value _MUST_ be set with `#setAsyncLoaderHooks` + * because it needs to copy `#asyncLoaderHooks.allowImportMetaResolve` * to this property and failure to do so will cause undefined * behavior when invoking `import.meta.resolve`. - * @see {ModuleLoader.setCustomizations} - * @type {CustomizedModuleLoader} + * + * When the ModuleLoader is created on a loader hook thread, this is + * {@link AsyncLoaderHooksOnLoaderHookWorker}, and its methods directly call out + * to loader methods. Otherwise, this is {@link AsyncLoaderHooksProxiedToLoaderHookWorker}, + * and its methods post messages to the loader thread and possibly block on it. + * @see {ModuleLoader.#setAsyncLoaderHooks} + * @type {AsyncLoaderHooks} */ - #customizations; + #asyncLoaderHooks; - constructor(customizations) { - this.setCustomizations(customizations); + constructor(asyncLoaderHooks) { + this.#setAsyncLoaderHooks(asyncLoaderHooks); } /** - * Change the currently activate customizations for this module - * loader to be the provided `customizations`. + * Change the currently activate async loader hooks for this module + * loader to be the provided `AsyncLoaderHooks`. * * If present, this class customizes its core functionality to the - * `customizations` object, including registration, loading, and resolving. + * `AsyncLoaderHooks` object, including registration, loading, and resolving. * There are some responsibilities that this class _always_ takes - * care of, like validating outputs, so that the customizations object + * care of, like validating outputs, so that the AsyncLoaderHooks object * does not have to do so. * - * The customizations object has the shape: - * - * ```ts - * interface LoadResult { - * format: ModuleFormat; - * source: ModuleSource; - * } - * - * interface ResolveResult { - * format: string; - * url: URL['href']; - * } - * - * interface Customizations { - * allowImportMetaResolve: boolean; - * load(url: string, context: object): Promise - * resolve( - * originalSpecifier: - * string, parentURL: string, - * importAttributes: Record - * ): Promise - * resolveSync( - * originalSpecifier: - * string, parentURL: string, - * importAttributes: Record - * ) ResolveResult; - * register(specifier: string, parentURL: string): any; - * forceLoadHooks(): void; - * } - * ``` - * - * Note that this class _also_ implements the `Customizations` - * interface, as does `CustomizedModuleLoader` and `Hooks`. - * * Calling this function alters how modules are loaded and should be * invoked with care. - * @param {CustomizedModuleLoader} customizations + * @param {AsyncLoaderHooks} asyncLoaderHooks */ - setCustomizations(customizations) { - this.#customizations = customizations; - if (customizations) { - this.allowImportMetaResolve = customizations.allowImportMetaResolve; + #setAsyncLoaderHooks(asyncLoaderHooks) { + this.#asyncLoaderHooks = asyncLoaderHooks; + if (asyncLoaderHooks) { + this.allowImportMetaResolve = asyncLoaderHooks.allowImportMetaResolve; } else { this.allowImportMetaResolve = true; } @@ -707,18 +678,18 @@ class ModuleLoader { } /** - * @see {@link CustomizedModuleLoader.register} + * @see {@link AsyncLoaderHooks.register} * @returns {any} */ register(specifier, parentURL, data, transferList, isInternal) { - if (!this.#customizations) { - // `CustomizedModuleLoader` is defined at the bottom of this file and - // available well before this line is ever invoked. This is here in - // order to preserve the git diff instead of moving the class. - // eslint-disable-next-line no-use-before-define - this.setCustomizations(new CustomizedModuleLoader()); + if (!this.#asyncLoaderHooks) { + // On the loader hook worker thread, the #asyncLoaderHooks must already have been initialized + // to be an instance of AsyncLoaderHooksOnLoaderHookWorker, so this branch can only ever + // be hit on a non-loader-hook thread that will talk to the loader hook worker thread. + const { AsyncLoaderHooksProxiedToLoaderHookWorker } = require('internal/modules/esm/hooks'); + this.#setAsyncLoaderHooks(new AsyncLoaderHooksProxiedToLoaderHookWorker()); } - return this.#customizations.register(`${specifier}`, `${parentURL}`, data, transferList, isInternal); + return this.#asyncLoaderHooks.register(`${specifier}`, `${parentURL}`, data, transferList, isInternal); } /** @@ -733,12 +704,12 @@ class ModuleLoader { */ resolve(specifier, parentURL, importAttributes) { specifier = `${specifier}`; - if (resolveHooks.length) { + if (syncResolveHooks.length) { // Has module.registerHooks() hooks, use the synchronous variant that can handle both hooks. return this.resolveSync(specifier, parentURL, importAttributes); } - if (this.#customizations) { // Only has module.register hooks. - return this.#customizations.resolve(specifier, parentURL, importAttributes); + if (this.#asyncLoaderHooks) { // Only has module.register hooks. + return this.#asyncLoaderHooks.resolve(specifier, parentURL, importAttributes); } return this.#cachedDefaultResolve(specifier, { __proto__: null, @@ -796,8 +767,8 @@ class ModuleLoader { * @returns {{ format: string, url: string }} */ #resolveAndMaybeBlockOnLoaderThread(specifier, context) { - if (this.#customizations) { - return this.#customizations.resolveSync(specifier, context.parentURL, context.importAttributes); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.resolveSync(specifier, context.parentURL, context.importAttributes); } return this.#cachedDefaultResolve(specifier, context); } @@ -817,10 +788,10 @@ class ModuleLoader { */ resolveSync(specifier, parentURL, importAttributes = { __proto__: null }) { specifier = `${specifier}`; - if (resolveHooks.length) { + if (syncResolveHooks.length) { // Has module.registerHooks() hooks, chain the asynchronous hooks in the default step. - return resolveWithHooks(specifier, parentURL, importAttributes, this.#defaultConditions, - this.#resolveAndMaybeBlockOnLoaderThread.bind(this)); + return resolveWithSyncHooks(specifier, parentURL, importAttributes, this.#defaultConditions, + this.#resolveAndMaybeBlockOnLoaderThread.bind(this)); } return this.#resolveAndMaybeBlockOnLoaderThread(specifier, { __proto__: null, @@ -838,12 +809,12 @@ class ModuleLoader { * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }> | { format: ModuleFormat, source: ModuleSource }} */ load(url, context) { - if (loadHooks.length) { + if (syncLoadHooks.length) { // Has module.registerHooks() hooks, use the synchronous variant that can handle both hooks. return this.#loadSync(url, context); } - if (this.#customizations) { - return this.#customizations.load(url, context); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.load(url, context); } defaultLoadSync ??= require('internal/modules/esm/load').defaultLoadSync; @@ -858,8 +829,8 @@ class ModuleLoader { * @returns {{ format: ModuleFormat, source: ModuleSource }} */ #loadAndMaybeBlockOnLoaderThread(url, context) { - if (this.#customizations) { - return this.#customizations.loadSync(url, context); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.loadSync(url, context); } defaultLoadSync ??= require('internal/modules/esm/load').defaultLoadSync; return defaultLoadSync(url, context); @@ -877,12 +848,12 @@ class ModuleLoader { * @returns {{ format: ModuleFormat, source: ModuleSource }} */ #loadSync(url, context) { - if (loadHooks.length) { + if (syncLoadHooks.length) { // Has module.registerHooks() hooks, chain the asynchronous hooks in the default step. // TODO(joyeecheung): construct the ModuleLoadContext in the loaders directly instead // of converting them from plain objects in the hooks. - return loadWithHooks(url, context.format, context.importAttributes, this.#defaultConditions, - this.#loadAndMaybeBlockOnLoaderThread.bind(this), validateLoadSloppy); + return loadWithSyncHooks(url, context.format, context.importAttributes, this.#defaultConditions, + this.#loadAndMaybeBlockOnLoaderThread.bind(this), validateLoadSloppy); } return this.#loadAndMaybeBlockOnLoaderThread(url, context); } @@ -894,8 +865,8 @@ class ModuleLoader { } importMetaInitialize(meta, context) { - if (this.#customizations) { - return this.#customizations.importMetaInitialize(meta, context, this); + if (this.#asyncLoaderHooks) { + return this.#asyncLoaderHooks.importMetaInitialize(meta, context, this); } importMetaInitializer ??= require('internal/modules/esm/initialize_import_meta').initializeImportMeta; meta = importMetaInitializer(meta, context, this); @@ -903,94 +874,29 @@ class ModuleLoader { } /** + * Block until the async loader hooks have been initialized. + * * No-op when no hooks have been supplied. */ - forceLoadHooks() { - this.#customizations?.forceLoadHooks(); + waitForAsyncLoaderHookInitialization() { + this.#asyncLoaderHooks?.waitForLoaderHookInitialization(); } } ObjectSetPrototypeOf(ModuleLoader.prototype, null); -class CustomizedModuleLoader { - - allowImportMetaResolve = true; - - /** - * Instantiate a module loader that uses user-provided custom loader hooks. - */ - constructor() { - getHooksProxy(); - } - - /** - * Register some loader specifier. - * @param {string} originalSpecifier The specified URL path of the loader to - * be registered. - * @param {string} parentURL The parent URL from where the loader will be - * registered if using it package name as specifier - * @param {any} [data] Arbitrary data to be passed from the custom loader - * (user-land) to the worker. - * @param {any[]} [transferList] Objects in `data` that are changing ownership - * @param {boolean} [isInternal] For internal loaders that should not be publicly exposed. - * @returns {{ format: string, url: URL['href'] }} - */ - register(originalSpecifier, parentURL, data, transferList, isInternal) { - return hooksProxy.makeSyncRequest('register', transferList, originalSpecifier, parentURL, data, isInternal); - } - - /** - * Resolve the location of the module. - * @param {string} originalSpecifier The specified URL path of the module to - * be resolved. - * @param {string} [parentURL] The URL path of the module's parent. - * @param {ImportAttributes} importAttributes Attributes from the import - * statement or expression. - * @returns {{ format: string, url: URL['href'] }} - */ - resolve(originalSpecifier, parentURL, importAttributes) { - return hooksProxy.makeAsyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); - } - - resolveSync(originalSpecifier, parentURL, importAttributes) { - // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec. - return hooksProxy.makeSyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes); - } - - /** - * Provide source that is understood by one of Node's translators. - * @param {URL['href']} url The URL/path of the module to be loaded - * @param {object} [context] Metadata about the module - * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>} - */ - load(url, context) { - return hooksProxy.makeAsyncRequest('load', undefined, url, context); - } - loadSync(url, context) { - return hooksProxy.makeSyncRequest('load', undefined, url, context); - } - - importMetaInitialize(meta, context, loader) { - hooksProxy.importMetaInitialize(meta, context, loader); - } - - forceLoadHooks() { - hooksProxy.waitForWorker(); - } -} - let emittedLoaderFlagWarning = false; /** * A loader instance is used as the main entry point for loading ES modules. Currently, this is a singleton; there is * only one used for loading the main module and everything in its dependency graph, though separate instances of this * class might be instantiated as part of bootstrap for other purposes. + * @param {AsyncLoaderHooksOnLoaderHookWorker|undefined} [asyncLoaderHooks] + * Only provided when run on the loader hook thread. * @returns {ModuleLoader} */ -function createModuleLoader() { - let customizations = null; - // Don't spawn a new worker if custom loaders are disabled. For instance, if - // we're already in a worker thread created by instantiating - // CustomizedModuleLoader; doing so would cause an infinite loop. - if (!forceDefaultLoader()) { +function createModuleLoader(asyncLoaderHooks) { + // Don't spawn a new loader hook worker if we are already in a loader hook worker to avoid infinite recursion. + if (shouldSpawnLoaderHookWorker()) { + assert(asyncLoaderHooks === undefined, 'asyncLoaderHooks should only be provided on the loader hook thread itself'); const userLoaderPaths = getOptionValue('--experimental-loader'); if (userLoaderPaths.length > 0) { if (!emittedLoaderFlagWarning) { @@ -1012,44 +918,37 @@ function createModuleLoader() { ); emittedLoaderFlagWarning = true; } - customizations = new CustomizedModuleLoader(); + const { AsyncLoaderHooksProxiedToLoaderHookWorker } = require('internal/modules/esm/hooks'); + asyncLoaderHooks = new AsyncLoaderHooksProxiedToLoaderHookWorker(); } } - return new ModuleLoader(customizations); -} - - -/** - * Get the HooksProxy instance. If it is not defined, then create a new one. - * @returns {HooksProxy} - */ -function getHooksProxy() { - if (!hooksProxy) { - const { HooksProxy } = require('internal/modules/esm/hooks'); - hooksProxy = new HooksProxy(); - } - - return hooksProxy; + return new ModuleLoader(asyncLoaderHooks); } let cascadedLoader; /** * This is a singleton ESM loader that integrates the loader hooks, if any. - * It it used by other internal built-ins when they need to load ESM code + * It it used by other internal built-ins when they need to load user-land ESM code * while also respecting hooks. * When built-ins need access to this loader, they should do * require('internal/module/esm/loader').getOrInitializeCascadedLoader() * lazily only right before the loader is actually needed, and don't do it * in the top-level, to avoid circular dependencies. + * @param {AsyncLoaderHooksOnLoaderHookWorker|undefined} [asyncLoaderHooks] + * Only provided when run on the loader hook thread. * @returns {ModuleLoader} */ -function getOrInitializeCascadedLoader() { - cascadedLoader ??= createModuleLoader(); +function getOrInitializeCascadedLoader(asyncLoaderHooks) { + cascadedLoader ??= createModuleLoader(asyncLoaderHooks); return cascadedLoader; } +function isCascadedLoaderInitialized() { + return cascadedLoader !== undefined; +} + /** * Register a single loader programmatically. * @param {string|URL} specifier @@ -1094,7 +993,7 @@ function register(specifier, parentURL = undefined, options) { module.exports = { createModuleLoader, - getHooksProxy, getOrInitializeCascadedLoader, + isCascadedLoaderInitialized, register, }; diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js index 4a4279459341e8..a9076a7ae94128 100644 --- a/lib/internal/modules/esm/utils.js +++ b/lib/internal/modules/esm/utils.js @@ -35,13 +35,8 @@ const { ERR_INVALID_ARG_VALUE, } = require('internal/errors').codes; const { getOptionValue } = require('internal/options'); -const { - loadPreloadModules, - initializeFrozenIntrinsics, -} = require('internal/process/pre_execution'); const { emitExperimentalWarning, - getCWDURL, kEmptyObject, } = require('internal/util'); const assert = require('internal/assert'); @@ -283,15 +278,14 @@ async function importModuleDynamicallyCallback(referrerSymbol, specifier, phase, throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING(); } -let _forceDefaultLoader = false; +let _shouldSpawnLoaderHookWorker = true; /** * Initializes handling of ES modules. - * This is configured during pre-execution. Specifically it's set to true for - * the loader worker in internal/main/worker_thread.js. - * @param {boolean} [forceDefaultLoader] - A boolean indicating disabling custom loaders. + * @param {boolean} [shouldSpawnLoaderHookWorker] Whether the custom loader worker + * should be spawned later. */ -function initializeESM(forceDefaultLoader = false) { - _forceDefaultLoader = forceDefaultLoader; +function initializeESM(shouldSpawnLoaderHookWorker = true) { + _shouldSpawnLoaderHookWorker = shouldSpawnLoaderHookWorker; initializeDefaultConditions(); // Setup per-realm callbacks that locate data or callbacks that we keep // track of for different ESM modules. @@ -300,46 +294,12 @@ function initializeESM(forceDefaultLoader = false) { } /** - * Determine whether custom loaders are disabled and it is forced to use the - * default loader. + * Determine whether the custom loader worker should be spawned when initializing + * the singleton ESM loader. * @returns {boolean} */ -function forceDefaultLoader() { - return _forceDefaultLoader; -} - -/** - * Register module customization hooks. - * @returns {Promise} - */ -async function initializeHooks() { - const customLoaderURLs = getOptionValue('--experimental-loader'); - - const { Hooks } = require('internal/modules/esm/hooks'); - const cascadedLoader = require('internal/modules/esm/loader').getOrInitializeCascadedLoader(); - - const hooks = new Hooks(); - cascadedLoader.setCustomizations(hooks); - - // We need the loader customizations to be set _before_ we start invoking - // `--require`, otherwise loops can happen because a `--require` script - // might call `register(...)` before we've installed ourselves. These - // global values are magically set in `setupUserModules` just for us and - // we call them in the correct order. - // N.B. This block appears here specifically in order to ensure that - // `--require` calls occur before `--loader` ones do. - loadPreloadModules(); - initializeFrozenIntrinsics(); - - const parentURL = getCWDURL().href; - for (let i = 0; i < customLoaderURLs.length; i++) { - await hooks.register( - customLoaderURLs[i], - parentURL, - ); - } - - return hooks; +function shouldSpawnLoaderHookWorker() { + return _shouldSpawnLoaderHookWorker; } /** @@ -375,10 +335,8 @@ function compileSourceTextModule(url, source, cascadedLoader, context = kEmptyOb module.exports = { registerModule, initializeESM, - initializeHooks, getDefaultConditions, getConditionsSet, - loaderWorkerId: 'internal/modules/esm/worker', - forceDefaultLoader, + shouldSpawnLoaderHookWorker, compileSourceTextModule, }; diff --git a/lib/internal/modules/esm/worker.js b/lib/internal/modules/esm/worker.js index 6624b740d0a98b..fa5fa87d20efb1 100644 --- a/lib/internal/modules/esm/worker.js +++ b/lib/internal/modules/esm/worker.js @@ -21,12 +21,63 @@ const { isTypedArray, } = require('util/types'); +const { getOptionValue } = require('internal/options'); +const { + loadPreloadModules, + initializeModuleLoaders, + initializeFrozenIntrinsics, +} = require('internal/process/pre_execution'); const { receiveMessageOnPort } = require('internal/worker/io'); const { WORKER_TO_MAIN_THREAD_NOTIFICATION, } = require('internal/modules/esm/shared_constants'); -const { initializeHooks } = require('internal/modules/esm/utils'); const { isMarkedAsUntransferable } = require('internal/buffer'); +const { getCWDURL } = require('internal/util'); +const { isCascadedLoaderInitialized, getOrInitializeCascadedLoader } = require('internal/modules/esm/loader'); +const { AsyncLoaderHooksOnLoaderHookWorker } = require('internal/modules/esm/hooks'); + +/** + * Register asynchronus module loader customization hooks. This should only be run in the loader + * hooks worker. In a non-loader-hooks thread, if any asynchronous loader hook is registered, the + * ModuleLoader#asyncLoaderHooks are initialized to be AsyncLoaderHooksProxiedToLoaderHookWorker + * which posts the messages to the async loader hook worker thread. + * When no asynchronous loader hook is registered, the loader hook worker is not spawned and module + * loading is entiredly done in-thread. + * @returns {Promise} + */ +async function initializeAsyncLoaderHooksOnLoaderHookWorker() { + const customLoaderURLs = getOptionValue('--experimental-loader'); + + // The worker thread spawned for handling asynchronous loader hooks should not + // further spawn other hook threads or there will be an infinite recursion. + const shouldSpawnLoaderHookWorker = false; + // The worker thread for async loader hooks will preload user modules itself in + // initializeAsyncLoaderHooksOnLoaderHookWorker(). + const shouldPreloadModules = false; + initializeModuleLoaders({ shouldSpawnLoaderHookWorker, shouldPreloadModules }); + + assert(!isCascadedLoaderInitialized(), + 'ModuleLoader should be initialized in initializeAsyncLoaderHooksOnLoaderHookWorker()'); + const asyncLoaderHooks = new AsyncLoaderHooksOnLoaderHookWorker(); + getOrInitializeCascadedLoader(asyncLoaderHooks); + + // We need the async loader hooks to be set _before_ we start invoking + // `--require`, otherwise loops can happen because a `--require` script + // might call `register(...)` before we've installed ourselves. These + // global values are magically set in `initializeModuleLoaders` just for us and + // we call them in the correct order. + // N.B. This block appears here specifically in order to ensure that + // `--require` calls occur before `--loader` ones do. + loadPreloadModules(); + initializeFrozenIntrinsics(); + + const parentURL = getCWDURL().href; + for (let i = 0; i < customLoaderURLs.length; i++) { + await asyncLoaderHooks.register(customLoaderURLs[i], parentURL); + } + + return asyncLoaderHooks; +} /** * Transfers an ArrayBuffer, TypedArray, or DataView to a worker thread. @@ -82,7 +133,7 @@ function wrapMessage(status, body) { } /** - * Initializes a worker thread for a customized module loader. + * Initializes the loader hooks worker thread with customized asynchronous module loading hooks. * @param {SharedArrayBuffer} lock - The lock used to synchronize communication between the worker and the main thread. * @param {MessagePort} syncCommPort - The message port used for synchronous communication between the worker and the * main thread. @@ -90,7 +141,7 @@ function wrapMessage(status, body) { * @returns {Promise} A promise that resolves when the worker thread has been initialized. */ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { - let hooks; + let asyncLoaderHooks; let initializationError; let hasInitializationError = false; @@ -106,9 +157,8 @@ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { }; } - try { - hooks = await initializeHooks(); + asyncLoaderHooks = await initializeAsyncLoaderHooksOnLoaderHookWorker(); } catch (exception) { // If there was an error while parsing and executing a user loader, for example if because a // loader contained a syntax error, then we need to send the error to the main thread so it can @@ -178,7 +228,7 @@ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { // the main thread. let hasError = false; let shouldRemoveGlobalErrorHandler = false; - assert(typeof hooks[method] === 'function'); + assert(typeof asyncLoaderHooks[method] === 'function', `${method} is not implemented in the loader worker`); if (port == null && !hasUncaughtExceptionCaptureCallback()) { // When receiving sync messages, we want to unlock the main thread when there's an exception. process.on('uncaughtException', errorHandler); @@ -198,7 +248,7 @@ async function customizedModuleWorker(lock, syncCommPort, errorHandler) { let response; try { - response = await ReflectApply(hooks[method], hooks, args); + response = await ReflectApply(asyncLoaderHooks[method], asyncLoaderHooks, args); } catch (exception) { hasError = true; response = exception; diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 2a9ef56d156808..d337eeca801b9c 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -96,7 +96,7 @@ async function asyncRunEntryPointWithESMLoader(callback) { await cascadedLoader.import(userImports[i], parentURL, kEmptyObject); } } else { - cascadedLoader.forceLoadHooks(); + cascadedLoader.waitForAsyncLoaderHookInitialization(); } await callback(cascadedLoader); } catch (err) { diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index f93b7dae1bca9b..76ec7d821cb4cc 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -47,6 +47,8 @@ function prepareMainThreadExecution(expandArgv1 = false, initializeModules = tru expandArgv1, initializeModules, isMainThread: true, + shouldSpawnLoaderHookWorker: initializeModules, + shouldPreloadModules: initializeModules, }); } @@ -55,15 +57,20 @@ function prepareTestRunnerMainExecution(loadUserModules = true) { expandArgv1: false, initializeModules: true, isMainThread: true, - forceDefaultLoader: !loadUserModules, + shouldSpawnLoaderHookWorker: loadUserModules, + shouldPreloadModules: loadUserModules, }); } function prepareWorkerThreadExecution() { prepareExecution({ expandArgv1: false, - initializeModules: false, isMainThread: false, + // Module loader initialization in workers are delayed until the worker thread + // is ready for execution. + initializeModules: false, + shouldSpawnLoaderHookWorker: false, + shouldPreloadModules: false, }); } @@ -74,7 +81,7 @@ function prepareShadowRealmExecution() { setupDebugEnv(); // Disable custom loaders in ShadowRealm. - setupUserModules(true); + initializeModuleLoaders({ shouldSpawnLoaderHookWorker: false, shouldPreloadModules: false }); const { privateSymbols: { host_defined_option_symbol, @@ -96,7 +103,7 @@ function prepareShadowRealmExecution() { } function prepareExecution(options) { - const { expandArgv1, initializeModules, isMainThread, forceDefaultLoader } = options; + const { expandArgv1, initializeModules, isMainThread, shouldSpawnLoaderHookWorker, shouldPreloadModules } = options; refreshRuntimeOptions(); @@ -154,8 +161,9 @@ function prepareExecution(options) { assert(!initializeModules); } + setupVmModules(); if (initializeModules) { - setupUserModules(forceDefaultLoader); + initializeModuleLoaders({ shouldSpawnLoaderHookWorker, shouldPreloadModules }); } // This has to be done after the user module loader is initialized, @@ -165,6 +173,21 @@ function prepareExecution(options) { return mainEntry; } +function setupVmModules() { + // Patch the vm module when --experimental-vm-modules is on. + // Please update the comments in vm.js when this block changes. + // TODO(joyeecheung): move this to vm.js? + if (getOptionValue('--experimental-vm-modules')) { + const { + Module, SourceTextModule, SyntheticModule, + } = require('internal/vm/module'); + const vm = require('vm'); + vm.Module = Module; + vm.SourceTextModule = SourceTextModule; + vm.SyntheticModule = SyntheticModule; + } +} + function setupHttpProxy() { // This normalized from both --use-env-proxy and NODE_USE_ENV_PROXY settings. if (!getOptionValue('--use-env-proxy')) { @@ -186,22 +209,32 @@ function setupHttpProxy() { // existing libraries that sets the global dispatcher or monkey patches the global agent. } -function setupUserModules(forceDefaultLoader = false) { - initializeCJSLoader(); - initializeESMLoader(forceDefaultLoader); +function initializeModuleLoaders(options) { + const { shouldSpawnLoaderHookWorker, shouldPreloadModules } = options; + // Initialize certain special module.Module properties and the CJS conditions. + const { initializeCJS } = require('internal/modules/cjs/loader'); + initializeCJS(); + // Initialize the ESM loader and a few module callbacks. + // If shouldSpawnLoaderHookWorker is true, later when the ESM loader is instantiated on-demand, + // it will spawn a loader worker thread to handle async custom loader hooks. + const { initializeESM } = require('internal/modules/esm/utils'); + initializeESM(shouldSpawnLoaderHookWorker); + const { hasStartedUserCJSExecution, hasStartedUserESMExecution, } = require('internal/modules/helpers'); + // At this point, no user module has been executed yet. assert(!hasStartedUserCJSExecution()); assert(!hasStartedUserESMExecution()); + if (getEmbedderOptions().hasEmbedderPreload) { runEmbedderPreload(); } // Do not enable preload modules if custom loaders are disabled. // For example, loader workers are responsible for doing this themselves. // And preload modules are not supported in ShadowRealm as well. - if (!forceDefaultLoader) { + if (shouldPreloadModules) { loadPreloadModules(); } // Need to be done after --require setup. @@ -638,28 +671,6 @@ function initializePermission() { } } -function initializeCJSLoader() { - const { initializeCJS } = require('internal/modules/cjs/loader'); - initializeCJS(); -} - -function initializeESMLoader(forceDefaultLoader) { - const { initializeESM } = require('internal/modules/esm/utils'); - initializeESM(forceDefaultLoader); - - // Patch the vm module when --experimental-vm-modules is on. - // Please update the comments in vm.js when this block changes. - if (getOptionValue('--experimental-vm-modules')) { - const { - Module, SourceTextModule, SyntheticModule, - } = require('internal/vm/module'); - const vm = require('vm'); - vm.Module = Module; - vm.SourceTextModule = SourceTextModule; - vm.SyntheticModule = SyntheticModule; - } -} - function initializeSourceMapsHandlers() { const { setSourceMapsSupport, @@ -729,7 +740,7 @@ function getHeapSnapshotFilename(diagnosticDir) { } module.exports = { - setupUserModules, + initializeModuleLoaders, prepareMainThreadExecution, prepareWorkerThreadExecution, prepareShadowRealmExecution, diff --git a/test/es-module/test-esm-loader-modulemap.js b/test/es-module/test-esm-loader-modulemap.js index 83125fce738139..f581c7f507640e 100644 --- a/test/es-module/test-esm-loader-modulemap.js +++ b/test/es-module/test-esm-loader-modulemap.js @@ -16,7 +16,7 @@ const jsonModuleDataUrl = 'data:application/json,""'; const stubJsModule = createDynamicModule([], ['default'], jsModuleDataUrl); const stubJsonModule = createDynamicModule([], ['default'], jsonModuleDataUrl); -const loader = createModuleLoader(false); +const loader = createModuleLoader(); const jsModuleJob = new ModuleJob(loader, stubJsModule.module, undefined, () => new Promise(() => {})); const jsonModuleJob = new ModuleJob(loader, stubJsonModule.module, diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index a66a3ec9ce360f..b050f5bffde04a 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -107,15 +107,15 @@ expected.beforePreExec = new Set([ 'NativeModule internal/modules/typescript', 'NativeModule internal/data_url', 'NativeModule internal/mime', -]); - -expected.atRunTime = new Set([ + 'NativeModule internal/modules/esm/utils', 'Internal Binding worker', 'NativeModule internal/modules/run_main', 'NativeModule internal/net', 'NativeModule internal/dns/utils', +]); + +expected.atRunTime = new Set([ 'NativeModule internal/process/pre_execution', - 'NativeModule internal/modules/esm/utils', ]); const { isMainThread } = require('worker_threads'); @@ -173,7 +173,7 @@ if (common.hasIntl) { if (process.features.inspector) { expected.beforePreExec.add('Internal Binding inspector'); expected.beforePreExec.add('NativeModule internal/util/inspector'); - expected.atRunTime.add('NativeModule internal/inspector_async_hook'); + expected.beforePreExec.add('NativeModule internal/inspector_async_hook'); } // This is loaded if the test is run with NODE_V8_COVERAGE.