module: refactor and clarify async loader hook customizations

lib/internal/main/worker_thread.js

@@ -11,15 +11,14 @@ const {
   ObjectDefineProperty,
   PromisePrototypeThen,
   RegExpPrototypeExec,
-  SafeWeakMap,
   globalThis: {
     SharedArrayBuffer,
   },
 } = primordials;
 
 const {
   prepareWorkerThreadExecution,
-  setupUserModules,
+  initializeModuleLoaders,
   markBootstrapComplete,
 } = require('internal/process/pre_execution');
 
@@ -138,11 +137,13 @@ port.on('message', (message) => {
       workerIo.sharedCwdCounter = cwdCounter;
     }
 
-    const isLoaderWorker =
-      doEval === 'internal' &&
-      filename === require('internal/modules/esm/utils').loaderWorkerId;
-    // Disable custom loaders in loader worker.
-    setupUserModules(isLoaderWorker);
+    const isLoaderHookWorker = (filename === 'internal/modules/esm/worker' && doEval === 'internal');
+    if (!isLoaderHookWorker) {
+      // If we are in the loader hook worker, delay the module loader initializations until
+      // initializeAsyncLoaderHooksOnLoaderHookWorker() which needs to run preloads
+      // after the asynchronous loader hooks are registered.
+      initializeModuleLoaders({ shouldSpawnLoaderHookWorker: true, shouldPreloadModules: true });
+    }
 
     if (!hasStdin)
       process.stdin.push(null);
@@ -152,9 +153,10 @@ port.on('message', (message) => {
     port.postMessage({ type: UP_AND_RUNNING });
     switch (doEval) {
       case 'internal': {
-        // Create this WeakMap in js-land because V8 has no C++ API for WeakMap.
-        internalBinding('module_wrap').callbackMap = new SafeWeakMap();
-        require(filename)(workerData, publicPort);
+        // Currently the only user of internal eval is the async loader hook thread.
+        assert(isLoaderHookWorker, `Unexpected internal eval ${filename}`);
+        const setupModuleWorker = require('internal/modules/esm/worker');
+        setupModuleWorker(workerData, publicPort);
         break;
       }
 

lib/internal/modules/esm/hooks.js

@@ -54,7 +54,6 @@ const {
 } = require('internal/modules/esm/resolve');
 const {
   getDefaultConditions,
-  loaderWorkerId,
 } = require('internal/modules/esm/utils');
 const { deserializeError } = require('internal/error_serdes');
 const {
@@ -105,7 +104,39 @@ function defineImportAssertionAlias(context) {
 
 // [2] `validate...()`s throw the wrong error
 
-class Hooks {
+/**
+ * @typedef {{ format: ModuleFormat, source: ModuleSource }} LoadResult
+ */
+
+/**
+ * @typedef {{ format: ModuleFormat, url: string, importAttributes: Record<string, string> }} ResolveResult
+ */
+
+/**
+ * Interface for classes that implement asynchronous loader hooks that can be attached to the ModuleLoader
+ * via `ModuleLoader.#setAsyncLoaderHooks()`.
+ * @typedef {object} AsyncLoaderHooks
+ * @property {boolean} allowImportMetaResolve Whether to allow the use of `import.meta.resolve`.
+ * @property {(url: string, context: object, defaultLoad: Function) => Promise<LoadResult>} load
+ *   Calling the asynchronous `load` hook asynchronously.
+ * @property {(url: string, context: object, defaultLoad: Function) => LoadResult} loadSync
+ *   Calling the asynchronous `load` hook synchronously.
+ * @property {(originalSpecifier: string, parentURL: string,
+ *             importAttributes: Record<string, string>) => Promise<ResolveResult>} resolve
+ *   Calling the asynchronous `resolve` hook asynchronously.
+ * @property {(originalSpecifier: string, parentURL: string,
+ *            importAttributes: Record<string, string>) => ResolveResult} resolveSync
+ *   Calling the asynchronous `resolve` hook synchronously.
+ * @property {(specifier: string, parentURL: string) => any} register Register asynchronous loader hooks
+ * @property {() => void} waitForLoaderHookInitialization Force loading of hooks.
+ */
+
+/**
+ * @implements {AsyncLoaderHooks}
+ * Instances of this class run directly on the loader hook worker thread and customize the module
+ * loading of the hooks worker itself.
+ */
+class AsyncLoaderHooksOnLoaderHookWorker {
   #chains = {
     /**
      * Phase 1 of 2 in ESM loading.
@@ -452,7 +483,7 @@ class Hooks {
     };
   }
 
-  forceLoadHooks() {
+  waitForLoaderHookInitialization() {
     // No-op
   }
 
@@ -462,14 +493,20 @@ class Hooks {
     return meta;
   }
 }
-ObjectSetPrototypeOf(Hooks.prototype, null);
+ObjectSetPrototypeOf(AsyncLoaderHooksOnLoaderHookWorker.prototype, null);
 
 /**
- * There may be multiple instances of Hooks/HooksProxy, but there is only 1 Internal worker, so
- * there is only 1 MessageChannel.
+ * There is only one loader hook thread for each non-loader-hook worker thread
+ * (i.e. the non-loader-hook thread and any worker threads that are not loader hook workers themselves),
+ * so there is only 1 MessageChannel.
  */
 let MessageChannel;
-class HooksProxy {
+
+/**
+ * Abstraction over a worker thread that runs the asynchronous module loader hooks.
+ * Instances of this class run on the non-loader-hook thread and communicate with the loader hooks worker thread.
+ */
+class AsyncLoaderHookWorker {
   /**
    * Shared memory. Always use Atomics method to read or write to it.
    * @type {Int32Array}
@@ -503,7 +540,7 @@ class HooksProxy {
     const lock = new SharedArrayBuffer(SHARED_MEMORY_BYTE_LENGTH);
     this.#lock = new Int32Array(lock);
 
-    this.#worker = new InternalWorker(loaderWorkerId, {
+    this.#worker = new InternalWorker('internal/modules/esm/worker', {
       stderr: false,
       stdin: false,
       stdout: false,
@@ -644,7 +681,7 @@ class HooksProxy {
     this.#importMetaInitializer(meta, context, loader);
   }
 }
-ObjectSetPrototypeOf(HooksProxy.prototype, null);
+ObjectSetPrototypeOf(AsyncLoaderHookWorker.prototype, null);
 
 // TODO(JakobJingleheimer): Remove this when loaders go "stable".
 let globalPreloadWarningWasEmitted = false;
@@ -757,6 +794,95 @@ function nextHookFactory(current, meta, { validateArgs, validateOutput }) {
   );
 }
 
+/**
+ * @type {AsyncLoaderHookWorker}
+ * Worker instance used to run async loader hooks in a separate thread. This is a singleton for each
+ * non-loader-hook worker thread (i.e. the main thread and any worker threads that are not
+ * loader hook workers themselves).
+ */
+let asyncLoaderHookWorker;
+/**
+ * Get the AsyncLoaderHookWorker instance. If it is not defined, then create a new one.
+ * @returns {AsyncLoaderHookWorker}
+ */
+function getAsyncLoaderHookWorker() {
+  asyncLoaderHookWorker ??= new AsyncLoaderHookWorker();
+  return asyncLoaderHookWorker;
+}
+
+/**
+ * @implements {AsyncLoaderHooks}
+ * Instances of this class are created in the non-loader-hook thread and communicate with the worker thread
+ * spawned to run the async loader hooks.
+ */
+class AsyncLoaderHooksProxiedToLoaderHookWorker {
+
+  allowImportMetaResolve = true;
+
+  /**
+   * Instantiate a module loader that uses user-provided custom loader hooks.
+   */
+  constructor() {
+    getAsyncLoaderHookWorker();
+  }
+
+  /**
+   * Register some loader specifier.
+   * @param {string} originalSpecifier The specified URL path of the loader to
+   *   be registered.
+   * @param {string} parentURL The parent URL from where the loader will be
+   *   registered if using it package name as specifier
+   * @param {any} [data] Arbitrary data to be passed from the custom loader
+   *   (user-land) to the worker.
+   * @param {any[]} [transferList] Objects in `data` that are changing ownership
+   * @param {boolean} [isInternal] For internal loaders that should not be publicly exposed.
+   * @returns {{ format: string, url: URL['href'] }}
+   */
+  register(originalSpecifier, parentURL, data, transferList, isInternal) {
+    return asyncLoaderHookWorker.makeSyncRequest('register', transferList, originalSpecifier, parentURL,
+                                                 data, isInternal);
+  }
+
+  /**
+   * Resolve the location of the module.
+   * @param {string} originalSpecifier The specified URL path of the module to
+   *   be resolved.
+   * @param {string} [parentURL] The URL path of the module's parent.
+   * @param {ImportAttributes} importAttributes Attributes from the import
+   *   statement or expression.
+   * @returns {{ format: string, url: URL['href'] }}
+   */
+  resolve(originalSpecifier, parentURL, importAttributes) {
+    return asyncLoaderHookWorker.makeAsyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes);
+  }
+
+  resolveSync(originalSpecifier, parentURL, importAttributes) {
+    // This happens only as a result of `import.meta.resolve` calls, which must be sync per spec.
+    return asyncLoaderHookWorker.makeSyncRequest('resolve', undefined, originalSpecifier, parentURL, importAttributes);
+  }
+
+  /**
+   * Provide source that is understood by one of Node's translators.
+   * @param {URL['href']} url The URL/path of the module to be loaded
+   * @param {object} [context] Metadata about the module
+   * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>}
+   */
+  load(url, context) {
+    return asyncLoaderHookWorker.makeAsyncRequest('load', undefined, url, context);
+  }
+  loadSync(url, context) {
+    return asyncLoaderHookWorker.makeSyncRequest('load', undefined, url, context);
+  }
+
+  importMetaInitialize(meta, context, loader) {
+    asyncLoaderHookWorker.importMetaInitialize(meta, context, loader);
+  }
+
+  waitForLoaderHookInitialization() {
+    asyncLoaderHookWorker.waitForWorker();
+  }
+}
 
-exports.Hooks = Hooks;
-exports.HooksProxy = HooksProxy;
+exports.AsyncLoaderHooksProxiedToLoaderHookWorker = AsyncLoaderHooksProxiedToLoaderHookWorker;
+exports.AsyncLoaderHooksOnLoaderHookWorker = AsyncLoaderHooksOnLoaderHookWorker;
+exports.AsyncLoaderHookWorker = AsyncLoaderHookWorker;