module: implement register utility

PR-URL: #46826 Reviewed-By: Jacob Smith <jacob@frende.me> Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com> Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

Author: jlenon7

doc/api/errors.md

@@ -1267,6 +1267,23 @@ provided.
 Encoding provided to `TextDecoder()` API was not one of the
 [WHATWG Supported Encodings][].
 
+<a id="ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE"></a>
+
+### `ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Programmatically registering custom ESM loaders
+currently requires at least one custom loader to have been
+registered via the `--experimental-loader` flag. A no-op
+loader registered via CLI is sufficient
+(for example: `--experimental-loader data:text/javascript,`;
+do not omit the necessary trailing comma).
+A future version of Node.js will support the programmatic
+registration of loaders without needing to also use the flag.
+
 <a id="ERR_EVAL_ESM_CANNOT_PRINT"></a>
 
 ### `ERR_EVAL_ESM_CANNOT_PRINT`

doc/api/esm.md

@@ -1225,6 +1225,17 @@ console.log('some module!');
 If you run `node --experimental-loader ./import-map-loader.js main.js`
 the output will be `some module!`.
 
+### Register loaders programmatically
+
+<!-- YAML
+added: REPLACEME
+-->
+
+In addition to using the `--experimental-loader` option in the CLI,
+loaders can also be registered programmatically. You can find
+detailed information about this process in the documentation page
+for [`module.register()`][].
+
 ## Resolution and loading algorithm
 
 ### Features
@@ -1599,6 +1610,7 @@ for ESM specifiers is [commonjs-extension-resolution-loader][].
 [`import.meta.url`]: #importmetaurl
 [`import`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import
 [`module.createRequire()`]: module.md#modulecreaterequirefilename
+[`module.register()`]: module.md#moduleregister
 [`module.syncBuiltinESMExports()`]: module.md#modulesyncbuiltinesmexports
 [`package.json`]: packages.md#nodejs-packagejson-field-definitions
 [`port.ref()`]: https://nodejs.org/dist/latest-v17.x/docs/api/worker_threads.html#portref

doc/api/module.md

@@ -80,6 +80,101 @@ isBuiltin('fs'); // true
 isBuiltin('wss'); // false
 ```
 
+### `module.register()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+In addition to using the `--experimental-loader` option in the CLI,
+loaders can be registered programmatically using the
+`module.register()` method.
+
+```mjs
+import { register } from 'node:module';
+
+register('http-to-https', import.meta.url);
+
+// Because this is a dynamic `import()`, the `http-to-https` hooks will run
+// before importing `./my-app.mjs`.
+await import('./my-app.mjs');
+```
+
+In the example above, we are registering the `http-to-https` loader,
+but it will only be available for subsequently imported modules—in
+this case, `my-app.mjs`. If the `await import('./my-app.mjs')` had
+instead been a static `import './my-app.mjs'`, _the app would already
+have been loaded_ before the `http-to-https` hooks were
+registered. This is part of the design of ES modules, where static
+imports are evaluated from the leaves of the tree first back to the
+trunk. There can be static imports _within_ `my-app.mjs`, which
+will not be evaluated until `my-app.mjs` is when it's dynamically
+imported.
+
+The `--experimental-loader` flag of the CLI can be used together
+with the `register` function; the loaders registered with the
+function will follow the same evaluation chain of loaders registered
+within the CLI:
+
+```console
+node \
+ --experimental-loader unpkg \
+ --experimental-loader http-to-https \
+ --experimental-loader cache-buster \
+ entrypoint.mjs
+```
+
+```mjs
+// entrypoint.mjs
+import { URL } from 'node:url';
+import { register } from 'node:module';
+
+const loaderURL = new URL('./my-programmatically-loader.mjs', import.meta.url);
+
+register(loaderURL);
+await import('./my-app.mjs');
+```
+
+The `my-programmatic-loader.mjs` can leverage `unpkg`,
+`http-to-https`, and `cache-buster` loaders.
+
+It's also possible to use `register` more than once:
+
+```mjs
+// entrypoint.mjs
+import { URL } from 'node:url';
+import { register } from 'node:module';
+
+register(new URL('./first-loader.mjs', import.meta.url));
+register('./second-loader.mjs', import.meta.url);
+await import('./my-app.mjs');
+```
+
+Both loaders (`first-loader.mjs` and `second-loader.mjs`) can use
+all the resources provided by the loaders registered in the CLI. But
+remember that they will only be available in the next imported
+module (`my-app.mjs`). The evaluation order of the hooks when
+importing `my-app.mjs` and consecutive modules in the example above
+will be:
+
+```console
+resolve: second-loader.mjs
+resolve: first-loader.mjs
+resolve: cache-buster
+resolve: http-to-https
+resolve: unpkg
+load: second-loader.mjs
+load: first-loader.mjs
+load: cache-buster
+load: http-to-https
+load: unpkg
+globalPreload: second-loader.mjs
+globalPreload: first-loader.mjs
+globalPreload: cache-buster
+globalPreload: http-to-https
+globalPreload: unpkg
+```
+
 ### `module.syncBuiltinESMExports()`
 
 <!-- YAML

lib/internal/errors.js

@@ -1036,6 +1036,11 @@ E('ERR_ENCODING_INVALID_ENCODED_DATA', function(encoding, ret) {
 }, TypeError);
 E('ERR_ENCODING_NOT_SUPPORTED', 'The "%s" encoding is not supported',
  RangeError);
+E('ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE', 'Programmatically registering custom ESM loaders ' +
+ 'currently requires at least one custom loader to have been registered via the --experimental-loader ' +
+ 'flag. A no-op loader registered via CLI is sufficient (for example: `--experimental-loader ' +
+ '"data:text/javascript,"` with the necessary trailing comma). A future version of Node.js ' +
+ 'will remove this requirement.', Error);
 E('ERR_EVAL_ESM_CANNOT_PRINT', '--print cannot be used with ESM input', Error);
 E('ERR_EVENT_RECURSION', 'The event "%s" is already being dispatched', Error);
 E('ERR_FALSY_VALUE_REJECTION', function(reason) {

lib/internal/modules/esm/hooks.js

@@ -45,6 +45,8 @@ const {
  validateString,
 } = require('internal/validators');
 
+const { kEmptyObject } = require('internal/util');
+
 const {
  defaultResolve,
  throwIfInvalidParentURL,
@@ -117,6 +119,23 @@ class Hooks {
  // Cache URLs we've already validated to avoid repeated validation
  #validatedUrls = new SafeSet();
 
+ /**
+ * Import and register custom/user-defined module loader hook(s).
+ * @param {string} urlOrSpecifier
+ * @param {string} parentURL
+ */
+ async register(urlOrSpecifier, parentURL) {
+ const moduleLoader = require('internal/process/esm_loader').esmLoader;
+
+ const keyedExports = await moduleLoader.import(
+ urlOrSpecifier,
+ parentURL,
+ kEmptyObject,
+ );
+
+ this.addCustomLoader(urlOrSpecifier, keyedExports);
+ }
+
  /**
  * Collect custom/user-defined module loader hook(s).
  * After all hooks have been collected, the global preload hook(s) must be initialized.

lib/internal/modules/esm/loader.js

@@ -11,6 +11,7 @@ const {
 } = primordials;
 
 const {
+ ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE,
  ERR_UNKNOWN_MODULE_FORMAT,
 } = require('internal/errors').codes;
 const { getOptionValue } = require('internal/options');
@@ -287,12 +288,19 @@ class CustomizedModuleLoader extends DefaultModuleLoader {
  constructor() {
  super();
 
- if (hooksProxy) {
- // The worker proxy is shared across all instances, so don't recreate it if it already exists.
- return;
- }
- const { HooksProxy } = require('internal/modules/esm/hooks');
- hooksProxy = new HooksProxy(); // The user's custom hooks are loaded within the worker as part of its startup.
+ getHooksProxy();
+ }
+
+ /**
+ * Register some loader specifier.
+ * @param {string} originalSpecifier The specified URL path of the loader to
+ * be registered.
+ * @param {string} parentURL The parent URL from where the loader will be
+ * registered if using it package name as specifier
+ * @returns {{ format: string, url: URL['href'] }}
+ */
+ register(originalSpecifier, parentURL) {
+ return hooksProxy.makeSyncRequest('register', originalSpecifier, parentURL);
  }
 
  /**
@@ -370,7 +378,46 @@ function createModuleLoader(useCustomLoadersIfPresent = true) {
  return new DefaultModuleLoader();
 }
 
+/**
+ * Get the HooksProxy instance. If it is not defined, then create a new one.
+ * @returns {HooksProxy}
+ */
+function getHooksProxy() {
+ if (!hooksProxy) {
+ const { HooksProxy } = require('internal/modules/esm/hooks');
+ hooksProxy = new HooksProxy();
+ }
+
+ return hooksProxy;
+}
+
+/**
+ * Register a single loader programmatically.
+ * @param {string} specifier
+ * @param {string} [parentURL]
+ * @returns {void}
+ * @example
+ * ```js
+ * register('./myLoader.js');
+ * register('ts-node/esm', import.meta.url);
+ * register('./myLoader.js', import.meta.url);
+ * register(new URL('./myLoader.js', import.meta.url));
+ * ```
+ */
+function register(specifier, parentURL = 'data:') {
+ // TODO: Remove this limitation in a follow-up before `register` is released publicly
+ if (getOptionValue('--experimental-loader').length < 1) {
+ throw new ERR_ESM_LOADER_REGISTRATION_UNAVAILABLE();
+ }
+
+ const moduleLoader = require('internal/process/esm_loader').esmLoader;
+
+ moduleLoader.register(`${specifier}`, parentURL);
+}
+
 module.exports = {
  DefaultModuleLoader,
  createModuleLoader,
+ getHooksProxy,
+ register,
 };

lib/internal/modules/esm/utils.js

@@ -146,9 +146,9 @@ async function initializeHooks() {
  load(url, context) { return hooks.load(url, context); }
  }
  const privateModuleLoader = new ModuleLoader();
-
  const parentURL = pathToFileURL(cwd).href;
 
+ // TODO(jlenon7): reuse the `Hooks.register()` method for registering loaders.
  for (let i = 0; i < customLoaderURLs.length; i++) {
  const customLoaderURL = customLoaderURLs[i];
 

lib/module.js

@@ -2,8 +2,10 @@
 
 const { findSourceMap } = require('internal/source_map/source_map_cache');
 const { Module } = require('internal/modules/cjs/loader');
+const { register } = require('internal/modules/esm/loader');
 const { SourceMap } = require('internal/source_map/source_map');
 
 Module.findSourceMap = findSourceMap;
+Module.register = register;
 Module.SourceMap = SourceMap;
 module.exports = Module;

test/es-module/test-esm-loader-hooks.mjs

@@ -24,6 +24,28 @@ describe('Loader hooks', { concurrency: true }, () => {
  assert.match(lines[3], /{"source":{"type":"Buffer","data":\[.*\]},"format":"json","shortCircuit":true}/);
  });
 
+ it('are called with all expected arguments using register function', async () => {
+ const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [
+ '--no-warnings',
+ '--experimental-loader=data:text/javascript,',
+ '--input-type=module',
+ '--eval',
+ "import { register } from 'node:module';" +
+ `register(${JSON.stringify(fixtures.fileURL('/es-module-loaders/hooks-input.mjs'))});` +
+ `await import(${JSON.stringify(fixtures.fileURL('/es-modules/json-modules.mjs'))});`,
+ ]);
+
+ assert.strictEqual(stderr, '');
+ assert.strictEqual(code, 0);
+ assert.strictEqual(signal, null);
+
+ const lines = stdout.split('\n');
+ assert.match(lines[0], /{"url":"file:\/\/\/.*\/json-modules\.mjs","format":"test","shortCircuit":true}/);
+ assert.match(lines[1], /{"source":{"type":"Buffer","data":\[.*\]},"format":"module","shortCircuit":true}/);
+ assert.match(lines[2], /{"url":"file:\/\/\/.*\/experimental\.json","format":"test","shortCircuit":true}/);
+ assert.match(lines[3], /{"source":{"type":"Buffer","data":\[.*\]},"format":"json","shortCircuit":true}/);
+ });
+
  describe('should handle never-settling hooks in ESM files', { concurrency: true }, () => {
  it('top-level await of a never-settling resolve', async () => {
  const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [