Merge pull request #2636 from cloudflare/jsnell/implement-node-compat…

…-module-createrequire

src/node/internal/module.d.ts

@@ -0,0 +1,6 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+export function createRequire(path: string): (specifier: string) => unknown;
+export function isBuiltin(specifier: string): boolean;

src/node/module.ts

@@ -0,0 +1,122 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+/* eslint-disable */
+
+import { default as moduleUtil } from 'node-internal:module';
+import { ERR_INVALID_ARG_VALUE } from 'node-internal:internal_errors';
+
+export function createRequire(
+  path: string | URL
+): (specifier: string) => unknown {
+  // Note that per Node.js' requirements, path must be one of either
+  // an absolute file path or a file URL. We do not currently handle
+  // module specifiers as URLs yet but we'll try to get close.
+
+  path = `${path}`;
+  if (
+    !(path as string).startsWith('/') &&
+    !(path as string).startsWith('file:')
+  ) {
+    throw new ERR_INVALID_ARG_VALUE(
+      'path',
+      path,
+      'The argument must be a file URL object, ' +
+        'a file URL string, or an absolute path string.'
+    );
+  }
+
+  return moduleUtil.createRequire(path as string);
+}
+
+// Indicates only that the given specifier is known to be a
+// Node.js built-in module specifier with or with the the
+// 'node:' prefix. A true return value does not guarantee that
+// the module is actually implemented in the runtime.
+export function isBuiltin(specifier: string): boolean {
+  return moduleUtil.isBuiltin(specifier);
+}
+
+// Intentionally does not include modules with mandatory 'node:'
+// prefix like `node:test`.
+// See: See https://nodejs.org/docs/latest/api/modules.html#built-in-modules-with-mandatory-node-prefix
+// TODO(later): This list duplicates the list that is in
+// workerd/jsg/modules.c++. Later we should source these
+// from the same place so we don't have to maintain two lists.
+export const builtinModules = [
+  '_http_agent',
+  '_http_client',
+  '_http_common',
+  '_http_incoming',
+  '_http_outgoing',
+  '_http_server',
+  '_stream_duplex',
+  '_stream_passthrough',
+  '_stream_readable',
+  '_stream_transform',
+  '_stream_wrap',
+  '_stream_writable',
+  '_tls_common',
+  '_tls_wrap',
+  'assert',
+  'assert/strict',
+  'async_hooks',
+  'buffer',
+  'child_process',
+  'cluster',
+  'console',
+  'constants',
+  'crypto',
+  'dgram',
+  'diagnostics_channel',
+  'dns',
+  'dns/promises',
+  'domain',
+  'events',
+  'fs',
+  'fs/promises',
+  'http',
+  'http2',
+  'https',
+  'inspector',
+  'inspector/promises',
+  'module',
+  'net',
+  'os',
+  'path',
+  'path/posix',
+  'path/win32',
+  'perf_hooks',
+  'process',
+  'punycode',
+  'querystring',
+  'readline',
+  'readline/promises',
+  'repl',
+  'stream',
+  'stream/consumers',
+  'stream/promises',
+  'stream/web',
+  'string_decoder',
+  'sys',
+  'timers',
+  'timers/promises',
+  'tls',
+  'trace_events',
+  'tty',
+  'url',
+  'util',
+  'util/types',
+  'v8',
+  'vm',
+  'wasi',
+  'worker_threads',
+  'zlib',
+];
+Object.freeze(builtinModules);
+
+export default {
+  createRequire,
+  isBuiltin,
+  builtinModules,
+};

src/workerd/api/node/BUILD.bazel

@@ -157,3 +157,9 @@ wd_test(
     args = ["--experimental"],
     data = ["tests/zlib-nodejs-test.js"],
 )
+
+wd_test(
+    src = "tests/module-create-require-test.wd-test",
+    args = ["--experimental"],
+    data = ["tests/module-create-require-test.js"],
+)

src/workerd/api/node/module.c++

@@ -0,0 +1,112 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+#include "module.h"
+#include <workerd/jsg/url.h>
+
+namespace workerd::api::node {
+
+bool ModuleUtil::isBuiltin(kj::String specifier) {
+  return jsg::checkNodeSpecifier(specifier) != kj::none;
+}
+
+jsg::JsValue ModuleUtil::createRequire(jsg::Lock& js, kj::String path) {
+  // Node.js requires that the specifier path is a File URL or an absolute
+  // file path string. To be compliant, we will convert whatever specifier
+  // is into a File URL if possible, then take the path as the actual
+  // specifier to use.
+  auto parsed = JSG_REQUIRE_NONNULL(jsg::Url::tryParse(path.asPtr(), "file:///"_kj), TypeError,
+      "The argument must be a file URL object, "
+      "a file URL string, or an absolute path string.");
+
+  // We do not currently handle specifiers as URLs, so let's treat any
+  // input that has query string params or hash fragments as errors.
+  if (parsed.getSearch().size() > 0 || parsed.getHash().size() > 0) {
+    JSG_FAIL_REQUIRE(
+        Error, "The specifier must not have query string parameters or hash fragments.");
+  }
+
+  // The specifier must be a file: URL
+  JSG_REQUIRE(parsed.getProtocol() == "file:"_kj, TypeError, "The specifier must be a file: URL.");
+
+  return jsg::JsValue(js.wrapReturningFunction(js.v8Context(),
+      [referrer = kj::str(parsed.getPathname())](
+          jsg::Lock& js, const v8::FunctionCallbackInfo<v8::Value>& args) -> v8::Local<v8::Value> {
+    auto registry = jsg::ModuleRegistry::from(js);
+
+    // TODO(soon): This will need to be updated to support the new module registry
+    // when that is fully implemented.
+    JSG_REQUIRE(registry != nullptr, Error, "Module registry not available.");
+
+    auto ref = ([&] {
+      try {
+        return kj::Path::parse(referrer.slice(1));
+      } catch (kj::Exception& e) {
+        JSG_FAIL_REQUIRE(Error, kj::str("Invalid referrer path: ", referrer.slice(1)));
+      }
+    })();
+
+    auto spec = kj::str(args[0]);
+
+    if (jsg::isNodeJsCompatEnabled(js)) {
+      KJ_IF_SOME(nodeSpec, jsg::checkNodeSpecifier(spec)) {
+        spec = kj::mv(nodeSpec);
+      }
+    }
+
+    static const kj::Path kRoot = kj::Path::parse("");
+
+    kj::Path targetPath = ([&] {
+      // If the specifier begins with one of our known prefixes, let's not resolve
+      // it against the referrer.
+      try {
+        if (spec.startsWith("node:") || spec.startsWith("cloudflare:") ||
+            spec.startsWith("workerd:")) {
+          return kj::Path::parse(spec);
+        }
+
+        return ref == kRoot ? kj::Path::parse(spec) : ref.parent().eval(spec);
+      } catch (kj::Exception&) {
+        JSG_FAIL_REQUIRE(Error, kj::str("Invalid specifier path: ", spec));
+      }
+    })();
+
+    // require() is only exposed to worker bundle modules so the resolve here is only
+    // permitted to require worker bundle or built-in modules. Internal modules are
+    // excluded.
+    auto& info = JSG_REQUIRE_NONNULL(
+        registry->resolve(js, targetPath, ref, jsg::ModuleRegistry::ResolveOption::DEFAULT,
+            jsg::ModuleRegistry::ResolveMethod::REQUIRE, spec.asPtr()),
+        Error, "No such module \"", targetPath.toString(), "\".");
+
+    bool isEsm = info.maybeSynthetic == kj::none;
+
+    auto module = info.module.getHandle(js);
+
+    jsg::instantiateModule(js, module);
+    auto handle = jsg::check(module->Evaluate(js.v8Context()));
+    KJ_ASSERT(handle->IsPromise());
+    auto prom = handle.As<v8::Promise>();
+    if (prom->State() == v8::Promise::PromiseState::kPending) {
+      js.runMicrotasks();
+    }
+    JSG_REQUIRE(prom->State() != v8::Promise::PromiseState::kPending, Error,
+        "Module evaluation did not complete synchronously.");
+    if (module->GetStatus() == v8::Module::kErrored) {
+      jsg::throwTunneledException(js.v8Isolate, module->GetException());
+    }
+
+    if (isEsm) {
+      // If the import is an esm module, we will return the namespace object.
+      jsg::JsObject obj(module->GetModuleNamespace().As<v8::Object>());
+      if (obj.get(js, "__cjsUnwrapDefault"_kj) == js.boolean(true)) {
+        return obj.get(js, "default"_kj);
+      }
+      return obj;
+    }
+
+    return jsg::JsValue(js.v8Get(module->GetModuleNamespace().As<v8::Object>(), "default"_kj));
+  }));
+}
+
+}  // namespace workerd::api::node

src/workerd/api/node/module.h

@@ -0,0 +1,30 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+#pragma once
+
+#include <workerd/jsg/jsg.h>
+
+namespace workerd::api::node {
+
+class ModuleUtil final: public jsg::Object {
+public:
+  ModuleUtil() = default;
+  ModuleUtil(jsg::Lock&, const jsg::Url&) {}
+
+  jsg::JsValue createRequire(jsg::Lock& js, kj::String specifier);
+
+  // Returns true if the specifier is a known node.js built-in module specifier.
+  // Ignores whether or not the module actually exists (use process.getBuiltinModule()
+  // for that purpose).
+  bool isBuiltin(kj::String specifier);
+
+  JSG_RESOURCE_TYPE(ModuleUtil) {
+    JSG_METHOD(createRequire);
+    JSG_METHOD(isBuiltin);
+  }
+};
+
+#define EW_NODE_MODULE_ISOLATE_TYPES api::node::ModuleUtil
+
+}  // namespace workerd::api::node

src/workerd/api/node/node.h

@@ -4,6 +4,7 @@
 #include "buffer.h"
 #include "crypto.h"
 #include "diagnostics-channel.h"
+#include "module.h"
 #include "url.h"
 #include "util.h"
 #include "zlib-util.h"
@@ -43,6 +44,7 @@ class CompatibilityFlags: public jsg::Object {
   V(AsyncHooksModule, "node-internal:async_hooks")                                                 \
   V(BufferUtil, "node-internal:buffer")                                                            \
   V(CryptoImpl, "node-internal:crypto")                                                            \
+  V(ModuleUtil, "node-internal:module")                                                            \
   V(UtilModule, "node-internal:util")                                                              \
   V(DiagnosticsChannelModule, "node-internal:diagnostics_channel")                                 \
   V(ZlibUtil, "node-internal:zlib")                                                                \
@@ -137,4 +139,5 @@ kj::Own<jsg::modules::ModuleBundle> getExternalNodeJsCompatModuleBundle(auto fea
 #define EW_NODE_ISOLATE_TYPES                                                                      \
   api::node::CompatibilityFlags, EW_NODE_BUFFER_ISOLATE_TYPES, EW_NODE_CRYPTO_ISOLATE_TYPES,       \
       EW_NODE_DIAGNOSTICCHANNEL_ISOLATE_TYPES, EW_NODE_ASYNCHOOKS_ISOLATE_TYPES,                   \
-      EW_NODE_UTIL_ISOLATE_TYPES, EW_NODE_ZLIB_ISOLATE_TYPES, EW_NODE_URL_ISOLATE_TYPES
+      EW_NODE_UTIL_ISOLATE_TYPES, EW_NODE_ZLIB_ISOLATE_TYPES, EW_NODE_URL_ISOLATE_TYPES,           \
+      EW_NODE_MODULE_ISOLATE_TYPES\

src/workerd/api/node/tests/module-create-require-test.js

@@ -0,0 +1,75 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+import { createRequire, isBuiltin, builtinModules } from 'node:module';
+import { ok, strictEqual, throws } from 'node:assert';
+
+export const doTheTest = {
+  async test() {
+    const require = createRequire('/');
+    ok(typeof require === 'function');
+
+    const foo = require('foo');
+    const bar = require('bar');
+    const baz = require('baz');
+    const qux = require('worker/qux');
+
+    strictEqual(foo.default, 1);
+    strictEqual(bar, 2);
+    strictEqual(baz, 3);
+    strictEqual(qux, '4');
+
+    const assert = await import('node:assert');
+    const required = require('node:assert');
+    strictEqual(assert, required);
+
+    throws(() => require('invalid'), {
+      message: 'Module evaluation did not complete synchronously.',
+    });
+
+    throws(() => require('does not exist'));
+    throws(() => createRequire('not a valid path'), {
+      message: /The argument must be a file URL object/,
+    });
+    throws(() => createRequire(new URL('http://example.org')), {
+      message: /The argument must be a file URL object/,
+    });
+
+    // TODO(soon): Later when we when complete the new module registry, query strings
+    // and hash fragments will be allowed when the new registry is being used.
+    throws(() => createRequire('file://test?abc'), {
+      message:
+        'The specifier must not have query string parameters or hash fragments.',
+    });
+    throws(() => createRequire('file://test#123'), {
+      message:
+        'The specifier must not have query string parameters or hash fragments.',
+    });
+
+    // These should not throw...
+    createRequire('file:///');
+    createRequire('file:///tmp');
+    createRequire(new URL('file:///'));
+  },
+};
+
+export const isBuiltinTest = {
+  test() {
+    ok(isBuiltin('fs'));
+    ok(isBuiltin('http'));
+    ok(isBuiltin('https'));
+    ok(isBuiltin('path'));
+    ok(isBuiltin('node:fs'));
+    ok(isBuiltin('node:http'));
+    ok(isBuiltin('node:https'));
+    ok(isBuiltin('node:path'));
+    ok(isBuiltin('node:test'));
+    ok(!isBuiltin('test'));
+    ok(!isBuiltin('worker'));
+    ok(!isBuiltin('worker/qux'));
+
+    builtinModules.forEach((module) => {
+      ok(isBuiltin(module));
+    });
+  },
+};