Merge branch 'main' into milan/revert-extensions

samples/pyodide-secret/config.capnp

@@ -0,0 +1,37 @@
+using Workerd = import "/workerd/workerd.capnp";
+
+const config :Workerd.Config = (
+  services = [
+    (name = "main", worker = .mainWorker),
+  ],
+
+  sockets = [
+    # Serve HTTP on port 8080.
+    ( name = "http",
+      address = "*:8080",
+      http = (),
+      service = "main"
+    ),
+  ],
+  autogates = [
+    # Pyodide is included as a builtin wasm module so it requires the
+    # corresponding autogate flag.
+    "workerd-autogate-builtin-wasm-modules",
+  ]
+);
+
+const mainWorker :Workerd.Worker = (
+  modules = [
+    (name = "worker.py", pythonModule = embed "./worker.py"),
+  ],
+  compatibilityDate = "2023-12-18",
+  compatibilityFlags = ["experimental"],
+  bindings = [
+    (
+      name = "secret",
+      text = "thisisasecret"
+    ),
+  ],
+  # Learn more about compatibility dates at:
+  # https://developers.cloudflare.com/workers/platform/compatibility-dates/
+);

samples/pyodide-secret/worker.py

@@ -0,0 +1,11 @@
+from js import Response
+
+
+def fetch(request, env):
+    print(env.secret)
+    return Response.new("hello world")
+
+
+def test(ctx, env):
+    print(env.secret)
+    print("Hi there, this is a test")

src/pyodide/internal/metadatafs.js

@@ -0,0 +1,37 @@
+import { default as MetadataReader } from "pyodide-internal:runtime-generated/metadata";
+import { createReadonlyFS } from "pyodide-internal:readOnlyFS";
+
+export function createMetadataFS(Module) {
+  const TIMESTAMP = Date.now();
+  const names = MetadataReader.getNames();
+  const sizes = MetadataReader.getSizes();
+  const nameToIndex = new Map(names.map((val, idx) => [val, idx]));
+
+  const FSOps = {
+    getNodeMode(parent, name, index) {
+      return {
+        permissions: 0o555, // read and execute but not write
+        isDir: index === undefined,
+      };
+    },
+    setNodeAttributes(node, index, isDir) {
+      node.modtime = TIMESTAMP;
+      node.usedBytes = 0;
+      if (!isDir) {
+        node.index = index;
+        node.usedBytes = sizes[index];
+      }
+    },
+    readdir(node) {
+      return names;
+    },
+    lookup(parent, name) {
+      return nameToIndex.get(name);
+    },
+    read(stream, position, buffer) {
+      return MetadataReader.read(stream.node.index, position, buffer);
+    },
+  };
+
+  return createReadonlyFS(FSOps, Module);
+}

src/pyodide/internal/python.js

@@ -1,5 +1,6 @@
 import { parseTarInfo } from "pyodide-internal:tar";
 import { createTarFS } from "pyodide-internal:tarfs";
+import { createMetadataFS } from "pyodide-internal:metadatafs";
 
 /**
  * This file is a simplified version of the Pyodide loader:
@@ -262,10 +263,14 @@ function simpleRunPython(emscriptenModule, code) {
 function mountLib(pyodide) {
   const [info, _] = parseTarInfo();
   const tarFS = createTarFS(pyodide._module);
+  const mdFS = createMetadataFS(pyodide._module);
   pyodide.FS.mkdirTree("/session/lib/python3.11/site-packages");
+  pyodide.FS.mkdirTree("/session/metadata");
   pyodide.FS.mount(tarFS, { info }, "/session/lib/python3.11/site-packages");
+  pyodide.FS.mount(mdFS, {}, "/session/metadata");
   const sys = pyodide.pyimport("sys");
   sys.path.push("/session/lib/python3.11/site-packages");
+  sys.path.push("/session/metadata");
   sys.destroy();
 }
 

src/pyodide/internal/readOnlyFS.js

@@ -0,0 +1,77 @@
+export function createReadonlyFS(FSOps, Module) {
+  const FS = Module.FS;
+  const ReadOnlyFS = {
+    mount(mount) {
+      return ReadOnlyFS.createNode(null, "/", mount.opts.info);
+    },
+    createNode(parent, name, info) {
+      let { permissions: mode, isDir } = FSOps.getNodeMode(parent, name, info);
+      if (isDir) {
+        mode |= 1 << 14; // set S_IFDIR
+      } else {
+        mode |= 1 << 15; // set S_IFREG
+      }
+      var node = FS.createNode(parent, name, mode);
+      node.node_ops = ReadOnlyFS.node_ops;
+      node.stream_ops = ReadOnlyFS.stream_ops;
+      FSOps.setNodeAttributes(node, info, isDir);
+      return node;
+    },
+    node_ops: {
+      getattr(node) {
+        const size = node.usedBytes;
+        const mode = node.mode;
+        const t = new Date(node.modtime);
+        const blksize = 4096;
+        const blocks = ((size + blksize - 1) / blksize) | 0;
+        return {
+          dev: 1,
+          ino: node.id,
+          mode,
+          nlink: 1,
+          uid: 0,
+          gid: 0,
+          rdev: 0,
+          size,
+          atime: t,
+          mtime: t,
+          ctime: t,
+          blksize,
+          blocks,
+        };
+      },
+      readdir(node) {
+        return FSOps.readdir(node);
+      },
+      lookup(parent, name) {
+        const child = FSOps.lookup(parent, name);
+        if (child === undefined) {
+          throw FS.genericErrors[44]; // ENOENT
+        }
+        return ReadOnlyFS.createNode(parent, name, child);
+      },
+    },
+    stream_ops: {
+      llseek(stream, offset, whence) {
+        let position = offset;
+        if (whence === 1) {
+          // SEEK_CUR
+          position += stream.position;
+        } else if (whence === 2) {
+          // SEEK_END
+          if (FS.isFile(stream.node.mode)) {
+            position += stream.node.info.size;
+          }
+        }
+        return position;
+      },
+      read(stream, buffer, offset, length, position) {
+        if (position >= stream.node.usedBytes) return 0;
+        var size = Math.min(stream.node.usedBytes - position, length);
+        buffer = buffer.subarray(offset, offset + size);
+        return FSOps.read(stream, position, buffer);
+      },
+    },
+  };
+  return ReadOnlyFS;
+}

src/pyodide/internal/relaxed_call.py

@@ -0,0 +1,88 @@
+"""
+Allow calling functions with more arguments than they want
+
+Copied from
+https://github.com/pyodide/pyodide/pull/4392/
+"""
+
+from collections.abc import Callable
+from functools import lru_cache, wraps
+from inspect import Parameter, Signature, signature
+from typing import Any, ParamSpec, TypeVar
+
+
+def _relaxed_call_sig(func: Callable[..., Any]) -> Signature | None:
+    try:
+        sig = signature(func)
+    except (TypeError, ValueError):
+        return None
+    new_params = list(sig.parameters.values())
+    idx: int | None = -1
+    for idx, param in enumerate(new_params):
+        if param.kind in (Parameter.KEYWORD_ONLY, Parameter.VAR_KEYWORD):
+            break
+        if param.kind == Parameter.VAR_POSITIONAL:
+            idx = None
+            break
+    else:
+        idx += 1
+    if idx is not None:
+        new_params.insert(idx, Parameter("__var_positional", Parameter.VAR_POSITIONAL))
+
+    for param in new_params:
+        if param.kind == Parameter.VAR_KEYWORD:
+            break
+    else:
+        new_params.append(Parameter("__var_keyword", Parameter.VAR_KEYWORD))
+    new_sig = sig.replace(parameters=new_params)
+    return new_sig
+
+
+@lru_cache
+def _relaxed_call_sig_cached(func: Callable[..., Any]) -> Signature | None:
+    return _relaxed_call_sig(func)
+
+
+def _do_call(
+    func: Callable[..., Any], sig: Signature, args: Any, kwargs: dict[str, Any]
+) -> Any:
+    bound = sig.bind(*args, **kwargs)
+    bound.arguments.pop("__var_positional", None)
+    bound.arguments.pop("__var_keyword", None)
+    return func(*bound.args, **bound.kwargs)
+
+
+Param = ParamSpec("Param")
+Param2 = ParamSpec("Param2")
+RetType = TypeVar("RetType")
+
+
+def relaxed_wrap(func: Callable[Param, RetType]) -> Callable[..., RetType]:
+    """Decorator which creates a function that ignores extra arguments
+
+    If extra positional or keyword arguments are provided they will be
+    discarded.
+    """
+    sig = _relaxed_call_sig(func)
+    if sig is None:
+        raise TypeError("Cannot wrap function")
+    else:
+        sig2 = sig
+
+    @wraps(func)
+    def wrapper(*args: Param.args, **kwargs: Param.kwargs) -> RetType:
+        return _do_call(func, sig2, args, kwargs)
+
+    return wrapper
+
+
+def relaxed_call(func: Callable[..., RetType], *args: Any, **kwargs: Any) -> RetType:
+    """Call the function ignoring extra arguments
+
+    If extra positional or keyword arguments are provided they will be
+    discarded.
+    """
+    sig = _relaxed_call_sig_cached(func)
+    if sig is None:
+        return func(*args, **kwargs)
+    return _do_call(func, sig, args, kwargs)

src/pyodide/internal/tar.js

@@ -4,7 +4,6 @@ import { default as Reader } from "pyodide-internal:packages_tar_reader";
 // And some trial and error with real tar files.
 // https://en.wikipedia.org/wiki/Tar_(computing)#File_format
 
-
 const decoder = new TextDecoder();
 function decodeString(buf) {
   const nullIdx = buf.indexOf(0);