From b83afe8885ae169b89c6b9629ed0b1789387db04 Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Thu, 5 Sep 2024 14:21:56 +0200
Subject: [PATCH] Add code to generate a Python baseline snapshot

---
 .../make-python-baseline-snapshot.yml         | 70 +++++++++++++++++++
 src/workerd/io/compatibility-date.capnp       |  6 ++
 src/workerd/server/workerd-api.c++            |  6 +-
 3 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/make-python-baseline-snapshot.yml

diff --git a/.github/workflows/make-python-baseline-snapshot.yml b/.github/workflows/make-python-baseline-snapshot.yml
new file mode 100644
index 00000000000..cc269e56583
--- /dev/null
+++ b/.github/workflows/make-python-baseline-snapshot.yml
@@ -0,0 +1,70 @@
+name: Make Python Baseline Snapshot
+
+permissions:
+    id-token: write # This is required for requesting the JWT
+    contents: read  # This is required for actions/checkout
+
+on:
+    workflow_dispatch:
+        inputs:
+            tag:
+                description: The release tag/version string
+            dry-run:
+                description: Actually upload or just test build step?
+                default: false
+                type: boolean
+    push:
+      branches:
+        - 'hoodmane/pyodide-load-from-file'
+
+jobs:
+  version:
+    outputs:
+      version: ${{ steps.echo.outputs.version }}
+    # version job uses ubuntu 24.04, this way we don't have to install the updated clang while
+    # the build job uses 20.04 for libc compatibility.
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - id: echo
+        run: |
+          echo "::set-output name=version::1.$(cat src/workerd/io/supported-compatibility-date.txt | tr -d '-').0"
+
+  build:
+    runs-on: ubuntu-20.04
+    name: build Python runtime
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Setup Linux
+        if: runner.os == 'Linux'
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          wget https://apt.llvm.org/llvm.sh
+          sed -i '/apt-get install/d' llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 15
+          sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev
+          echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+          echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+      - name: Configure download mirrors
+        shell: bash
+        run: |
+          if [ ! -z "${{ secrets.WORKERS_MIRROR_URL }}" ] ; then
+            # Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it.
+            sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE
+          fi
+      - name: Bazel build
+        # Strip debug info here – we don't generate debug info but some is pulled in from external
+        # static libraries, for example the Rust STL. This is equivalent to the -Wl,-S linker
+        # option, symbols will not be removed.
+        run: |
+          bazelisk build --config=release_linux @workerd//src/workerd/server:workerd
+          cd samples/pyodide
+          ../../bazel-bin/src/workerd/server/workerd test config.capnp \
+            --python-save-baseline-snapshot \
+            --pyodide-package-disk-cache-dir=save_dir \
+          | tee log.txt
+          cat log.txt | grep inputs.tag
+          cp save_dir/snapshot.bin pyodide_${{ inputs.tag }}.capnp.bin
diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp
index fe6b249d1d0..f7a3ca197b7 100644
--- a/src/workerd/io/compatibility-date.capnp
+++ b/src/workerd/io/compatibility-date.capnp
@@ -601,4 +601,10 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   # Enables the withSessions(commitTokenOrConstraint) method that allows users
   # to use read-replication for D1.
   # Experimental since this is not yet ready and is only meant for internal testing during development.
+  pythonWorkers2 @62 :Bool
+      $compatEnableFlag("python_workers_2")
+      $pythonSnapshotRelease(pyodide = "0.26.2", pyodideRevision = "2024-09-04",
+          packages = "2024-03-01", backport = 2)
+      $impliedByAfterDate(name = "pythonWorkers", date = "2000-01-01");
+
 }
diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++
index 7b63ad9c427..67a1513e1de 100644
--- a/src/workerd/server/workerd-api.c++
+++ b/src/workerd/server/workerd-api.c++
@@ -440,6 +440,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe<kj::Own<const kj::Directory>>&
 
 kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
     const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
+  KJ_DBG(version);
   KJ_IF_SOME(version, pyConfig.pyodideBundleManager.getPyodideBundle(version)) {
     return version;
   }
@@ -476,12 +477,13 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
       kj::HttpHeaders headers(table);
 
       kj::String url =
-          kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_",
-              version, ".capnp.bin");
+          kj::str("https://www.googleapis.com/storage/v1/b/cloudflare-edgeworker-python-packages/o/python-runtime-capnp-bin%2Fpyodide_",
+              version, ".capnp.bin?alt=media");
 
       auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers);
 
       auto res = req.response.wait(io.waitScope);
+      KJ_DBG(url, res.statusCode);
       auto body = res.body->readAllBytes().wait(io.waitScope);
 
       writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body);