Skip to content

Commit

Permalink
Mount python worker files after taking memory snapshot
Browse files Browse the repository at this point in the history
This ensures that the contents of worker files cannot be accessed prior to
taking the snapshot and so won't appear in the linear memory.
  • Loading branch information
hoodmane committed Oct 1, 2024
1 parent 4eef7fd commit 08bad5a
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 6 deletions.
8 changes: 6 additions & 2 deletions src/pyodide/internal/python.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ import {
TRANSITIVE_REQUIREMENTS,
SITE_PACKAGES,
adjustSysPath,
mountLib,
mountSitePackages,
mountWorkerFiles,
} from 'pyodide-internal:setupPackages';
import { reportError } from 'pyodide-internal:util';
import {
Expand Down Expand Up @@ -201,7 +202,7 @@ async function instantiateEmscriptenModule(
*/
async function prepareWasmLinearMemory(Module: Module): Promise<void> {
// Note: if we are restoring from a snapshot, runtime is not initialized yet.
mountLib(Module, SITE_PACKAGES.rootInfo);
mountSitePackages(Module, SITE_PACKAGES.rootInfo);
entropyMountFiles(Module);
if (SHOULD_RESTORE_SNAPSHOT) {
restoreSnapshot(Module);
Expand Down Expand Up @@ -229,6 +230,9 @@ export async function loadPyodide(
prepareWasmLinearMemory(Module)
);
maybeSetupSnapshotUpload(Module);
// Mount worker files after doing snapshot upload so we ensure that data from the files is never
// present in snapshot memory.
mountWorkerFiles(Module);

// Finish setting up Pyodide's ffi so we can use the nice Python interface
await enterJaegerSpan('finalize_bootstrap', Module.API.finalizeBootstrap);
Expand Down
13 changes: 10 additions & 3 deletions src/pyodide/internal/setupPackages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,19 +189,26 @@ export function getSitePackagesPath(Module: Module): string {
* details, so even though we want these directories to be on sys.path, we
* handle that separately in adjustSysPath.
*/
export function mountLib(Module: Module, info: TarFSInfo): void {
export function mountSitePackages(Module: Module, info: TarFSInfo): void {
const tarFS = createTarFS(Module);
const mdFS = createMetadataFS(Module);
const site_packages = getSitePackagesPath(Module);
Module.FS.mkdirTree(site_packages);
Module.FS.mkdirTree('/session/metadata');
if (!LOAD_WHEELS_FROM_R2 && !LOAD_WHEELS_FROM_ARTIFACT_BUNDLER) {
// if we are not loading additional wheels, then we're done
// with site-packages and we can mount it here. Otherwise, we must mount it in
// loadPackages().
Module.FS.mount(tarFS, { info }, site_packages);
}
}

export function mountWorkerFiles(Module: Module) {
Module.FS.mkdirTree('/session/metadata');
const mdFS = createMetadataFS(Module);
Module.FS.mount(mdFS, {}, '/session/metadata');
simpleRunPython(
Module,
`from importlib import invalidate_caches; invalidate_caches(); del invalidate_caches`
);
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/workerd/io/compatibility-date.capnp
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
pythonWorkers @43 :Bool
$compatEnableFlag("python_workers")
$pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
packages = "2024-03-01", backport = 0)
packages = "2024-03-01", backport = 1)
$impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
# Enables Python Workers. Access to this flag is not restricted, instead bundles containing
# Python modules are restricted in EWC.
Expand Down

0 comments on commit 08bad5a

Please sign in to comment.