Merge pull request #467 from ruby/katei/dynamic-component

Use wasi-virt for Componentized build

builders/wasm32-unknown-emscripten/Dockerfile

@@ -23,7 +23,7 @@ ENV PATH=/usr/bin:$PATH
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.75
+    RUST_VERSION=1.76.0
 
 RUN set -eux pipefail; \
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \

builders/wasm32-unknown-wasi/Dockerfile

@@ -27,7 +27,7 @@ RUN set -eux pipefail; \
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.75
+    RUST_VERSION=1.76.0
 
 RUN set -eux pipefail; \
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \

ext/ruby_wasm/Cargo.toml

@@ -15,4 +15,6 @@ bytes = "1"
 wizer = "4.0.0"
 wasi-vfs-cli = { git = "https://github.com/kateinoigakukun/wasi-vfs/", tag = "0.5.2" }
 structopt = "0.3.26"
-wit-component = "0.203.0"
+wit-component = "0.212.0"
+wasm-compose = "0.212.0"
+wasi-virt = { git = "https://github.com/bytecodealliance/wasi-virt", rev = "02de7b495eba3f1bf786bf17cf2236b7277be7b0", default-features = false }

ext/ruby_wasm/src/lib.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, path::PathBuf};
+use std::{collections::HashMap, env, path::PathBuf, time::SystemTime};
 
 use magnus::{
     eval, exception, function, method,
@@ -8,6 +8,7 @@ use magnus::{
 };
 use structopt::StructOpt;
 use wizer::Wizer;
+use wasi_virt;
 
 static RUBY_WASM: value::Lazy<RModule> =
     value::Lazy::new(|ruby| ruby.define_module("RubyWasmExt").unwrap());
@@ -222,6 +223,99 @@ impl ComponentEncode {
     }
 }
 
+#[wrap(class = "RubyWasmExt::WasiVirt")]
+struct WasiVirt(std::cell::RefCell<Option<wasi_virt::WasiVirt>>);
+
+impl WasiVirt {
+    fn new() -> Self {
+        Self(std::cell::RefCell::new(Some(wasi_virt::WasiVirt::new())))
+    }
+
+    fn virt<R>(
+        &self,
+        body: impl FnOnce(&mut wasi_virt::WasiVirt) -> Result<R, Error>,
+    ) -> Result<R, Error> {
+        let mut virt = self.0.take().ok_or_else(|| {
+            Error::new(
+                exception::standard_error(),
+                "wasi virt is already consumed".to_string(),
+            )
+        })?;
+        let result = body(&mut virt)?;
+        self.0.replace(Some(virt));
+        Ok(result)
+    }
+
+    fn allow_all(&self) -> Result<(), Error> {
+        self.virt(|virt| {
+            virt.allow_all();
+            // Disable sockets for now since `sockets/ip-name-lookup` is not
+            // supported by @bytecodealliance/preview2-shim yet
+            virt.sockets(false);
+            Ok(())
+        })
+    }
+
+    fn map_dir(&self, guest_dir: String, host_dir: String) -> Result<(), Error> {
+        self.virt(|virt| {
+            virt.fs().virtual_preopen(guest_dir, host_dir);
+            Ok(())
+        })
+    }
+
+    fn finish(&self) -> Result<bytes::Bytes, Error> {
+        self.virt(|virt| {
+            let result = virt.finish().map_err(|e| {
+                Error::new(
+                    exception::standard_error(),
+                    format!("failed to generate virtualization adapter: {}", e),
+                )
+            })?;
+            Ok(result.adapter.into())
+        })
+    }
+
+    fn compose(&self, component_bytes: bytes::Bytes) -> Result<bytes::Bytes, Error> {
+        let virt_adapter = self.finish()?;
+        let tmpdir = env::temp_dir();
+        let tmp_virt = tmpdir.join(format!("virt{}.wasm", timestamp()));
+        std::fs::write(&tmp_virt, &virt_adapter).map_err(|e| {
+            Error::new(
+                exception::standard_error(),
+                format!("failed to write virt adapter: {}", e),
+            )
+        })?;
+        let tmp_component = tmpdir.join(format!("component{}.wasm", timestamp()));
+        std::fs::write(&tmp_component, &component_bytes).map_err(|e| {
+            Error::new(
+                exception::standard_error(),
+                format!("failed to write component: {}", e),
+            )
+        })?;
+
+        use wasm_compose::{composer, config};
+        let config = config::Config {
+            definitions: vec![tmp_virt],
+            ..Default::default()
+        };
+        let composer = composer::ComponentComposer::new(&tmp_component, &config);
+        let composed = composer.compose().map_err(|e| {
+            Error::new(
+                exception::standard_error(),
+                format!("failed to compose component: {}", e),
+            )
+        })?;
+        return Ok(composed.into());
+
+        fn timestamp() -> u64 {
+            match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
+                Ok(n) => n.as_secs(),
+                Err(_) => panic!(),
+            }
+        }
+    }
+}
+
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = RUBY_WASM.get_inner_with(ruby);
@@ -266,5 +360,12 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     )?;
     component_encode.define_method("encode", method!(ComponentEncode::encode, 0))?;
 
+    let wasi_virt = module.define_class("WasiVirt", ruby.class_object())?;
+    wasi_virt.define_singleton_method("new", function!(WasiVirt::new, 0))?;
+    wasi_virt.define_method("allow_all", method!(WasiVirt::allow_all, 0))?;
+    wasi_virt.define_method("map_dir", method!(WasiVirt::map_dir, 2))?;
+    wasi_virt.define_method("finish", method!(WasiVirt::finish, 0))?;
+    wasi_virt.define_method("compose", method!(WasiVirt::compose, 1))?;
+
     Ok(())
 }

lib/ruby_wasm/build/product/crossruby.rb

@@ -80,7 +80,7 @@ def do_extconf(executor, crossruby)
         "--target-rbconfig=#{rbconfig_rb}",
       ]
       extconf_args << "--enable-component-model" if @features.support_component_model?
-      executor.system Gem.ruby, *extconf_args
+      executor.system crossruby.baseruby_path, *extconf_args
     end
 
     def do_legacy_extconf(executor, crossruby)

lib/ruby_wasm/packager.rb

@@ -44,7 +44,7 @@ def package(executor, dest_dir, options)
       fs.remove_stdlib(executor)
     end
 
-    if full_build_options[:target] == "wasm32-unknown-wasip1" && !features.support_dynamic_linking?
+    if full_build_options[:target] == "wasm32-unknown-wasip1" && !features.support_component_model?
       # wasi-vfs supports only WASI target
       wasi_vfs = RubyWasmExt::WasiVfs.new
       wasi_vfs.map_dir("/bundle", fs.bundle_dir)
@@ -54,6 +54,20 @@ def package(executor, dest_dir, options)
     end
     wasm_bytes = ruby_core.link_gem_exts(executor, fs.ruby_root, fs.bundle_dir, wasm_bytes)
 
+    if features.support_component_model?
+      wasi_virt = RubyWasmExt::WasiVirt.new
+      wasi_virt.allow_all
+      [
+        { guest: "/bundle", host: fs.bundle_dir },
+        { guest: "/usr", host: File.dirname(fs.ruby_root) }
+      ].each do |map|
+        map => { guest:, host: }
+        RubyWasm.logger.debug "Adding files into VFS: #{host} => #{guest}"
+        wasi_virt.map_dir(guest, host)
+      end
+      wasm_bytes = wasi_virt.compose(wasm_bytes)
+    end
+
     wasm_bytes = RubyWasmExt.preinitialize(wasm_bytes) if options[:optimize]
     wasm_bytes
   end

lib/ruby_wasm/packager/core.rb

@@ -67,6 +67,14 @@ def wasi_exec_model
       use_js_gem ? "reactor" : "command"
     end
 
+    def with_unbundled_env(&block)
+      __skip__ = if defined?(Bundler)
+        Bundler.with_unbundled_env(&block)
+      else
+        block.call
+      end
+    end
+
     def cache_key(digest)
       raise NotImplementedError
     end
@@ -87,27 +95,22 @@ def build(executor, options)
       end
       build.crossruby.clean(executor) if options[:clean]
 
-      do_build =
-        proc do
-          build.crossruby.build(
-            executor,
-            remake: options[:remake],
-            reconfigure: options[:reconfigure]
-          )
-        end
+      self.with_unbundled_env do
+        build.crossruby.build(
+          executor,
+          remake: options[:remake],
+          reconfigure: options[:reconfigure]
+        )
+      end
 
-      __skip__ =
-        if defined?(Bundler)
-          Bundler.with_unbundled_env(&do_build)
-        else
-          do_build.call
-        end
       build.crossruby.artifact
     end
 
     def build_gem_exts(executor, gem_home)
       build = derive_build
-      self._build_gem_exts(executor, build, gem_home)
+      self.with_unbundled_env do
+        self._build_gem_exts(executor, build, gem_home)
+      end
     end
 
     def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
@@ -269,21 +272,14 @@ def build(executor, options)
       end
       build.crossruby.clean(executor) if options[:clean]
 
-      do_build =
-        proc do
-          build.crossruby.build(
-            executor,
-            remake: options[:remake],
-            reconfigure: options[:reconfigure]
-          )
-        end
+      self.with_unbundled_env do
+        build.crossruby.build(
+          executor,
+          remake: options[:remake],
+          reconfigure: options[:reconfigure]
+        )
+      end
 
-      __skip__ =
-        if defined?(Bundler)
-          Bundler.with_unbundled_env(&do_build)
-        else
-          do_build.call
-        end
       build.crossruby.artifact
     end
 

packages/npm-packages/ruby-wasm-wasi/test/init.js

@@ -73,8 +73,13 @@ async function initComponentRubyVM({ suppressStderr } = { suppressStderr: false
     return module;
   }
   const vm = await RubyVM._instantiate(async (jsRuntime) => {
-    const { cli, clocks, filesystem, io, random, sockets } = preview2Shim;
-    filesystem._setPreopens({})
+    const { cli, clocks, filesystem, io, random, sockets, http } = preview2Shim;
+    if (process.env.RUBY_BUILD_ROOT) {
+      filesystem._setPreopens({
+        "/usr": path.join(process.env.RUBY_BUILD_ROOT, "usr"),
+        "/bundle": path.join(process.env.RUBY_BUILD_ROOT, "bundle"),
+      })
+    }
     cli._setArgs(["ruby.wasm"].concat(process.argv.slice(2)));
     cli._setCwd("/")
     const root = await instantiate(getCoreModule, {
@@ -98,6 +103,9 @@ async function initComponentRubyVM({ suppressStderr } = { suppressStderr: false
       "wasi:io/streams": io.streams,
       "wasi:random/random": random.random,
       "wasi:sockets/tcp": sockets.tcp,
+      "wasi:http/types": http.types,
+      "wasi:http/incoming-handler": http.incomingHandler,
+      "wasi:http/outgoing-handler": http.outgoingHandler,
     })
     return root.rubyRuntime;
   }, {})

packages/npm-packages/ruby-wasm-wasi/tools/run-test-unit.mjs

@@ -35,11 +35,14 @@ const instantiateComponent = async (rootTestFile) => {
     return WebAssembly.compile(buffer);
   }
   const vm = await RubyVM._instantiate(async (jsRuntime) => {
-    const { cli, clocks, filesystem, io, random, sockets } = preview2Shim;
+    const { cli, clocks, filesystem, io, random, sockets, http } = preview2Shim;
     const dirname = path.dirname(new URL(import.meta.url).pathname);
-    filesystem._setPreopens({
-      "/__root__": path.join(dirname, ".."),
-    })
+    const preopens = { "/__root__": path.join(dirname, "..") };
+    if (process.env.RUBY_ROOT) {
+      preopens["/usr"] = path.join(process.env.RUBY_ROOT, "usr");
+      preopens["/bundle"] = path.join(process.env.RUBY_ROOT, "bundle");
+    }
+    filesystem._setPreopens(preopens);
     cli._setArgs(["ruby.wasm"].concat(process.argv.slice(2)));
     cli._setCwd("/")
     const root = await instantiate(getCoreModule, {
@@ -63,6 +66,9 @@ const instantiateComponent = async (rootTestFile) => {
       "wasi:io/streams": io.streams,
       "wasi:random/random": random.random,
       "wasi:sockets/tcp": sockets.tcp,
+      "wasi:http/types": http.types,
+      "wasi:http/incoming-handler": http.incomingHandler,
+      "wasi:http/outgoing-handler": http.outgoingHandler,
     })
     return root.rubyRuntime;
   }, {

rakelib/packaging.rake

@@ -80,7 +80,7 @@ namespace :npm do
               component_path = File.join(pkg_dir, "tmp", "ruby.component.wasm")
               FileUtils.mkdir_p(File.dirname(component_path))
 
-              sh env.merge("RUBY_WASM_EXPERIMENTAL_COMPONENT_MODEL" => "1"),
+              sh env.merge("RUBY_WASM_EXPERIMENTAL_DYNAMIC_LINKING" => "1"),
                  *build_command, "-o", component_path
               sh "npx", "jco", "transpile",
                 "--no-wasi-shim", "--instantiation", "--valid-lifting-optimization",

sig/ruby_wasm/ext.rbs

@@ -33,4 +33,12 @@ module RubyWasmExt
     def import_name_map: (Hash[String, String] map) -> void
     def encode: () -> bytes
   end
+
+  class WasiVirt
+    def initialize: () -> void
+    def allow_all: () -> void
+    def map_dir: (String guest_path, String host_path) -> void
+    def finish: () -> bytes
+    def compose: (bytes component_bytes) -> bytes
+  end
 end

sig/ruby_wasm/packager.rbs

@@ -77,6 +77,7 @@ class RubyWasm::Packager
       def build_gem_exts: (RubyWasm::BuildExecutor, string gem_home) -> void
       def link_gem_exts: (RubyWasm::BuildExecutor, string ruby_root, string gem_home, bytes module_bytes) -> bytes
       def wasi_exec_model: () -> String
+      def with_unbundled_env: () { () -> void } -> void
     end
 
     class DynamicLinking < RubyWasm::Packager::Core::BuildStrategy