Merge pull request #4654 from wasmerio/store-hash-in-artifact

Store and restore hash from a compiled module

.github/cross-linux-riscv64/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update  && \
 # install rust tools
 RUN curl --proto "=https" --tlsv1.2 --retry 3 -sSfL https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
-RUN rustup -v toolchain install 1.73
+RUN rustup -v toolchain install 1.74
 # add docker the manual way
 COPY install_docker.sh /
 RUN /install_docker.sh
@@ -61,7 +61,7 @@ ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc
     RUST_TEST_THREADS=1 \
     PKG_CONFIG_PATH="/usr/lib/riscv64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"
 
-RUN rustup target add riscv64gc-unknown-linux-gnu --toolchain 1.73-x86_64-unknown-linux-gnu
+RUN rustup target add riscv64gc-unknown-linux-gnu --toolchain 1.74-x86_64-unknown-linux-gnu
 
 #compile libssl-dev for riscv64!
 COPY build_openssl.sh /

.github/workflows/benchmark.yaml

@@ -7,7 +7,7 @@ on:
 
 env:
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 jobs:
   run_benchmark:

.github/workflows/build.yml

@@ -3,7 +3,7 @@ name: Builds
 env:
   RUST_BACKTRACE: 1
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 on:
   push:

.github/workflows/cloudcompiler.yaml

@@ -2,7 +2,7 @@ name: Release cloudcompiler.wasm
 
 env:
   RUST_BACKTRACE: 1
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 on:
   push:

.github/workflows/documentation.yaml

@@ -8,7 +8,7 @@ on:
       - 'lib/**'
 
 env:
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 jobs:
   documentation:

.github/workflows/test.yaml

@@ -24,7 +24,7 @@ env:
   # Rust, but it's not stable on 1.69 yet. By explicitly setting the protocol we
   # can override that behaviour
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
-  MSRV: "1.73"
+  MSRV: "1.74"
   NEXTEST_PROFILE: "ci"
 
 jobs:

Cargo.toml

@@ -83,7 +83,7 @@ edition = "2021"
 homepage = "https://wasmer.io/"
 license = "MIT"
 repository = "https://github.com/wasmerio/wasmer"
-rust-version = "1.73"
+rust-version = "1.74"
 version = "4.3.0-beta.1"
 
 [workspace.dependencies]

lib/api/src/jsc/module.rs

@@ -67,6 +67,7 @@ impl Module {
     ///
     pub(crate) unsafe fn from_js_module(module: JSObject, binary: impl IntoBytes) -> Self {
         let binary = binary.into_bytes();
+
         // The module is now validated, so we can safely parse it's types
         let info = crate::module_info_polyfill::translate_module(&binary[..])
             .unwrap()

lib/api/src/sys/engine.rs

@@ -6,7 +6,7 @@ pub use wasmer_compiler::{
 };
 #[cfg(feature = "compiler")]
 use wasmer_types::Features;
-use wasmer_types::{DeserializeError, Target};
+use wasmer_types::{DeserializeError, HashAlgorithm, Target};
 
 /// Get the default config for the sys Engine
 #[allow(unreachable_code)]
@@ -57,6 +57,9 @@ pub trait NativeEngineExt {
     #[cfg(feature = "compiler")]
     fn new(compiler_config: Box<dyn CompilerConfig>, target: Target, features: Features) -> Self;
 
+    /// Sets the hash algorithm
+    fn set_hash_algorithm(&mut self, hash_algorithm: Option<HashAlgorithm>);
+
     /// Create a headless `Engine`
     ///
     /// A headless engine is an engine without any compiler attached.
@@ -153,4 +156,8 @@ impl NativeEngineExt for crate::engine::Engine {
             artifact,
         )))
     }
+
+    fn set_hash_algorithm(&mut self, hash_algorithm: Option<HashAlgorithm>) {
+        self.0.set_hash_algorithm(hash_algorithm)
+    }
 }

lib/cli-compiler/src/commands/compile.rs

@@ -59,7 +59,9 @@ impl Compile {
             })
             .unwrap_or_default();
         let (engine_builder, compiler_type) = self.store.get_engine_for_target(target.clone())?;
-        let engine = engine_builder.engine();
+        let engine = engine_builder
+            .set_hash_algorithm(Some(wasmer_types::HashAlgorithm::Sha256))
+            .engine();
         let output_filename = self
             .output
             .file_stem()
@@ -105,6 +107,7 @@ impl Compile {
             &target,
             memory_styles,
             table_styles,
+            engine.hash_algorithm(),
         )?;
         let serialized = artifact.serialize()?;
         fs::write(output_filename, serialized)?;

lib/cli/src/commands/binfmt.rs

@@ -55,7 +55,7 @@ fn seccheck(path: &Path) -> Result<()> {
 
 impl Binfmt {
     /// The filename used to register the wasmer CLI as a binfmt interpreter.
-    pub const FILENAME: &str = "wasmer-binfmt-interpreter";
+    pub const FILENAME: &'static str = "wasmer-binfmt-interpreter";
 
     /// execute [Binfmt]
     pub fn execute(&self) -> Result<()> {

lib/cli/src/commands/compile.rs

@@ -4,7 +4,7 @@ use anyhow::{Context, Result};
 use clap::Parser;
 use wasmer::*;
 
-use crate::{store::StoreOptions, warning};
+use crate::{common::HashAlgorithm, store::StoreOptions, warning};
 
 #[derive(Debug, Parser)]
 /// The options for the `wasmer compile` subcommand
@@ -26,6 +26,10 @@ pub struct Compile {
 
     #[clap(short = 'm')]
     cpu_features: Vec<CpuFeature>,
+
+    /// Hashing algorithm to be used for module hash
+    #[clap(long, value_enum)]
+    hash_algorithm: Option<HashAlgorithm>,
 }
 
 impl Compile {
@@ -54,6 +58,11 @@ impl Compile {
             })
             .unwrap_or_default();
         let (store, compiler_type) = self.store.get_store_for_target(target.clone())?;
+
+        let mut engine = store.engine().clone();
+        let hash_algorithm = self.hash_algorithm.unwrap_or_default().into();
+        engine.set_hash_algorithm(Some(hash_algorithm));
+
         let output_filename = self
             .output
             .file_stem()

lib/cli/src/commands/create_exe.rs

@@ -21,7 +21,10 @@ use webc::{
 };
 
 use self::utils::normalize_atom_name;
-use crate::{common::normalize_path, store::CompilerOptions};
+use crate::{
+    common::{normalize_path, HashAlgorithm},
+    store::CompilerOptions,
+};
 
 const LINK_SYSTEM_LIBRARIES_WINDOWS: &[&str] = &["userenv", "Ws2_32", "advapi32", "bcrypt"];
 
@@ -91,6 +94,10 @@ pub struct CreateExe {
 
     #[clap(flatten)]
     compiler: CompilerOptions,
+
+    /// Hashing algorithm to be used for module hash
+    #[clap(long, value_enum)]
+    hash_algorithm: Option<HashAlgorithm>,
 }
 
 /// Url or version to download the release from
@@ -216,6 +223,10 @@ impl CreateExe {
 
         let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?;
 
+        let mut engine = store.engine().clone();
+        let hash_algorithm = self.hash_algorithm.unwrap_or_default().into();
+        engine.set_hash_algorithm(Some(hash_algorithm));
+
         println!("Compiler: {}", compiler_type.to_string());
         println!("Target: {}", target.triple());
         println!(
@@ -380,7 +391,7 @@ pub(super) fn compile_pirita_into_directory(
     let volume_path = target_dir.join("volumes").join("volume.o");
     write_volume_obj(&volume_bytes, volume_name, &volume_path, target)?;
     let volume_path = volume_path.canonicalize()?;
-    let volume_path = pathdiff::diff_paths(&volume_path, &target_dir).unwrap();
+    let volume_path = pathdiff::diff_paths(volume_path, &target_dir).unwrap();
 
     std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| {
         anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display())

lib/cli/src/commands/inspect.rs

@@ -27,6 +27,7 @@ impl Inspect {
 
     fn inner_execute(&self) -> Result<()> {
         let (store, _compiler_type) = self.store.get_store()?;
+
         let module_contents = std::fs::read(&self.path)?;
         let iswasm = is_wasm(&module_contents);
         let module_len = module_contents.len();

lib/cli/src/commands/run/mod.rs

@@ -21,14 +21,18 @@ use once_cell::sync::Lazy;
 use sha2::{Digest, Sha256};
 use tempfile::NamedTempFile;
 use url::Url;
+#[cfg(feature = "sys")]
+use wasmer::NativeEngineExt;
 use wasmer::{
     DeserializeError, Engine, Function, Imports, Instance, Module, Store, Type, TypedFunction,
     Value,
 };
+
 #[cfg(feature = "compiler")]
 use wasmer_compiler::ArtifactBuild;
 use wasmer_config::package::PackageSource as PackageSpecifier;
 use wasmer_registry::{wasmer_env::WasmerEnv, Package};
+use wasmer_types::ModuleHash;
 #[cfg(feature = "journal")]
 use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger};
 use wasmer_wasix::{
@@ -43,16 +47,17 @@ use wasmer_wasix::{
         MappedCommand, MappedDirectory, Runner,
     },
     runtime::{
-        module_cache::{CacheError, ModuleHash},
-        package_loader::PackageLoader,
-        resolver::QueryError,
+        module_cache::CacheError, package_loader::PackageLoader, resolver::QueryError,
         task_manager::VirtualTaskManagerExt,
     },
     Runtime, WasiError,
 };
 use webc::{metadata::Manifest, Container};
 
-use crate::{commands::run::wasi::Wasi, error::PrettyError, logging::Output, store::StoreOptions};
+use crate::{
+    commands::run::wasi::Wasi, common::HashAlgorithm, error::PrettyError, logging::Output,
+    store::StoreOptions,
+};
 
 const TICK: Duration = Duration::from_millis(250);
 
@@ -81,6 +86,9 @@ pub struct Run {
     input: PackageSource,
     /// Command-line arguments passed to the package
     args: Vec<String>,
+    /// Hashing algorithm to be used for module hash
+    #[clap(long, value_enum)]
+    hash_algorithm: Option<HashAlgorithm>,
 }
 
 impl Run {
@@ -115,12 +123,21 @@ impl Run {
 
         let _guard = handle.enter();
         let (store, _) = self.store.get_store()?;
-        let runtime = self.wasi.prepare_runtime(
-            store.engine().clone(),
-            &self.env,
-            runtime,
-            preferred_webc_version,
-        )?;
+
+        #[cfg(feature = "sys")]
+        let engine = {
+            let mut engine = store.engine().clone();
+            let hash_algorithm = self.hash_algorithm.unwrap_or_default().into();
+            engine.set_hash_algorithm(Some(hash_algorithm));
+
+            engine
+        };
+        #[cfg(not(feature = "sys"))]
+        let engine = store.engine().clone();
+
+        let runtime =
+            self.wasi
+                .prepare_runtime(engine, &self.env, runtime, preferred_webc_version)?;
 
         // This is a slow operation, so let's temporarily wrap the runtime with
         // something that displays progress
@@ -484,6 +501,7 @@ impl Run {
             coredump_on_trap: None,
             input: PackageSource::infer(executable)?,
             args: args.to_vec(),
+            hash_algorithm: None,
         })
     }
 }
@@ -721,10 +739,10 @@ impl ExecutableTarget {
                 let engine = runtime.engine();
                 pb.set_message("Deserializing pre-compiled WebAssembly module");
                 let module = unsafe { Module::deserialize_from_file(&engine, path)? };
-                let module_hash = {
-                    let wasm = std::fs::read(path)?;
-                    ModuleHash::xxhash(wasm)
-                };
+
+                let module_hash = module.info().hash.ok_or_else(|| {
+                    anyhow::Error::msg("module hash is not present in the artifact")
+                })?;
 
                 Ok(ExecutableTarget::WebAssembly {
                     module,

lib/cli/src/commands/run/wasi.rs

@@ -15,6 +15,7 @@ use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuild
 use wasmer::{Engine, Function, Instance, Memory32, Memory64, Module, RuntimeError, Store, Value};
 use wasmer_config::package::PackageSource as PackageSpecifier;
 use wasmer_registry::wasmer_env::WasmerEnv;
+use wasmer_types::ModuleHash;
 #[cfg(feature = "journal")]
 use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger};
 use wasmer_wasix::{
@@ -27,7 +28,7 @@ use wasmer_wasix::{
     rewind_ext,
     runners::{MappedCommand, MappedDirectory},
     runtime::{
-        module_cache::{FileSystemCache, ModuleCache, ModuleHash},
+        module_cache::{FileSystemCache, ModuleCache},
         package_loader::{BuiltinPackageLoader, PackageLoader},
         resolver::{FileSystemSource, InMemorySource, MultiSource, Source, WapmSource, WebSource},
         task_manager::{

lib/cli/src/commands/validate.rs

@@ -25,6 +25,7 @@ impl Validate {
     }
     fn inner_execute(&self) -> Result<()> {
         let (store, _compiler_type) = self.store.get_store()?;
+
         let module_contents = std::fs::read(&self.path)?;
         if !is_wasm(&module_contents) {
             bail!("`wasmer validate` only validates WebAssembly files");