Pinned heap test

benchmarks/lucet-benchmarks/src/context.rs

@@ -9,7 +9,8 @@ fn context_init(c: &mut Criterion) {
 
     c.bench_function("context_init", move |b| {
         b.iter(|| {
-            ContextHandle::create_and_init(&mut *stack, f as usize, &[]).unwrap();
+            ContextHandle::create_and_init(&mut *stack, f as usize, &[], std::ptr::null_mut())
+                .unwrap();
         })
     });
 }
@@ -22,7 +23,13 @@ fn context_swap_return(c: &mut Criterion) {
         b.iter_batched(
             || {
                 let mut stack = vec![0u64; 1024].into_boxed_slice();
-                let child = ContextHandle::create_and_init(&mut *stack, f as usize, &[]).unwrap();
+                let child = ContextHandle::create_and_init(
+                    &mut *stack,
+                    f as usize,
+                    &[],
+                    std::ptr::null_mut(),
+                )
+                .unwrap();
                 (stack, child)
             },
             |(stack, mut child)| unsafe {
@@ -44,8 +51,13 @@ fn context_init_swap_return(c: &mut Criterion) {
             || vec![0u64; 1024].into_boxed_slice(),
             |mut stack| {
                 let mut parent = ContextHandle::new();
-                let mut child =
-                    ContextHandle::create_and_init(&mut *stack, f as usize, &[]).unwrap();
+                let mut child = ContextHandle::create_and_init(
+                    &mut *stack,
+                    f as usize,
+                    &[],
+                    std::ptr::null_mut(),
+                )
+                .unwrap();
                 unsafe { Context::swap(&mut parent, &mut child) };
                 stack
             },
@@ -332,8 +344,13 @@ fn context_init_swap_return_many_args(c: &mut Criterion) {
             || vec![0u64; 1024].into_boxed_slice(),
             |mut stack| {
                 let mut parent = ContextHandle::new();
-                let mut child =
-                    ContextHandle::create_and_init(&mut *stack, f as usize, &args).unwrap();
+                let mut child = ContextHandle::create_and_init(
+                    &mut *stack,
+                    f as usize,
+                    &args,
+                    std::ptr::null_mut(),
+                )
+                .unwrap();
                 unsafe { Context::swap(&mut parent, &mut child) };
                 stack
             },

lucet-module/src/module_data.rs

@@ -59,6 +59,7 @@ pub struct ModuleFeatures {
     pub lzcnt: bool,
     pub popcnt: bool,
     pub instruction_count: bool,
+    pub pinned_heap: bool,
     _hidden: (),
 }
 
@@ -75,6 +76,7 @@ impl ModuleFeatures {
             lzcnt: false,
             popcnt: false,
             instruction_count: false,
+            pinned_heap: false,
             _hidden: (),
         }
     }

lucet-runtime/lucet-runtime-internals/src/alloc/tests.rs

@@ -610,6 +610,7 @@ macro_rules! alloc_tests {
                     inst.alloc_mut().stack_u64_mut(),
                     heap_touching_child as usize,
                     &[Val::CPtr(heap_ptr)],
+                    heap_ptr,
                 )
                 .expect("context init succeeds");
                 Context::swap(&mut parent, &mut child);
@@ -659,6 +660,7 @@ macro_rules! alloc_tests {
                     inst.alloc_mut().stack_u64_mut(),
                     stack_pattern_child as usize,
                     &[Val::CPtr(heap_ptr)],
+                    heap_ptr,
                 )
                 .expect("context init succeeds");
                 Context::swap(&mut parent, &mut child);

lucet-runtime/lucet-runtime-internals/src/context/context_asm.S

@@ -118,6 +118,7 @@ _lucet_context_swap:
     mov %r14, (6*8)(%rdi)
     mov %r15, (7*8)(%rdi)
     mov %rsi, (8*8)(%rdi)
+    mov %r8,  (9*8)(%rdi)
 
     movdqu %xmm0, (10*8 + 0*16)(%rdi)
     movdqu %xmm1, (10*8 + 1*16)(%rdi)
@@ -137,6 +138,7 @@ _lucet_context_swap:
     mov (5*8)(%rsi), %r13
     mov (6*8)(%rsi), %r14
     mov (7*8)(%rsi), %r15
+    mov (9*8)(%rsi), %r8
 
     movdqu (10*8 + 0*16)(%rsi), %xmm0
     movdqu (10*8 + 1*16)(%rsi), %xmm1

lucet-runtime/lucet-runtime-internals/src/context/mod.rs

@@ -35,6 +35,7 @@ pub(crate) struct GpRegs {
     r14: u64,
     r15: u64,
     pub(crate) rsi: u64,
+    r8: u64,
 }
 
 impl GpRegs {
@@ -49,6 +50,7 @@ impl GpRegs {
             r14: 0,
             r15: 0,
             rsi: 0,
+            r8: 0,
         }
     }
 }
@@ -206,9 +208,10 @@ impl ContextHandle {
         stack: &mut [u64],
         fptr: usize,
         args: &[Val],
+        heap: *mut core::ffi::c_void,
     ) -> Result<ContextHandle, Error> {
         let mut child = ContextHandle::new();
-        Context::init(stack, &mut child, fptr, args)?;
+        Context::init(stack, &mut child, fptr, args, heap)?;
         Ok(child)
     }
 }
@@ -301,6 +304,7 @@ impl Context {
     ///     &mut child,
     ///     entrypoint as usize,
     ///     &[Val::U64(120), Val::F32(3.14)],
+    ///     std::ptr::null_mut(),
     /// );
     /// assert!(res.is_ok());
     /// ```
@@ -324,6 +328,7 @@ impl Context {
     ///     &mut child,
     ///     entrypoint as usize,
     ///     &[Val::U64(120), Val::F32(3.14)],
+    ///     std::ptr::null_mut(),
     /// );
     /// assert!(res.is_ok());
     /// ```
@@ -365,6 +370,7 @@ impl Context {
         child: &mut Context,
         fptr: usize,
         args: &[Val],
+        heap: *mut core::ffi::c_void,
     ) -> Result<(), Error> {
         Context::init_with_callback(
             stack,
@@ -373,6 +379,7 @@ impl Context {
             ptr::null_mut(),
             fptr,
             args,
+            heap,
         )
     }
 
@@ -391,6 +398,7 @@ impl Context {
         backstop_data: *mut Instance,
         fptr: usize,
         args: &[Val],
+        heap: *mut core::ffi::c_void,
     ) -> Result<(), Error> {
         if !stack_is_aligned(stack) {
             return Err(Error::UnalignedStack);
@@ -471,6 +479,9 @@ impl Context {
 
         child.gpr.rbp = child as *const Context as u64;
 
+        // testing out heap pinning
+        child.gpr.r15 = heap as u64;
+
         // Read the mask to be restored if we ever need to jump out of a signal handler. If this
         // isn't possible, die.
         signal::pthread_sigmask(
@@ -552,6 +563,7 @@ impl Context {
     ///     &mut child,
     ///     entrypoint as usize,
     ///     &[],
+    ///     std::ptr::null_mut(),
     /// ).unwrap();
     ///
     /// unsafe { Context::swap(&mut parent, &mut child); }

lucet-runtime/lucet-runtime-internals/src/context/tests/c_child.rs

@@ -55,6 +55,7 @@ macro_rules! init_and_swap {
                 &mut *$stack,
                 $fn as usize,
                 &[$( $args ),*],
+                std::ptr::null_mut(),
             ).unwrap()));
 
             child_regs = child;

lucet-runtime/lucet-runtime-internals/src/context/tests/mod.rs

@@ -31,7 +31,12 @@ fn init_rejects_unaligned() {
     let mut stack_unaligned = unsafe { slice::from_raw_parts_mut(ptr, len) };
 
     // now we have the unaligned stack, let's make sure it blows up right
-    let res = ContextHandle::create_and_init(&mut stack_unaligned, dummy as usize, &[]);
+    let res = ContextHandle::create_and_init(
+        &mut stack_unaligned,
+        dummy as usize,
+        &[],
+        std::ptr::null_mut(),
+    );
 
     if let Err(Error::UnalignedStack) = res {
         assert!(true);

lucet-runtime/lucet-runtime-internals/src/context/tests/rust_child.rs

@@ -51,6 +51,7 @@ macro_rules! init_and_swap {
                 &mut *$stack,
                 $fn as usize,
                 &[$( $args ),*],
+                std::ptr::null_mut(),
             ).unwrap();
             CHILD = Some(child);
 

lucet-runtime/lucet-runtime-internals/src/instance.rs

@@ -843,7 +843,8 @@ impl Instance {
 
         self.entrypoint = Some(func.ptr);
 
-        let mut args_with_vmctx = vec![Val::from(self.alloc.slot().heap)];
+        let heap = self.alloc.slot().heap;
+        let mut args_with_vmctx = vec![Val::from(heap)];
         args_with_vmctx.extend_from_slice(args);
 
         let self_ptr = self as *mut _;
@@ -854,6 +855,7 @@ impl Instance {
             self_ptr,
             func.ptr.as_usize(),
             &args_with_vmctx,
+            heap,
         )?;
 
         // Set up the guest to set itself as terminable, then continue to

lucet-runtime/lucet-runtime-tests/src/stack.rs

@@ -157,7 +157,9 @@ macro_rules! stack_tests {
             expect_stack_overflow(
                 // Same note as `expect_ok_locals64_481`
                 stack_testcase(64 - 4).expect("generate stack_testcase 64"),
-                481,
+                // TODO: pick some high enough number that this test overflows still
+                // cranelift changes made 480 the wrong magic number
+                591,
                 true,
             );
         }

lucet-spectest/src/script.rs

@@ -74,6 +74,7 @@ impl ScriptEnv {
             true,
             &None,
             false,
+            false,
         )
         .map_err(program_error)?;
 

lucet-wasi-sdk/tests/lucetc.rs

@@ -55,6 +55,7 @@ mod lucetc_tests {
             false,
             &Some(v),
             false,
+            false,
         )
         .expect("compile empty");
         let mdata = c.module_data().unwrap();
@@ -101,6 +102,7 @@ mod lucetc_tests {
             false,
             &Some(v),
             false,
+            false,
         )
         .expect("compile c");
         let mdata = c.module_data().unwrap();
@@ -133,6 +135,7 @@ mod lucetc_tests {
             false,
             &Some(v),
             false,
+            false,
         )
         .expect("compile d");
         let mdata = c.module_data().unwrap();
@@ -161,6 +164,7 @@ mod lucetc_tests {
             false,
             &Some(v),
             false,
+            false,
         )
         .expect("compile c & d");
         let mdata = c.module_data().unwrap();
@@ -213,6 +217,7 @@ mod lucetc_tests {
             false,
             &Some(v),
             false,
+            false,
         )
         .expect("compile empty");
         let mdata = c.module_data().unwrap();

lucetc/lucetc/main.rs

@@ -141,6 +141,10 @@ pub fn run(opts: &Options) -> Result<(), Error> {
         c.count_instructions(true);
     }
 
+    if opts.pinned_heap {
+        c.pinned_heap(true);
+    }
+
     match opts.codegen {
         CodegenOutput::Obj => c.object_file(&opts.output)?,
         CodegenOutput::SharedObj => c.shared_object_file(&opts.output)?,

lucetc/lucetc/options.rs

@@ -120,6 +120,7 @@ pub struct Options {
     pub pk_path: Option<PathBuf>,
     pub sk_path: Option<PathBuf>,
     pub count_instructions: bool,
+    pub pinned_heap: bool,
     pub error_style: ErrorStyle,
     pub target: Triple,
 }
@@ -212,6 +213,7 @@ impl Options {
         let sk_path = m.value_of("sk_path").map(PathBuf::from);
         let pk_path = m.value_of("pk_path").map(PathBuf::from);
         let count_instructions = m.is_present("count_instructions");
+        let pinned_heap = m.is_present("pinned_heap");
 
         let error_style = match m.value_of("error_style") {
             None => ErrorStyle::default(),
@@ -240,6 +242,7 @@ impl Options {
             sk_path,
             pk_path,
             count_instructions,
+            pinned_heap,
             error_style,
             target,
         })
@@ -453,6 +456,12 @@ SSE3 but not AVX:
                     .takes_value(false)
                     .help("Instrument the produced binary to count the number of wasm operations the translated program executes")
             )
+            .arg(
+                Arg::with_name("pinned_heap")
+                    .long("--pinned-heap-reg")
+                    .takes_value(false)
+                    .help("This feature is not stable - it may be removed in the future! Pin a register to use as this module's heap base. Typically improves performance.")
+            )
             .arg(
                 Arg::with_name("error_style")
                     .long("error-style")