diff --git a/benchmarks/lucet-benchmarks/src/context.rs b/benchmarks/lucet-benchmarks/src/context.rs index 57a9c3899..951936c15 100644 --- a/benchmarks/lucet-benchmarks/src/context.rs +++ b/benchmarks/lucet-benchmarks/src/context.rs @@ -9,7 +9,8 @@ fn context_init(c: &mut Criterion) { c.bench_function("context_init", move |b| { b.iter(|| { - ContextHandle::create_and_init(&mut *stack, f as usize, &[]).unwrap(); + ContextHandle::create_and_init(&mut *stack, f as usize, &[], std::ptr::null_mut()) + .unwrap(); }) }); } @@ -22,7 +23,13 @@ fn context_swap_return(c: &mut Criterion) { b.iter_batched( || { let mut stack = vec![0u64; 1024].into_boxed_slice(); - let child = ContextHandle::create_and_init(&mut *stack, f as usize, &[]).unwrap(); + let child = ContextHandle::create_and_init( + &mut *stack, + f as usize, + &[], + std::ptr::null_mut(), + ) + .unwrap(); (stack, child) }, |(stack, mut child)| unsafe { @@ -44,8 +51,13 @@ fn context_init_swap_return(c: &mut Criterion) { || vec![0u64; 1024].into_boxed_slice(), |mut stack| { let mut parent = ContextHandle::new(); - let mut child = - ContextHandle::create_and_init(&mut *stack, f as usize, &[]).unwrap(); + let mut child = ContextHandle::create_and_init( + &mut *stack, + f as usize, + &[], + std::ptr::null_mut(), + ) + .unwrap(); unsafe { Context::swap(&mut parent, &mut child) }; stack }, @@ -332,8 +344,13 @@ fn context_init_swap_return_many_args(c: &mut Criterion) { || vec![0u64; 1024].into_boxed_slice(), |mut stack| { let mut parent = ContextHandle::new(); - let mut child = - ContextHandle::create_and_init(&mut *stack, f as usize, &args).unwrap(); + let mut child = ContextHandle::create_and_init( + &mut *stack, + f as usize, + &args, + std::ptr::null_mut(), + ) + .unwrap(); unsafe { Context::swap(&mut parent, &mut child) }; stack }, diff --git a/lucet-module/src/module_data.rs b/lucet-module/src/module_data.rs index 2196a5985..e0746890a 100644 --- a/lucet-module/src/module_data.rs +++ b/lucet-module/src/module_data.rs @@ -59,6 +59,7 @@ pub struct ModuleFeatures { pub lzcnt: bool, pub popcnt: bool, pub instruction_count: bool, + pub pinned_heap: bool, _hidden: (), } @@ -75,6 +76,7 @@ impl ModuleFeatures { lzcnt: false, popcnt: false, instruction_count: false, + pinned_heap: false, _hidden: (), } } diff --git a/lucet-runtime/lucet-runtime-internals/src/alloc/tests.rs b/lucet-runtime/lucet-runtime-internals/src/alloc/tests.rs index a4c96a7af..2d2dc9884 100644 --- a/lucet-runtime/lucet-runtime-internals/src/alloc/tests.rs +++ b/lucet-runtime/lucet-runtime-internals/src/alloc/tests.rs @@ -655,6 +655,7 @@ macro_rules! alloc_tests { inst.alloc_mut().stack_u64_mut(), heap_touching_child as usize, &[Val::CPtr(heap_ptr)], + heap_ptr, ) .expect("context init succeeds"); Context::swap(&mut parent, &mut child); @@ -704,6 +705,7 @@ macro_rules! alloc_tests { inst.alloc_mut().stack_u64_mut(), stack_pattern_child as usize, &[Val::CPtr(heap_ptr)], + heap_ptr, ) .expect("context init succeeds"); Context::swap(&mut parent, &mut child); diff --git a/lucet-runtime/lucet-runtime-internals/src/context/context_asm.S b/lucet-runtime/lucet-runtime-internals/src/context/context_asm.S index f313ddea9..4cd4d3e7f 100644 --- a/lucet-runtime/lucet-runtime-internals/src/context/context_asm.S +++ b/lucet-runtime/lucet-runtime-internals/src/context/context_asm.S @@ -118,6 +118,7 @@ _lucet_context_swap: mov %r14, (6*8)(%rdi) mov %r15, (7*8)(%rdi) mov %rsi, (8*8)(%rdi) + mov %r8, (9*8)(%rdi) movdqu %xmm0, (10*8 + 0*16)(%rdi) movdqu %xmm1, (10*8 + 1*16)(%rdi) @@ -137,6 +138,7 @@ _lucet_context_swap: mov (5*8)(%rsi), %r13 mov (6*8)(%rsi), %r14 mov (7*8)(%rsi), %r15 + mov (9*8)(%rsi), %r8 movdqu (10*8 + 0*16)(%rsi), %xmm0 movdqu (10*8 + 1*16)(%rsi), %xmm1 diff --git a/lucet-runtime/lucet-runtime-internals/src/context/mod.rs b/lucet-runtime/lucet-runtime-internals/src/context/mod.rs index a6f3b13b3..7ba39ec4a 100644 --- a/lucet-runtime/lucet-runtime-internals/src/context/mod.rs +++ b/lucet-runtime/lucet-runtime-internals/src/context/mod.rs @@ -34,6 +34,7 @@ pub(crate) struct GpRegs { r14: u64, r15: u64, pub(crate) rsi: u64, + r8: u64, } impl GpRegs { @@ -48,6 +49,7 @@ impl GpRegs { r14: 0, r15: 0, rsi: 0, + r8: 0, } } } @@ -208,9 +210,10 @@ impl ContextHandle { stack: &mut [u64], fptr: usize, args: &[Val], + heap: *mut core::ffi::c_void, ) -> Result { let mut child = ContextHandle::new(); - Context::init(stack, &mut child, fptr, args)?; + Context::init(stack, &mut child, fptr, args, heap)?; Ok(child) } } @@ -303,6 +306,7 @@ impl Context { /// &mut child, /// entrypoint as usize, /// &[Val::U64(120), Val::F32(3.14)], + /// std::ptr::null_mut(), /// ); /// assert!(res.is_ok()); /// ``` @@ -326,6 +330,7 @@ impl Context { /// &mut child, /// entrypoint as usize, /// &[Val::U64(120), Val::F32(3.14)], + /// std::ptr::null_mut(), /// ); /// assert!(res.is_ok()); /// ``` @@ -367,6 +372,7 @@ impl Context { child: &mut Context, fptr: usize, args: &[Val], + heap: *mut core::ffi::c_void, ) -> Result<(), Error> { Context::init_with_callback( stack, @@ -375,6 +381,7 @@ impl Context { ptr::null_mut(), fptr, args, + heap, ) } @@ -393,6 +400,7 @@ impl Context { callback_data: *mut Instance, fptr: usize, args: &[Val], + heap: *mut core::ffi::c_void, ) -> Result<(), Error> { if !stack_is_aligned(stack) { return Err(Error::UnalignedStack); @@ -475,6 +483,10 @@ impl Context { // even at the entrypoint of the guest. child.gpr.rbp = child as *const Context as u64; + // Heap pinning: r15 is not used to pass any parameters on Windows/POSIX abis, we simply set this to be the value of the heap always. + // This value will be used only when the lucet module loaded is compiled requiring use of the pinned heap register. + child.gpr.r15 = heap as u64; + Ok(()) } @@ -547,6 +559,7 @@ impl Context { /// &mut child, /// entrypoint as usize, /// &[], + /// std::ptr::null_mut(), /// ).unwrap(); /// /// unsafe { Context::swap(&mut parent, &mut child); } diff --git a/lucet-runtime/lucet-runtime-internals/src/context/tests/c_child.rs b/lucet-runtime/lucet-runtime-internals/src/context/tests/c_child.rs index 6c881c02c..b1bc23628 100644 --- a/lucet-runtime/lucet-runtime-internals/src/context/tests/c_child.rs +++ b/lucet-runtime/lucet-runtime-internals/src/context/tests/c_child.rs @@ -55,6 +55,7 @@ macro_rules! init_and_swap { &mut *$stack, $fn as usize, &[$( $args ),*], + std::ptr::null_mut(), ).unwrap())); child_regs = child; diff --git a/lucet-runtime/lucet-runtime-internals/src/context/tests/mod.rs b/lucet-runtime/lucet-runtime-internals/src/context/tests/mod.rs index c98930b16..473acd1d7 100644 --- a/lucet-runtime/lucet-runtime-internals/src/context/tests/mod.rs +++ b/lucet-runtime/lucet-runtime-internals/src/context/tests/mod.rs @@ -31,7 +31,12 @@ fn init_rejects_unaligned() { let mut stack_unaligned = unsafe { slice::from_raw_parts_mut(ptr, len) }; // now we have the unaligned stack, let's make sure it blows up right - let res = ContextHandle::create_and_init(&mut stack_unaligned, dummy as usize, &[]); + let res = ContextHandle::create_and_init( + &mut stack_unaligned, + dummy as usize, + &[], + std::ptr::null_mut(), + ); if let Err(Error::UnalignedStack) = res { assert!(true); diff --git a/lucet-runtime/lucet-runtime-internals/src/context/tests/rust_child.rs b/lucet-runtime/lucet-runtime-internals/src/context/tests/rust_child.rs index 5c44053dc..7ffd9e902 100644 --- a/lucet-runtime/lucet-runtime-internals/src/context/tests/rust_child.rs +++ b/lucet-runtime/lucet-runtime-internals/src/context/tests/rust_child.rs @@ -51,6 +51,7 @@ macro_rules! init_and_swap { &mut *$stack, $fn as usize, &[$( $args ),*], + std::ptr::null_mut(), ).unwrap(); CHILD = Some(child); diff --git a/lucet-runtime/lucet-runtime-internals/src/instance.rs b/lucet-runtime/lucet-runtime-internals/src/instance.rs index 4ac603ec5..0ba25f18f 100644 --- a/lucet-runtime/lucet-runtime-internals/src/instance.rs +++ b/lucet-runtime/lucet-runtime-internals/src/instance.rs @@ -972,7 +972,8 @@ impl Instance { self.entrypoint = Some(func); - let mut args_with_vmctx = vec![Val::from(self.alloc.slot().heap)]; + let heap = self.alloc.slot().heap; + let mut args_with_vmctx = vec![Val::from(heap)]; args_with_vmctx.extend_from_slice(args); let self_ptr = self as *mut _; @@ -983,6 +984,7 @@ impl Instance { self_ptr, func.ptr.as_usize(), &args_with_vmctx, + heap, )?; self.install_activator(); diff --git a/lucet-runtime/lucet-runtime-tests/src/stack.rs b/lucet-runtime/lucet-runtime-tests/src/stack.rs index fc3ec8e05..b8c85c3ea 100644 --- a/lucet-runtime/lucet-runtime-tests/src/stack.rs +++ b/lucet-runtime/lucet-runtime-tests/src/stack.rs @@ -162,7 +162,9 @@ macro_rules! stack_tests { expect_stack_overflow( // Same note as `expect_ok_locals64_481` stack_testcase(64 - 4).expect("generate stack_testcase 64"), - 481, + // TODO: pick some high enough number that this test overflows still + // cranelift changes made 480 the wrong magic number + 591, true, ); } diff --git a/lucetc/lucetc/main.rs b/lucetc/lucetc/main.rs index 791a940fc..3c553135e 100644 --- a/lucetc/lucetc/main.rs +++ b/lucetc/lucetc/main.rs @@ -100,7 +100,8 @@ pub fn run(opts: &Options) -> Result<(), Error> { .with_bindings(bindings) .with_opt_level(opts.opt_level) .with_cpu_features(opts.cpu_features.clone()) - .with_target(opts.target.clone()); + .with_target(opts.target.clone()) + .with_pinned_heap(opts.pinned_heap); if let Some(validator) = validator.take() { c.validator(validator); diff --git a/lucetc/lucetc/options.rs b/lucetc/lucetc/options.rs index c40b968d1..2815efb86 100644 --- a/lucetc/lucetc/options.rs +++ b/lucetc/lucetc/options.rs @@ -120,6 +120,7 @@ pub struct Options { pub pk_path: Option, pub sk_path: Option, pub count_instructions: bool, + pub pinned_heap: bool, pub error_style: ErrorStyle, pub target: Triple, } @@ -211,6 +212,7 @@ impl Options { let sk_path = m.value_of("sk_path").map(PathBuf::from); let pk_path = m.value_of("pk_path").map(PathBuf::from); let count_instructions = m.is_present("count_instructions"); + let pinned_heap = m.is_present("pinned_heap"); let error_style = match m.value_of("error_style") { None => ErrorStyle::default(), @@ -239,6 +241,7 @@ impl Options { sk_path, pk_path, count_instructions, + pinned_heap, error_style, target, }) @@ -452,6 +455,12 @@ SSE3 but not AVX: .takes_value(false) .help("Instrument the produced binary to count the number of wasm operations the translated program executes") ) + .arg( + Arg::with_name("pinned_heap") + .long("--pinned-heap-reg") + .takes_value(false) + .help("This feature is not stable - it may be removed in the future! Pin a register to use as this module's heap base. Typically improves performance.") + ) .arg( Arg::with_name("error_style") .long("error-style") diff --git a/lucetc/src/compiler.rs b/lucetc/src/compiler.rs index 171803596..15f0a36d1 100644 --- a/lucetc/src/compiler.rs +++ b/lucetc/src/compiler.rs @@ -63,6 +63,7 @@ pub struct CompilerBuilder { heap_settings: HeapSettings, count_instructions: bool, canonicalize_nans: bool, + pinned_heap: bool, validator: Option, } @@ -75,6 +76,7 @@ impl CompilerBuilder { heap_settings: HeapSettings::default(), count_instructions: false, canonicalize_nans: false, + pinned_heap: false, validator: None, } } @@ -145,6 +147,15 @@ impl CompilerBuilder { self } + pub fn pinned_heap(&mut self, pinned_heap: bool) { + self.pinned_heap = pinned_heap; + } + + pub fn with_pinned_heap(mut self, pinned_heap: bool) -> Self { + self.pinned_heap(pinned_heap); + self + } + pub fn validator(&mut self, validator: Option) { self.validator = validator; } @@ -169,6 +180,7 @@ impl CompilerBuilder { self.count_instructions, &self.validator, self.canonicalize_nans, + self.pinned_heap, ) } } @@ -182,6 +194,7 @@ pub struct Compiler<'a> { count_instructions: bool, module_translation_state: ModuleTranslationState, canonicalize_nans: bool, + pinned_heap: bool, } impl<'a> Compiler<'a> { @@ -195,8 +208,15 @@ impl<'a> Compiler<'a> { count_instructions: bool, validator: &Option, canonicalize_nans: bool, + pinned_heap: bool, ) -> Result { - let isa = Self::target_isa(target.clone(), opt_level, &cpu_features, canonicalize_nans)?; + let isa = Self::target_isa( + target.clone(), + opt_level, + &cpu_features, + canonicalize_nans, + pinned_heap, + )?; let frontend_config = isa.frontend_config(); let mut module_info = ModuleInfo::new(frontend_config.clone()); @@ -250,6 +270,7 @@ impl<'a> Compiler<'a> { module_translation_state, target, canonicalize_nans, + pinned_heap, }) } @@ -260,6 +281,7 @@ impl<'a> Compiler<'a> { pub fn module_features(&self) -> ModuleFeatures { let mut mf: ModuleFeatures = (&self.cpu_features).into(); mf.instruction_count = self.count_instructions; + mf.pinned_heap = self.pinned_heap; mf } @@ -475,6 +497,7 @@ impl<'a> Compiler<'a> { self.opt_level, &self.cpu_features, self.canonicalize_nans, + self.pinned_heap, )?, )) } @@ -484,6 +507,7 @@ impl<'a> Compiler<'a> { opt_level: OptLevel, cpu_features: &CpuFeatures, canonicalize_nans: bool, + pinned_heap: bool, ) -> Result, Error> { let mut flags_builder = settings::builder(); let isa_builder = cpu_features.isa_builder(target)?; @@ -493,6 +517,10 @@ impl<'a> Compiler<'a> { if canonicalize_nans { flags_builder.enable("enable_nan_canonicalization").unwrap(); } + if pinned_heap { + flags_builder.enable("enable_pinned_reg").unwrap(); + flags_builder.enable("use_pinned_reg_as_heap_base").unwrap(); + } Ok(isa_builder.finish(settings::Flags::new(flags_builder))) } } diff --git a/lucetc/src/lib.rs b/lucetc/src/lib.rs index b9174875d..dd93cda7d 100755 --- a/lucetc/src/lib.rs +++ b/lucetc/src/lib.rs @@ -104,6 +104,8 @@ pub trait LucetcOpts { fn with_count_instructions(self, enable_count: bool) -> Self; fn canonicalize_nans(&mut self, enable_canonicalize_nans: bool); fn with_canonicalize_nans(self, enable_canonicalize_nans: bool) -> Self; + fn pinned_heap(&mut self, enable_pinned_heap: bool); + fn with_pinned_heap(self, enable_pinned_heap: bool) -> Self; } impl LucetcOpts for T { @@ -258,6 +260,15 @@ impl LucetcOpts for T { self.canonicalize_nans(enable_nans_canonicalization); self } + + fn pinned_heap(&mut self, enable_pinned_heap: bool) { + self.as_lucetc().builder.pinned_heap(enable_pinned_heap); + } + + fn with_pinned_heap(mut self, enable_pinned_heap: bool) -> Self { + self.pinned_heap(enable_pinned_heap); + self + } } impl Lucetc { @@ -315,7 +326,6 @@ impl Lucetc { let (module_contents, bindings) = self.build()?; let compiler = self.builder.create(&module_contents, &bindings)?; - compiler.cranelift_funcs()?.write(&output)?; Ok(()) diff --git a/lucetc/tests/wasm.rs b/lucetc/tests/wasm.rs index 4971cc9c1..cf159cd50 100644 --- a/lucetc/tests/wasm.rs +++ b/lucetc/tests/wasm.rs @@ -58,6 +58,7 @@ mod module_data { false, &None, false, + false, ) .expect("compiling exported_import"); let mdata = c.module_data().unwrap(); @@ -288,7 +289,6 @@ mod module_data { let h = HeapSettings::default(); let builder = Compiler::builder().with_heap_settings(h.clone()); let c = builder.create(&m, &b).expect("compiling heap_spec_import"); - assert_eq!( c.module_data().unwrap().heap_spec(), Some(&HeapSpec { @@ -313,7 +313,6 @@ mod module_data { let c = builder .create(&m, &b) .expect("compiling heap_spec_definition"); - assert_eq!( c.module_data().unwrap().heap_spec(), Some(&HeapSpec { @@ -573,6 +572,7 @@ mod validate { false, &Some(v), false, + false, ) .expect("compile"); let _obj = c.object_file().expect("codegen");