Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#4547: Refactor vmm builder code to simplify logic that creates the microVM to boot #4603

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

#4547: Refactor vmm builder code to simplify logic that creates the microVM to boot #4603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
68980ac
broken code
andr3wy May 7, 2024
71198b3
Refactor in progress. Needs further work to ideate on refactor path i…
andr3wy May 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
298 changes: 198 additions & 100 deletions src/vmm/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

//! Enables pre-boot setup, instantiation and booting of a Firecracker VMM.


#[cfg(target_arch = "x86_64")]
use std::convert::TryFrom;
use std::fmt::Debug;
Expand Down Expand Up @@ -73,6 +74,7 @@ use crate::vstate::vcpu::{Vcpu, VcpuConfig, VcpuError};
use crate::vstate::vm::Vm;
use crate::{device_manager, EventManager, Vmm, VmmError};


/// Errors associated with starting the instance.
#[derive(Debug, thiserror::Error, displaydoc::Display)]
pub enum StartMicrovmError {
Expand Down Expand Up @@ -144,100 +146,186 @@ impl std::convert::From<linux_loader::cmdline::Error> for StartMicrovmError {
}
}

#[cfg_attr(target_arch = "aarch64", allow(unused))]
fn create_vmm_and_vcpus(
instance_info: &InstanceInfo,
event_manager: &mut EventManager,
guest_memory: GuestMemoryMmap,
uffd: Option<Uffd>,
track_dirty_pages: bool,
vcpu_count: u8,
kvm_capabilities: Vec<KvmCapability>,
) -> Result<(Vmm, Vec<Vcpu>), StartMicrovmError> {
use self::StartMicrovmError::*;

// Set up Kvm Vm and register memory regions.
// Build custom CPU config if a custom template is provided.
let mut vm = Vm::new(kvm_capabilities)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)?;
vm.memory_init(&guest_memory, track_dirty_pages)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)?;

let vcpus_exit_evt = EventFd::new(libc::EFD_NONBLOCK)
.map_err(VmmError::EventFd)
.map_err(Internal)?;

let resource_allocator = ResourceAllocator::new()?;
// this module is for code specific to the aarch64 architecture
#[cfg(target_arch = "aarch64")]
mod Aarch64 {
use super::*;
use utils::eventfd::EventFd;
use crate::vstate::vm::Vm;
pub fn setup_vmm_and_vcpus(
instance_info: &InstanceInfo,
event_manager: &mut EventManager,
guest_memory: GuestMemoryMmap,
uffd: Option<Uffd>,
track_dirty_pages: bool,
vcpu_count: u8,
kvm_capabilities: Vec<KvmCapability>,
) -> Result<(Vmm, Vec<Vcpu>), StartMicrovmError> {
use self::StartMicrovmError::*;

// Set up Kvm Vm and register memory regions.
// Build custom CPU config if a custom template is provided.
let mut vm = Vm::new(kvm_capabilities)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)?;
vm.memory_init(&guest_memory, track_dirty_pages)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)?;

// Instantiate the MMIO device manager.
let mmio_device_manager = MMIODeviceManager::new();
let vcpus_exit_evt = EventFd::new(libc::EFD_NONBLOCK)
.map_err(VmmError::EventFd)
.map_err(Internal)?;

// Instantiate ACPI device manager.
#[cfg(target_arch = "x86_64")]
let acpi_device_manager = ACPIDeviceManager::new();
let resource_allocator = ResourceAllocator::new()?;

// For x86_64 we need to create the interrupt controller before calling `KVM_CREATE_VCPUS`
// while on aarch64 we need to do it the other way around.
#[cfg(target_arch = "x86_64")]
let (vcpus, pio_device_manager) = {
setup_interrupt_controller(&mut vm)?;
let vcpus = create_vcpus(&vm, vcpu_count, &vcpus_exit_evt).map_err(Internal)?;
// Instantiate the MMIO device manager.
let mmio_device_manager = MMIODeviceManager::new();

// Make stdout non blocking.
set_stdout_nonblocking();
// On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) before setting up the
// IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP
// was already initialized.
// Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c.
let vcpus = {
let vcpus = create_vcpus(&vm, vcpu_count, &vcpus_exit_evt).map_err(Internal)?;
#[cfg(target_arch = "aarch64")]
setup_interrupt_controller(&mut vm, vcpu_count)?;
vcpus
};

#[cfg(target_arch = "aarch64")]
let vmm = Vmm {
events_observer: Some(std::io::stdin()),
instance_info: instance_info.clone(),
shutdown_exit_code: None,
vm,
guest_memory,
uffd,
vcpus_handles: Vec::new(),
vcpus_exit_evt,
resource_allocator,
mmio_device_manager,
};

Ok((vmm, vcpus))
}
/// Sets up the irqchip for a aarch64 microVM.
#[cfg(target_arch = "aarch64")] // This is needed since setup_irqchip() itself is conditionally compiled
pub fn setup_interrupt_controller(vm: &mut Vm, vcpu_count: u8) -> Result<(), StartMicrovmError> {
vm.setup_irqchip(vcpu_count)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)
}
}

// Serial device setup.
let serial_device =
setup_serial_device(event_manager, std::io::stdin(), io::stdout()).map_err(Internal)?;
// this module is for functionality specific to the x86_64 platform
mod X86_64 {
use super::*;
pub fn setup_vmm_and_vcpus(
instance_info: &InstanceInfo,
event_manager: &mut EventManager,
guest_memory: GuestMemoryMmap,
uffd: Option<Uffd>,
track_dirty_pages: bool,
vcpu_count: u8,
kvm_capabilities: Vec<KvmCapability>,
) -> Result<(Vmm, Vec<Vcpu>), StartMicrovmError> {
use self::StartMicrovmError::*;

// Set up Kvm Vm and register memory regions.
// Build custom CPU config if a custom template is provided.
let mut vm = Vm::new(kvm_capabilities)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)?;
vm.memory_init(&guest_memory, track_dirty_pages)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)?;

// x86_64 uses the i8042 reset event as the Vmm exit event.
let reset_evt = vcpus_exit_evt
.try_clone()
let vcpus_exit_evt = EventFd::new(libc::EFD_NONBLOCK)
.map_err(VmmError::EventFd)
.map_err(Internal)?;

// create pio dev manager with legacy devices
let pio_device_manager = {
// TODO Remove these unwraps.
let mut pio_dev_mgr = PortIODeviceManager::new(serial_device, reset_evt).unwrap();
pio_dev_mgr.register_devices(vm.fd()).unwrap();
pio_dev_mgr
let resource_allocator = ResourceAllocator::new()?;

// Instantiate the MMIO device manager.
let mmio_device_manager = MMIODeviceManager::new();

// Instantiate ACPI device manager.
let acpi_device_manager = ACPIDeviceManager::new();

// For x86_64 we need to create the interrupt controller before calling `KVM_CREATE_VCPUS`
// while on aarch64 we need to do it the other way around.
let (vcpus, pio_device_manager) = {
setup_interrupt_controller(&mut vm)?;
let vcpus = create_vcpus(&vm, vcpu_count, &vcpus_exit_evt).map_err(Internal)?;

// Make stdout non blocking.
set_stdout_nonblocking();

// Serial device setup.
let serial_device =
setup_serial_device(event_manager, std::io::stdin(), io::stdout()).map_err(Internal)?;

// x86_64 uses the i8042 reset event as the Vmm exit event.
let reset_evt = vcpus_exit_evt
.try_clone()
.map_err(VmmError::EventFd)
.map_err(Internal)?;

// create pio dev manager with legacy devices
let pio_device_manager = {
// TODO Remove these unwraps.
let mut pio_dev_mgr = PortIODeviceManager::new(serial_device, reset_evt).unwrap();
pio_dev_mgr.register_devices(vm.fd()).unwrap();
pio_dev_mgr
};

(vcpus, pio_device_manager)
};

(vcpus, pio_device_manager)
};
let vmm = Vmm {
events_observer: Some(std::io::stdin()),
instance_info: instance_info.clone(),
shutdown_exit_code: None,
vm,
guest_memory,
uffd,
vcpus_handles: Vec::new(),
vcpus_exit_evt,
resource_allocator,
mmio_device_manager,
pio_device_manager,
acpi_device_manager,
};

Ok((vmm, vcpus))
}

/// Sets up the irqchip for a x86_64 microVM.
pub fn setup_interrupt_controller(vm: &mut Vm) -> Result<(), StartMicrovmError> {
vm.setup_irqchip()
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)
}
}

// On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) before setting up the
// IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP
// was already initialized.
// Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c.
#[cfg_attr(target_arch = "aarch64", allow(unused))]
fn create_vmm_and_vcpus(
instance_info: &InstanceInfo,
event_manager: &mut EventManager,
guest_memory: GuestMemoryMmap,
uffd: Option<Uffd>,
track_dirty_pages: bool,
vcpu_count: u8,
kvm_capabilities: Vec<KvmCapability>,
) -> Result<(Vmm, Vec<Vcpu>), StartMicrovmError> {
#[cfg(target_arch = "aarch64")]
let vcpus = {
let vcpus = create_vcpus(&vm, vcpu_count, &vcpus_exit_evt).map_err(Internal)?;
setup_interrupt_controller(&mut vm, vcpu_count)?;
vcpus
};

let vmm = Vmm {
events_observer: Some(std::io::stdin()),
instance_info: instance_info.clone(),
shutdown_exit_code: None,
vm,
guest_memory,
uffd,
vcpus_handles: Vec::new(),
vcpus_exit_evt,
resource_allocator,
mmio_device_manager,
#[cfg(target_arch = "x86_64")]
pio_device_manager,
#[cfg(target_arch = "x86_64")]
acpi_device_manager,
};
return Aarch64::setup_vmm_and_vcpus(instance_info, event_manager, guest_memory, uffd,
track_dirty_pages, vcpu_count, kvm_capabilities);

Ok((vmm, vcpus))
#[cfg(target_arch = "x86_64")]
return X86_64::setup_vmm_and_vcpus(instance_info, event_manager, guest_memory, uffd,
track_dirty_pages, vcpu_count, kvm_capabilities);
}

/// Builds and starts a microVM based on the current Firecracker VmResources configuration.
Expand Down Expand Up @@ -459,6 +547,8 @@ pub enum BuildMicrovmFromSnapshotError {
///
/// An `Arc` reference of the built `Vmm` is also plugged in the `EventManager`, while another
/// is returned.

//SEVEN ARGUMENTS
#[allow(clippy::too_many_arguments)]
pub fn build_microvm_from_snapshot(
instance_info: &InstanceInfo,
Expand Down Expand Up @@ -675,21 +765,21 @@ where
})
}

/// Sets up the irqchip for a x86_64 microVM.
#[cfg(target_arch = "x86_64")]
pub fn setup_interrupt_controller(vm: &mut Vm) -> Result<(), StartMicrovmError> {
vm.setup_irqchip()
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)
}

/// Sets up the irqchip for a aarch64 microVM.
#[cfg(target_arch = "aarch64")]
pub fn setup_interrupt_controller(vm: &mut Vm, vcpu_count: u8) -> Result<(), StartMicrovmError> {
vm.setup_irqchip(vcpu_count)
.map_err(VmmError::Vm)
.map_err(StartMicrovmError::Internal)
}
// /// Sets up the irqchip for a x86_64 microVM.
// #[cfg(target_arch = "x86_64")]
// pub fn setup_interrupt_controller(vm: &mut Vm) -> Result<(), StartMicrovmError> {
// vm.setup_irqchip()
// .map_err(VmmError::Vm)
// .map_err(StartMicrovmError::Internal)
// }

// /// Sets up the irqchip for a aarch64 microVM.
// #[cfg(target_arch = "aarch64")]
// pub fn setup_interrupt_controller(vm: &mut Vm, vcpu_count: u8) -> Result<(), StartMicrovmError> {
// vm.setup_irqchip(vcpu_count)
// .map_err(VmmError::Vm)
// .map_err(StartMicrovmError::Internal)
// }

/// Sets up the serial device.
pub fn setup_serial_device(
Expand Down Expand Up @@ -759,6 +849,8 @@ fn create_vcpus(vm: &Vm, vcpu_count: u8, exit_evt: &EventFd) -> Result<Vec<Vcpu>
}

/// Configures the system for booting Linux.


#[cfg_attr(target_arch = "aarch64", allow(unused))]
pub fn configure_system_for_boot(
vmm: &mut Vmm,
Expand Down Expand Up @@ -1036,6 +1128,12 @@ pub(crate) fn set_stdout_nonblocking() {
}
}

// This wrapper is here since this function is called in other files
#[cfg(target_arch = "x86_64")]
pub fn setup_interrupt_controller(vm: &mut Vm) -> Result<(), StartMicrovmError> {
return X86_64::setup_interrupt_controller(vm);
}

#[cfg(test)]
pub mod tests {
use std::io::Write;
Expand Down Expand Up @@ -1144,15 +1242,17 @@ pub mod tests {
.unwrap();

#[cfg(target_arch = "x86_64")]
setup_interrupt_controller(&mut vm).unwrap();
X86_64::setup_interrupt_controller(&mut vm).unwrap();

#[cfg(target_arch = "aarch64")]
{
let exit_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
let _vcpu = Vcpu::new(1, &vm, exit_evt).unwrap();
setup_interrupt_controller(&mut vm, 1).unwrap();
Aarch64::setup_interrupt_controller(&mut vm, 1).unwrap();
}



Vmm {
events_observer: Some(std::io::stdin()),
instance_info: InstanceInfo::default(),
Expand All @@ -1164,10 +1264,8 @@ pub mod tests {
vcpus_exit_evt,
resource_allocator: ResourceAllocator::new().unwrap(),
mmio_device_manager,
#[cfg(target_arch = "x86_64")]
pio_device_manager,
#[cfg(target_arch = "x86_64")]
acpi_device_manager,
acpi_device_manager
}
}

Expand Down Expand Up @@ -1381,7 +1479,7 @@ pub mod tests {
let evfd = EventFd::new(libc::EFD_NONBLOCK).unwrap();

#[cfg(target_arch = "x86_64")]
setup_interrupt_controller(&mut vm).unwrap();
X86_64::setup_interrupt_controller(&mut vm).unwrap();

let vcpu_vec = create_vcpus(&vm, vcpu_count, &evfd).unwrap();
assert_eq!(vcpu_vec.len(), vcpu_count as usize);
Expand Down
Loading