diff --git a/src/dragonball/src/device_manager/legacy.rs b/src/dragonball/src/device_manager/legacy.rs index 9dbb300d4..59f4206ab 100644 --- a/src/dragonball/src/device_manager/legacy.rs +++ b/src/dragonball/src/device_manager/legacy.rs @@ -13,6 +13,8 @@ use std::sync::{Arc, Mutex}; use dbs_device::device_manager::Error as IoManagerError; use dbs_legacy_devices::SerialDevice; +#[cfg(target_arch = "aarch64")] +use dbs_legacy_devices::RTCDevice; use vmm_sys_util::eventfd::EventFd; // The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself. @@ -42,6 +44,10 @@ pub enum Error { pub struct LegacyDeviceManager { #[cfg(target_arch = "x86_64")] i8042_reset_eventfd: EventFd, + #[cfg(target_arch = "aarch64")] + pub(crate) _rtc_device: Arc>, + #[cfg(target_arch = "aarch64")] + _rtc_eventfd: EventFd, pub(crate) com1_device: Arc>, _com1_eventfd: EventFd, pub(crate) com2_device: Arc>, @@ -140,6 +146,91 @@ pub(crate) mod x86_64 { } } +#[cfg(target_arch = "aarch64")] +pub(crate) mod aarch64 { + use super::*; + use dbs_device::device_manager::{IoManager}; + use dbs_device::resources::DeviceResources; + use std::collections::HashMap; + use kvm_ioctls::VmFd; + + type Result = ::std::result::Result; + + /// LegacyDeviceType: com1 + pub const COM1: &str = "com1"; + /// LegacyDeviceType: com2 + pub const COM2: &str = "com2"; + /// LegacyDeviceType: rtc + pub const RTC: &str = "rtc"; + + impl LegacyDeviceManager { + /// Create a LegacyDeviceManager instance handling legacy devices. + pub fn create_manager( + bus: &mut IoManager, + vm_fd: Option>, + resources: &HashMap, + ) -> Result { + let (com1_device, com1_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), resources.get(COM1).unwrap())?; + let (com2_device, com2_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), resources.get(COM2).unwrap())?; + let (rtc_device, rtc_eventfd) = + Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get(RTC).unwrap())?; + + Ok(LegacyDeviceManager { + _rtc_device: rtc_device, + _rtc_eventfd: rtc_eventfd, + com1_device, + _com1_eventfd: com1_eventfd, + com2_device, + _com2_eventfd: com2_eventfd, + }) + } + + fn create_com_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + resources: &DeviceResources, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(SerialDevice::new( + eventfd.try_clone().map_err(Error::EventFd)? + ))); + + bus.register_device_io(device.clone(), resources.get_all_resources()) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + let irq = resources.get_legacy_irq().unwrap(); + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + + fn create_rtc_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + resources: &DeviceResources, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(RTCDevice::new())); + + bus.register_device_io(device.clone(), resources.get_all_resources()) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + let irq = resources.get_legacy_irq().unwrap(); + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + } +} + #[cfg(test)] mod tests { #[cfg(target_arch = "x86_64")] diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index b8e1ea996..d9d03947d 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -5,6 +5,7 @@ use std::io; use std::sync::{Arc, Mutex, MutexGuard}; +use std::collections::HashMap; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; @@ -12,6 +13,8 @@ use dbs_address_space::AddressSpace; use dbs_arch::{DeviceType, MMIODeviceInfo}; use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext}; use dbs_device::resources::Resource; +#[cfg(target_arch = "aarch64")] +use dbs_device::resources::DeviceResources; use dbs_device::DeviceIo; use dbs_interrupt::KvmIrqManager; use dbs_legacy_devices::ConsoleHandler; @@ -51,7 +54,7 @@ pub mod console_manager; pub use self::console_manager::ConsoleManager; mod legacy; -pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; +pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager, aarch64::{COM1, COM2, RTC}}; #[cfg(feature = "virtio-vsock")] /// Device manager for user-space vsock devices. @@ -321,6 +324,33 @@ impl DeviceOpContext { Ok(()) } + + #[cfg(target_arch = "aarch64")] + fn generate_virtio_device_info(&self) -> Result> { + let mut dev_info = HashMap::new(); + #[cfg(feature = "dbs-virtio-devices")] + for (_index, device) in self.virtio_devices.iter().enumerate() { + let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_mmio_device_info(device)?; + let dev_type; + let device_id; + if let Some(mmiov2_device) = + device.as_any().downcast_ref::() + { + dev_type = mmiov2_device.get_device_type(); + device_id = None; + } else { + return Err(DeviceMgrError::InvalidOperation); + } + dev_info.insert( + ( + DeviceType::Virtio(dev_type), + format!("virtio-{}@0x{:08x?}", dev_type, mmio_base), + ), + MMIODeviceInfo::new(mmio_base, mmio_size, vec![irq], device_id), + ); + } + Ok(dev_info) + } } #[cfg(all(feature = "hotplug", not(feature = "dbs-upcall")))] @@ -504,11 +534,31 @@ impl DeviceManager { &mut self, ctx: &mut DeviceOpContext, ) -> std::result::Result<(), StartMicroVmError> { - #[cfg(target_arch = "x86_64")] + #[cfg( + any(target_arch = "x86_64", + all(target_arch = "aarch64", feature = "dbs-virtio-devices") + ) + )] { let mut tx = ctx.io_context.begin_tx(); - let legacy_manager = - LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + let legacy_manager; + + #[cfg(target_arch = "x86_64")] + { + let legacy_manager = + LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + } + + #[cfg(target_arch = "aarch64")] + #[cfg(feature = "dbs-virtio-devices")] + { + let resources = self.get_legacy_resources()?; + legacy_manager = LegacyDeviceManager::create_manager( + &mut tx.io_manager, + Some(self.vm_fd.clone()), + &resources, + ); + } match legacy_manager { Ok(v) => { @@ -626,6 +676,14 @@ impl DeviceManager { ctx.generate_kernel_boot_args(kernel_config) .map_err(StartMicroVmError::DeviceManager)?; + #[cfg(target_arch = "aarch64")] + { + let dev_info = ctx + .generate_virtio_device_info() + .map_err(StartMicroVmError::DeviceManager)?; + self.mmio_device_info.extend(dev_info); + } + Ok(()) } @@ -679,6 +737,97 @@ impl DeviceManager { pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { Some(&self.mmio_device_info) } + + #[cfg(feature = "dbs-virtio-devices")] + fn get_legacy_resources( + &mut self, + ) -> std::result::Result, StartMicroVmError> { + let mut resources = HashMap::new(); + let legacy_devices = vec![ + (DeviceType::Serial, String::from(COM1)), + (DeviceType::Serial, String::from(COM2)), + (DeviceType::RTC, String::from(RTC)), + ]; + + for (device_type, device_id) in legacy_devices { + let res = self.allocate_mmio_device_resource()?; + self.add_mmio_device_info(&res, device_type, device_id.clone(), None); + resources.insert(device_id.clone(), res); + } + + Ok(resources) + } + + fn mmio_device_info_to_resources( + &self, + key: &(DeviceType, String), + ) -> std::result::Result { + self.mmio_device_info + .get(key) + .map(|info| { + let mut resources = DeviceResources::new(); + resources.append(Resource::LegacyIrq(info.irqs[0])); + resources.append(Resource::MmioAddressRange { + base: info.base, + size: info.size, + }); + resources + }) + .ok_or(StartMicroVmError::DeviceManager( + DeviceMgrError::GetDeviceResource, + )) + } + + #[cfg(feature = "dbs-virtio-devices")] + fn allocate_mmio_device_resource( + &self, + ) -> std::result::Result { + let mut requests = Vec::new(); + requests.push(ResourceConstraint::MmioAddress { + range: None, + align: MMIO_DEFAULT_CFG_SIZE, + size: MMIO_DEFAULT_CFG_SIZE, + }); + requests.push(ResourceConstraint::LegacyIrq { irq: None }); + + self.res_manager + .allocate_device_resources(&requests, false) + .map_err(StartMicroVmError::AllocateResource) + } + + fn add_mmio_device_info( + &mut self, + resource: &DeviceResources, + device_type: DeviceType, + device_id: String, + msi_device_id: Option, + ) { + let (base, size) = resource.get_mmio_address_ranges()[0]; + let irq = resource.get_legacy_irq().unwrap(); + self.mmio_device_info.insert( + (device_type, device_id), + MMIODeviceInfo::new(base, size, vec![irq], msi_device_id), + ); + } + + #[cfg(feature = "dbs-virtio-devices")] + fn get_virtio_mmio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { + let resources = device.get_assigned_resources(); + let irq = resources + .get_legacy_irq() + .ok_or(DeviceMgrError::GetDeviceResource)?; + + if let Some(mmio_dev) = device + .as_any() + .downcast_ref::() + { + if let Resource::MmioAddressRange { base, size } = mmio_dev.get_mmio_cfg_res() { + return Ok((base, size, irq)); + } + } + + Err(DeviceMgrError::GetDeviceResource) + } } #[cfg(feature = "dbs-virtio-devices")] diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index b4ee40119..454ad45b5 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,7 +12,7 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; -use crate::{address_space_manager, device_manager, vcpu, vm}; +use crate::{address_space_manager, device_manager, vcpu, vm, resource_manager}; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; @@ -73,6 +73,10 @@ pub enum Error { /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] pub enum StartMicroVmError { + /// Failed to allocate resources. + #[error("cannot allocate resources")] + AllocateResource(#[source] resource_manager::ResourceError), + /// Cannot read from an Event file descriptor. #[error("failure while reading from EventFd file descriptor")] EventFd, diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 25450f380..ffecaf356 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -21,7 +21,7 @@ use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; -use crate::error::{Error, StartMicrovmError}; +use crate::error::{Error, StartMicroVmError}; use crate::event_manager::EventManager; /// Configures the system and should be called once per vm before starting vcpu threads. @@ -63,12 +63,12 @@ impl Vm { } /// Creates the irq chip in-kernel device model. - pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> { let vcpu_count = self.vm_config.vcpu_count; self.irqchip_handle = Some( dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupGIC(e)))?, ); Ok(()) @@ -88,16 +88,16 @@ impl Vm { epoll_mgr: EpollManager, vm_as: GuestAddressSpaceImpl, request_ts: TimestampUs, - ) -> Result<(), StartMicrovmError> { + ) -> Result<(), StartMicroVmError> { let reset_eventfd = - EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?; + EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicroVmError::EventFd)?; self.reset_eventfd = Some( reset_eventfd .try_clone() - .map_err(|_| StartMicrovmError::EventFd)?, + .map_err(|_| StartMicroVmError::EventFd)?, ); self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .set_reset_event_fd(reset_eventfd); // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before @@ -106,9 +106,9 @@ impl Vm { // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; self.setup_interrupt_controller()?; self.init_devices(epoll_mgr)?; @@ -124,8 +124,8 @@ impl Vm { vm_memory: &GuestMemoryImpl, cmdline: &Cmdline, initrd: Option, - ) -> std::result::Result<(), StartMicrovmError> { - let vcpu_manager = self.vcpu_manager().map_err(StartMicrovmError::Vcpu)?; + ) -> std::result::Result<(), StartMicroVmError> { + let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; let vcpu_mpidr = vcpu_manager .vcpus() .into_iter() @@ -141,17 +141,17 @@ impl Vm { self.get_irqchip(), &initrd, ) - .map_err(StartMicrovmError::ConfigureSystem) + .map_err(StartMicroVmError::ConfigureSystem) } pub(crate) fn register_events( &mut self, event_mgr: &mut EventManager, - ) -> std::result::Result<(), StartMicrovmError> { - let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?; + ) -> std::result::Result<(), StartMicroVmError> { + let reset_evt = self.get_reset_eventfd().ok_or(StartMicroVmError::EventFd)?; event_mgr .register_exit_eventfd(reset_evt) - .map_err(|_| StartMicrovmError::RegisterEvent)?; + .map_err(|_| StartMicroVmError::RegisterEvent)?; Ok(()) } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index e372af51e..e8c6d6446 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -63,7 +63,7 @@ pub enum VmError { /// Cannot setup GIC #[cfg(target_arch = "aarch64")] #[error("failed to configure GIC")] - SetupGIC(dbs_arch::gic::Error), + SetupGIC(GICError), } /// Configuration information for user defined NUMA nodes. @@ -187,7 +187,7 @@ pub struct Vm { // Arm specific fields. // On aarch64 we need to keep around the fd obtained by creating the VGIC device. #[cfg(target_arch = "aarch64")] - irqchip_handle: Option>, + irqchip_handle: Option>, #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_client: Option>>,