aljaz/kata-containers
1
0
Fork 0
mirror of https://github.com/aljazceru/kata-containers.git synced 2026-01-24 00:34:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4544 from openanolis/anolis/virtio_device_aarch64

Browse Source 
runtime-rs: Dragonball-sandbox - add virtio device feature support for aarch64
This commit is contained in:
Bin Liu
2022-07-12 12:39:31 +08:00
committed by GitHub
parent 58b0fc4794 d2584991eb
commit b424cf3c90
5 changed files with 266 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
src/dragonball/src/device_manager/legacy.rs
View File

@@ -13,6 +13,8 @@ use std::sync::{Arc, Mutex};
use dbs_device::device_manager::Error as IoManagerError;
use dbs_legacy_devices::SerialDevice;
#[cfg(target_arch = "aarch64")]
use dbs_legacy_devices::RTCDevice;
use vmm_sys_util::eventfd::EventFd;
// The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself.
@@ -42,6 +44,10 @@ pub enum Error {
pub struct LegacyDeviceManager {
#[cfg(target_arch = "x86_64")]
i8042_reset_eventfd: EventFd,
#[cfg(target_arch = "aarch64")]
pub(crate) _rtc_device: Arc<Mutex<RTCDevice>>,
#[cfg(target_arch = "aarch64")]
_rtc_eventfd: EventFd,
pub(crate) com1_device: Arc<Mutex<SerialDevice>>,
_com1_eventfd: EventFd,
pub(crate) com2_device: Arc<Mutex<SerialDevice>>,
@@ -140,6 +146,91 @@ pub(crate) mod x86_64 {
}
}
#[cfg(target_arch = "aarch64")]
pub(crate) mod aarch64 {
use super::*;
use dbs_device::device_manager::{IoManager};
use dbs_device::resources::DeviceResources;
use std::collections::HashMap;
use kvm_ioctls::VmFd;
type Result<T> = ::std::result::Result<T, Error>;
/// LegacyDeviceType: com1
pub const COM1: &str = "com1";
/// LegacyDeviceType: com2
pub const COM2: &str = "com2";
/// LegacyDeviceType: rtc
pub const RTC: &str = "rtc";
impl LegacyDeviceManager {
/// Create a LegacyDeviceManager instance handling legacy devices.
pub fn create_manager(
bus: &mut IoManager,
vm_fd: Option<Arc<VmFd>>,
resources: &HashMap<String, DeviceResources>,
) -> Result<Self> {
let (com1_device, com1_eventfd) =
Self::create_com_device(bus, vm_fd.as_ref(), resources.get(COM1).unwrap())?;
let (com2_device, com2_eventfd) =
Self::create_com_device(bus, vm_fd.as_ref(), resources.get(COM2).unwrap())?;
let (rtc_device, rtc_eventfd) =
Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get(RTC).unwrap())?;
Ok(LegacyDeviceManager {
_rtc_device: rtc_device,
_rtc_eventfd: rtc_eventfd,
com1_device,
_com1_eventfd: com1_eventfd,
com2_device,
_com2_eventfd: com2_eventfd,
})
}
fn create_com_device(
bus: &mut IoManager,
vm_fd: Option<&Arc<VmFd>>,
resources: &DeviceResources,
) -> Result<(Arc<Mutex<SerialDevice>>, EventFd)> {
let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?;
let device = Arc::new(Mutex::new(SerialDevice::new(
eventfd.try_clone().map_err(Error::EventFd)?
)));
bus.register_device_io(device.clone(), resources.get_all_resources())
.map_err(Error::BusError)?;
if let Some(fd) = vm_fd {
let irq = resources.get_legacy_irq().unwrap();
fd.register_irqfd(&eventfd, irq)
.map_err(Error::IrqManager)?;
}
Ok((device, eventfd))
}
fn create_rtc_device(
bus: &mut IoManager,
vm_fd: Option<&Arc<VmFd>>,
resources: &DeviceResources,
) -> Result<(Arc<Mutex<RTCDevice>>, EventFd)> {
let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?;
let device = Arc::new(Mutex::new(RTCDevice::new()));
bus.register_device_io(device.clone(), resources.get_all_resources())
.map_err(Error::BusError)?;
if let Some(fd) = vm_fd {
let irq = resources.get_legacy_irq().unwrap();
fd.register_irqfd(&eventfd, irq)
.map_err(Error::IrqManager)?;
}
Ok((device, eventfd))
}
}
}
#[cfg(test)]
mod tests {
#[cfg(target_arch = "x86_64")]

157
src/dragonball/src/device_manager/mod.rs
View File

@@ -5,6 +5,7 @@
use std::io;
use std::sync::{Arc, Mutex, MutexGuard};
use std::collections::HashMap;
use arc_swap::ArcSwap;
use dbs_address_space::AddressSpace;
@@ -12,6 +13,8 @@ use dbs_address_space::AddressSpace;
use dbs_arch::{DeviceType, MMIODeviceInfo};
use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext};
use dbs_device::resources::Resource;
#[cfg(target_arch = "aarch64")]
use dbs_device::resources::DeviceResources;
use dbs_device::DeviceIo;
use dbs_interrupt::KvmIrqManager;
use dbs_legacy_devices::ConsoleHandler;
@@ -51,7 +54,7 @@ pub mod console_manager;
pub use self::console_manager::ConsoleManager;
mod legacy;
pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager};
pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager, aarch64::{COM1, COM2, RTC}};
#[cfg(feature = "virtio-vsock")]
/// Device manager for user-space vsock devices.
@@ -321,6 +324,33 @@ impl DeviceOpContext {
Ok(())
}
#[cfg(target_arch = "aarch64")]
fn generate_virtio_device_info(&self) -> Result<HashMap<(DeviceType, String), MMIODeviceInfo>> {
let mut dev_info = HashMap::new();
#[cfg(feature = "dbs-virtio-devices")]
for (_index, device) in self.virtio_devices.iter().enumerate() {
let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_mmio_device_info(device)?;
let dev_type;
let device_id;
if let Some(mmiov2_device) =
device.as_any().downcast_ref::<DbsMmioV2Device>()
{
dev_type = mmiov2_device.get_device_type();
device_id = None;
} else {
return Err(DeviceMgrError::InvalidOperation);
}
dev_info.insert(
(
DeviceType::Virtio(dev_type),
format!("virtio-{}@0x{:08x?}", dev_type, mmio_base),
),
MMIODeviceInfo::new(mmio_base, mmio_size, vec![irq], device_id),
);
}
Ok(dev_info)
}
}
#[cfg(all(feature = "hotplug", not(feature = "dbs-upcall")))]
@@ -504,11 +534,31 @@ impl DeviceManager {
&mut self,
ctx: &mut DeviceOpContext,
) -> std::result::Result<(), StartMicroVmError> {
#[cfg(target_arch = "x86_64")]
#[cfg(
any(target_arch = "x86_64",
all(target_arch = "aarch64", feature = "dbs-virtio-devices")
)
)]
{
let mut tx = ctx.io_context.begin_tx();
let legacy_manager =
LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone()));
let legacy_manager;
#[cfg(target_arch = "x86_64")]
{
let legacy_manager =
LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone()));
}
#[cfg(target_arch = "aarch64")]
#[cfg(feature = "dbs-virtio-devices")]
{
let resources = self.get_legacy_resources()?;
legacy_manager = LegacyDeviceManager::create_manager(
&mut tx.io_manager,
Some(self.vm_fd.clone()),
&resources,
);
}
match legacy_manager {
Ok(v) => {
@@ -626,6 +676,14 @@ impl DeviceManager {
ctx.generate_kernel_boot_args(kernel_config)
.map_err(StartMicroVmError::DeviceManager)?;
#[cfg(target_arch = "aarch64")]
{
let dev_info = ctx
.generate_virtio_device_info()
.map_err(StartMicroVmError::DeviceManager)?;
self.mmio_device_info.extend(dev_info);
}
Ok(())
}
@@ -679,6 +737,97 @@ impl DeviceManager {
pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> {
Some(&self.mmio_device_info)
}
#[cfg(feature = "dbs-virtio-devices")]
fn get_legacy_resources(
&mut self,
) -> std::result::Result<HashMap<String, DeviceResources>, StartMicroVmError> {
let mut resources = HashMap::new();
let legacy_devices = vec![
(DeviceType::Serial, String::from(COM1)),
(DeviceType::Serial, String::from(COM2)),
(DeviceType::RTC, String::from(RTC)),
];
for (device_type, device_id) in legacy_devices {
let res = self.allocate_mmio_device_resource()?;
self.add_mmio_device_info(&res, device_type, device_id.clone(), None);
resources.insert(device_id.clone(), res);
}
Ok(resources)
}
fn mmio_device_info_to_resources(
&self,
key: &(DeviceType, String),
) -> std::result::Result<DeviceResources, StartMicroVmError> {
self.mmio_device_info
.get(key)
.map(|info| {
let mut resources = DeviceResources::new();
resources.append(Resource::LegacyIrq(info.irqs[0]));
resources.append(Resource::MmioAddressRange {
base: info.base,
size: info.size,
});
resources
})
.ok_or(StartMicroVmError::DeviceManager(
DeviceMgrError::GetDeviceResource,
))
}
#[cfg(feature = "dbs-virtio-devices")]
fn allocate_mmio_device_resource(
&self,
) -> std::result::Result<DeviceResources, StartMicroVmError> {
let mut requests = Vec::new();
requests.push(ResourceConstraint::MmioAddress {
range: None,
align: MMIO_DEFAULT_CFG_SIZE,
size: MMIO_DEFAULT_CFG_SIZE,
});
requests.push(ResourceConstraint::LegacyIrq { irq: None });
self.res_manager
.allocate_device_resources(&requests, false)
.map_err(StartMicroVmError::AllocateResource)
}
fn add_mmio_device_info(
&mut self,
resource: &DeviceResources,
device_type: DeviceType,
device_id: String,
msi_device_id: Option<u32>,
) {
let (base, size) = resource.get_mmio_address_ranges()[0];
let irq = resource.get_legacy_irq().unwrap();
self.mmio_device_info.insert(
(device_type, device_id),
MMIODeviceInfo::new(base, size, vec![irq], msi_device_id),
);
}
#[cfg(feature = "dbs-virtio-devices")]
fn get_virtio_mmio_device_info(device: &Arc<DbsMmioV2Device>) -> Result<(u64, u64, u32)> {
let resources = device.get_assigned_resources();
let irq = resources
.get_legacy_irq()
.ok_or(DeviceMgrError::GetDeviceResource)?;
if let Some(mmio_dev) = device
.as_any()
.downcast_ref::<DbsMmioV2Device>()
{
if let Resource::MmioAddressRange { base, size } = mmio_dev.get_mmio_cfg_res() {
return Ok((base, size, irq));
}
}
Err(DeviceMgrError::GetDeviceResource)
}
}
#[cfg(feature = "dbs-virtio-devices")]

6
src/dragonball/src/error.rs
View File

@@ -12,7 +12,7 @@
#[cfg(feature = "dbs-virtio-devices")]
use dbs_virtio_devices::Error as VirtIoError;
use crate::{address_space_manager, device_manager, vcpu, vm};
use crate::{address_space_manager, device_manager, vcpu, vm, resource_manager};
/// Shorthand result type for internal VMM commands.
pub type Result<T> = std::result::Result<T, Error>;
@@ -73,6 +73,10 @@ pub enum Error {
/// Errors associated with starting the instance.
#[derive(Debug, thiserror::Error)]
pub enum StartMicroVmError {
/// Failed to allocate resources.
#[error("cannot allocate resources")]
AllocateResource(#[source] resource_manager::ResourceError),
/// Cannot read from an Event file descriptor.
#[error("failure while reading from EventFd file descriptor")]
EventFd,

30
src/dragonball/src/vm/aarch64.rs
View File

@@ -21,7 +21,7 @@ use vmm_sys_util::eventfd::EventFd;
use super::{Vm, VmError};
use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
use crate::error::{Error, StartMicrovmError};
use crate::error::{Error, StartMicroVmError};
use crate::event_manager::EventManager;
/// Configures the system and should be called once per vm before starting vcpu threads.
@@ -63,12 +63,12 @@ impl Vm {
}
/// Creates the irq chip in-kernel device model.
pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> {
pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> {
let vcpu_count = self.vm_config.vcpu_count;
self.irqchip_handle = Some(
dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into())
.map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?,
.map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupGIC(e)))?,
);
Ok(())
@@ -88,16 +88,16 @@ impl Vm {
epoll_mgr: EpollManager,
vm_as: GuestAddressSpaceImpl,
request_ts: TimestampUs,
) -> Result<(), StartMicrovmError> {
) -> Result<(), StartMicroVmError> {
let reset_eventfd =
EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?;
EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicroVmError::EventFd)?;
self.reset_eventfd = Some(
reset_eventfd
.try_clone()
.map_err(|_| StartMicrovmError::EventFd)?,
.map_err(|_| StartMicroVmError::EventFd)?,
);
self.vcpu_manager()
.map_err(StartMicrovmError::Vcpu)?
.map_err(StartMicroVmError::Vcpu)?
.set_reset_event_fd(reset_eventfd);
// On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before
@@ -106,9 +106,9 @@ impl Vm {
// Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c.
let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?;
self.vcpu_manager()
.map_err(StartMicrovmError::Vcpu)?
.map_err(StartMicroVmError::Vcpu)?
.create_boot_vcpus(request_ts, kernel_loader_result.kernel_load)
.map_err(StartMicrovmError::Vcpu)?;
.map_err(StartMicroVmError::Vcpu)?;
self.setup_interrupt_controller()?;
self.init_devices(epoll_mgr)?;
@@ -124,8 +124,8 @@ impl Vm {
vm_memory: &GuestMemoryImpl,
cmdline: &Cmdline,
initrd: Option<InitrdConfig>,
) -> std::result::Result<(), StartMicrovmError> {
let vcpu_manager = self.vcpu_manager().map_err(StartMicrovmError::Vcpu)?;
) -> std::result::Result<(), StartMicroVmError> {
let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
let vcpu_mpidr = vcpu_manager
.vcpus()
.into_iter()
@@ -141,17 +141,17 @@ impl Vm {
self.get_irqchip(),
&initrd,
)
.map_err(StartMicrovmError::ConfigureSystem)
.map_err(StartMicroVmError::ConfigureSystem)
}
pub(crate) fn register_events(
&mut self,
event_mgr: &mut EventManager,
) -> std::result::Result<(), StartMicrovmError> {
let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?;
) -> std::result::Result<(), StartMicroVmError> {
let reset_evt = self.get_reset_eventfd().ok_or(StartMicroVmError::EventFd)?;
event_mgr
.register_exit_eventfd(reset_evt)
.map_err(|_| StartMicrovmError::RegisterEvent)?;
.map_err(|_| StartMicroVmError::RegisterEvent)?;
Ok(())
}

4
src/dragonball/src/vm/mod.rs
View File

@@ -63,7 +63,7 @@ pub enum VmError {
/// Cannot setup GIC
#[cfg(target_arch = "aarch64")]
#[error("failed to configure GIC")]
SetupGIC(dbs_arch::gic::Error),
SetupGIC(GICError),
}
/// Configuration information for user defined NUMA nodes.
@@ -187,7 +187,7 @@ pub struct Vm {
// Arm specific fields.
// On aarch64 we need to keep around the fd obtained by creating the VGIC device.
#[cfg(target_arch = "aarch64")]
irqchip_handle: Option<Box<dyn dbs_arch::gic::GICDevice>>,
irqchip_handle: Option<Box<dyn GICDevice>>,
#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
upcall_client: Option<Arc<UpcallClient<DevMgrService>>>,