Merge pull request #6009 from openanolis/dragonball/add_cpu_resize

Dragonball: add cpu resize ability

src/dragonball/src/api/v1/vmm_action.rs

@@ -35,6 +35,9 @@ pub use crate::device_manager::virtio_net_dev_mgr::{
 #[cfg(feature = "virtio-vsock")]
 pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError};
 
+#[cfg(feature = "hotplug")]
+pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
+
 use super::*;
 
 /// Wrapper for all errors associated with VMM actions.
@@ -44,9 +47,13 @@ pub enum VmmActionError {
     #[error("the virtual machine instance ID is invalid")]
     InvalidVMID,
 
+    /// VM doesn't exist and can't get VM information.
+    #[error("VM doesn't exist and can't get VM information")]
+    VmNotExist,
+
     /// Failed to hotplug, due to Upcall not ready.
     #[error("Upcall not ready, can't hotplug device.")]
-    UpcallNotReady,
+    UpcallServerNotReady,
 
     /// The action `ConfigureBootSource` failed either because of bad user input or an internal
     /// error.
@@ -85,6 +92,11 @@ pub enum VmmActionError {
     /// The action `InsertFsDevice` failed either because of bad user input or an internal error.
     #[error("virtio-fs device error: {0}")]
     FsDevice(#[source] FsDeviceError),
+
+    #[cfg(feature = "hotplug")]
+    /// The action `ResizeVcpu` Failed
+    #[error("vcpu resize error : {0}")]
+    ResizeVcpu(#[source] VcpuResizeError),
 }
 
 /// This enum represents the public interface of the VMM. Each action contains various
@@ -156,6 +168,10 @@ pub enum VmmAction {
     #[cfg(feature = "virtio-fs")]
     /// Update fs rate limiter, after microVM start.
     UpdateFsDevice(FsDeviceConfigUpdateInfo),
+
+    #[cfg(feature = "hotplug")]
+    /// Resize Vcpu number in the guest.
+    ResizeVcpu(VcpuResizeInfo),
 }
 
 /// The enum represents the response sent by the VMM in case of success. The response is either
@@ -256,6 +272,8 @@ impl VmmService {
             VmmAction::UpdateFsDevice(fs_update_cfg) => {
                 self.update_fs_rate_limiters(vmm, fs_update_cfg)
             }
+            #[cfg(feature = "hotplug")]
+            VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg),
         };
 
         debug!("send vmm response: {:?}", response);
@@ -462,8 +480,8 @@ impl VmmService {
         let ctx = vm
             .create_device_op_context(Some(event_mgr.epoll_manager()))
             .map_err(|e| {
-                if let StartMicroVmError::UpcallNotReady = e {
+                if let StartMicroVmError::UpcallServerNotReady = e {
-                    return VmmActionError::UpcallNotReady;
+                    return VmmActionError::UpcallServerNotReady;
                 }
                 VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
             })?;
@@ -518,8 +536,8 @@ impl VmmService {
             .map_err(|e| {
                 if let StartMicroVmError::MicroVMAlreadyRunning = e {
                     VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot)
-                } else if let StartMicroVmError::UpcallNotReady = e {
+                } else if let StartMicroVmError::UpcallServerNotReady = e {
-                    VmmActionError::UpcallNotReady
+                    VmmActionError::UpcallServerNotReady
                 } else {
                     VmmActionError::StartMicroVm(e)
                 }
@@ -595,6 +613,37 @@ impl VmmService {
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::FsDevice)
     }
+
+    #[cfg(feature = "hotplug")]
+    fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult {
+        if !cfg!(target_arch = "x86_64") {
+            // TODO: Arm need to support vcpu hotplug. issue: #6010
+            warn!("This arch do not support vm resize!");
+            return Ok(VmmData::Empty);
+        }
+
+        if !cfg!(feature = "dbs-upcall") {
+            warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature.");
+            return Ok(VmmData::Empty);
+        }
+
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::VmNotExist)?;
+
+        if !vm.is_vm_initialized() {
+            return Err(VmmActionError::ResizeVcpu(
+                VcpuResizeError::UpdateNotAllowedPreBoot,
+            ));
+        }
+
+        vm.resize_vcpu(config, None).map_err(|e| {
+            if let VcpuResizeError::UpcallServerNotReady = e {
+                return VmmActionError::UpcallServerNotReady;
+            }
+            VmmActionError::ResizeVcpu(e)
+        })?;
+
+        Ok(VmmData::Empty)
+    }
 }
 
 fn handle_cpu_topology(

src/dragonball/src/error.rs

@@ -100,7 +100,7 @@ pub enum StartMicroVmError {
 
     /// Upcall is not ready
     #[error("the upcall client is not ready")]
-    UpcallNotReady,
+    UpcallServerNotReady,
 
     /// Configuration passed in is invalidate.
     #[error("invalid virtual machine configuration: {0} ")]

src/dragonball/src/vcpu/mod.rs

@@ -10,7 +10,10 @@ mod vcpu_manager;
 #[cfg(target_arch = "x86_64")]
 use dbs_arch::cpuid::VpmuFeatureLevel;
 
-pub use vcpu_manager::{VcpuManager, VcpuManagerError};
+pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo};
+
+#[cfg(feature = "hotplug")]
+pub use vcpu_manager::VcpuResizeError;
 
 /// vcpu config collection
 pub struct VcpuConfig {

src/dragonball/src/vcpu/vcpu_impl.rs

@@ -214,10 +214,20 @@ pub enum VcpuResponse {
     CacheRevalidated,
 }
 
+#[derive(Debug, PartialEq)]
+/// Vcpu Hotplug Result returned from the guest
+pub enum VcpuResizeResult {
+    /// All vCPU hotplug / hot-unplug operations are successful
+    Success = 0,
+    /// vCPU hotplug / hot-unplug failed
+    Failed = 1,
+}
+
 /// List of events that the vcpu_state_sender can send.
 pub enum VcpuStateEvent {
-    /// (result, response) for hotplug, result 0 means failure, 1 means success.
+    /// (result, response) for hotplug / hot-unplugged.
-    Hotplug((i32, u32)),
+    /// response records how many cpu has successfully being hotplugged / hot-unplugged.
+    Hotplug((VcpuResizeResult, u32)),
 }
 
 /// Wrapper over vCPU that hides the underlying interactions with the vCPU thread.

src/dragonball/src/vcpu/vcpu_manager.rs

@@ -29,7 +29,7 @@ use crate::address_space_manager::GuestAddressSpaceImpl;
 use crate::api::v1::InstanceInfo;
 use crate::kvm_context::KvmContext;
 use crate::vcpu::vcpu_impl::{
-    Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResponse, VcpuStateEvent,
+    Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResizeResult, VcpuResponse, VcpuStateEvent,
 };
 use crate::vcpu::VcpuConfig;
 use crate::vm::VmConfigInfo;
@@ -113,11 +113,6 @@ pub enum VcpuManagerError {
     #[error("vcpu internal error: {0}")]
     Vcpu(#[source] VcpuError),
 
-    #[cfg(feature = "hotplug")]
-    /// vCPU resize error
-    #[error("resize vcpu error: {0}")]
-    VcpuResize(#[source] VcpuResizeError),
-
     /// Kvm Ioctl Error
     #[error("failure in issuing KVM ioctl command: {0}")]
     Kvm(#[source] kvm_ioctls::Error),
@@ -132,6 +127,10 @@ pub enum VcpuResizeError {
     VcpuIsHotplugging,
 
     /// Cannot update the configuration of the microvm pre boot.
+    #[error("resize vcpu operation is not allowed pre boot")]
+    UpdateNotAllowedPreBoot,
+
+    /// Cannot update the configuration of the microvm post boot.
     #[error("resize vcpu operation is not allowed after boot")]
     UpdateNotAllowedPostBoot,
 
@@ -147,10 +146,24 @@ pub enum VcpuResizeError {
     #[error("Removable vcpu not enough, removable vcpu num: {0}, number to remove: {1}, present vcpu count {2}")]
     LackRemovableVcpus(u16, u16, u16),
 
-    #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
+    #[cfg(feature = "dbs-upcall")]
     /// Cannot update the configuration by upcall channel.
     #[error("cannot update the configuration by upcall channel: {0}")]
     Upcall(#[source] dbs_upcall::UpcallClientError),
+
+    #[cfg(feature = "dbs-upcall")]
+    /// Cannot find upcall client
+    #[error("Cannot find upcall client")]
+    UpcallClientMissing,
+
+    #[cfg(feature = "dbs-upcall")]
+    /// Upcall server is not ready
+    #[error("Upcall server is not ready")]
+    UpcallServerNotReady,
+
+    /// Vcpu manager error
+    #[error("Vcpu manager error : {0}")]
+    Vcpu(#[source] VcpuManagerError),
 }
 
 /// Result for vCPU manager operations
@@ -163,6 +176,13 @@ enum VcpuAction {
     Hotunplug,
 }
 
+/// VcpuResizeInfo describes the information for vcpu hotplug / hot-unplug
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
+pub struct VcpuResizeInfo {
+    /// The desired vcpu count to resize.
+    pub vcpu_count: Option<u8>,
+}
+
 /// Infos related to per vcpu
 #[derive(Default)]
 pub(crate) struct VcpuInfo {
@@ -810,11 +830,9 @@ mod hotplug {
             &mut self,
             vcpu_count: u8,
             sync_tx: Option<Sender<bool>>,
-        ) -> std::result::Result<(), VcpuManagerError> {
+        ) -> std::result::Result<(), VcpuResizeError> {
             if self.get_vcpus_action() != VcpuAction::None {
-                return Err(VcpuManagerError::VcpuResize(
+                return Err(VcpuResizeError::VcpuIsHotplugging);
-                    VcpuResizeError::VcpuIsHotplugging,
-                ));
             }
             self.action_sycn_tx = sync_tx;
 
@@ -831,9 +849,7 @@ mod hotplug {
                     Ordering::Less => self.do_del_vcpu(vcpu_count, upcall),
                 }
             } else {
-                Err(VcpuManagerError::VcpuResize(
+                Err(VcpuResizeError::UpdateNotAllowedPostBoot)
-                    VcpuResizeError::UpdateNotAllowedPostBoot,
-                ))
             }
         }
 
@@ -841,28 +857,31 @@ mod hotplug {
             &mut self,
             vcpu_count: u8,
             upcall_client: Arc<UpcallClient<DevMgrService>>,
-        ) -> std::result::Result<(), VcpuManagerError> {
+        ) -> std::result::Result<(), VcpuResizeError> {
             info!("resize vcpu: add");
             if vcpu_count > self.vcpu_config.max_vcpu_count {
-                return Err(VcpuManagerError::VcpuResize(
+                return Err(VcpuResizeError::ExpectedVcpuExceedMax);
-                    VcpuResizeError::ExpectedVcpuExceedMax,
-                ));
             }
 
-            let created_vcpus = self.create_vcpus(vcpu_count, None, None)?;
+            let created_vcpus = self
-            let cpu_ids = self.activate_vcpus(vcpu_count, true).map_err(|e| {
+                .create_vcpus(vcpu_count, None, None)
-                // we need to rollback when activate vcpu error
+                .map_err(VcpuResizeError::Vcpu)?;
-                error!("activate vcpu error, rollback! {:?}", e);
+            let cpu_ids = self
-                let activated_vcpus: Vec<u8> = created_vcpus
+                .activate_vcpus(vcpu_count, true)
-                    .iter()
+                .map_err(|e| {
-                    .filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some())
+                    // we need to rollback when activate vcpu error
-                    .copied()
+                    error!("activate vcpu error, rollback! {:?}", e);
-                    .collect();
+                    let activated_vcpus: Vec<u8> = created_vcpus
-                if let Err(e) = self.exit_vcpus(&activated_vcpus) {
+                        .iter()
-                    error!("try to rollback error, stop_vcpu: {:?}", e);
+                        .filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some())
-                }
+                        .copied()
-                e
+                        .collect();
-            })?;
+                    if let Err(e) = self.exit_vcpus(&activated_vcpus) {
+                        error!("try to rollback error, stop_vcpu: {:?}", e);
+                    }
+                    e
+                })
+                .map_err(VcpuResizeError::Vcpu)?;
 
             let mut cpu_ids_array = [0u8; (u8::MAX as usize) + 1];
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
@@ -882,23 +901,19 @@ mod hotplug {
             &mut self,
             vcpu_count: u8,
             upcall_client: Arc<UpcallClient<DevMgrService>>,
-        ) -> std::result::Result<(), VcpuManagerError> {
+        ) -> std::result::Result<(), VcpuResizeError> {
             info!("resize vcpu: delete");
             if vcpu_count == 0 {
-                return Err(VcpuManagerError::VcpuResize(
+                return Err(VcpuResizeError::Vcpu0CanNotBeRemoved);
-                    VcpuResizeError::Vcpu0CanNotBeRemoved,
-                ));
             }
 
             let mut cpu_ids = self.calculate_removable_vcpus();
             let cpu_num_to_be_del = (self.present_vcpus_count() - vcpu_count) as usize;
             if cpu_num_to_be_del >= cpu_ids.len() {
-                return Err(VcpuManagerError::VcpuResize(
+                return Err(VcpuResizeError::LackRemovableVcpus(
-                    VcpuResizeError::LackRemovableVcpus(
+                    cpu_ids.len() as u16,
-                        cpu_ids.len() as u16,
+                    cpu_num_to_be_del as u16,
-                        cpu_num_to_be_del as u16,
+                    self.present_vcpus_count() as u16,
-                        self.present_vcpus_count() as u16,
-                    ),
                 ));
             }
 
@@ -924,7 +939,7 @@ mod hotplug {
             &self,
             _upcall_client: Arc<UpcallClient<DevMgrService>>,
             _request: DevMgrRequest,
-        ) -> std::result::Result<(), VcpuManagerError> {
+        ) -> std::result::Result<(), VcpuResizeError> {
             Ok(())
         }
 
@@ -933,7 +948,7 @@ mod hotplug {
             &self,
             upcall_client: Arc<UpcallClient<DevMgrService>>,
             request: DevMgrRequest,
-        ) -> std::result::Result<(), VcpuManagerError> {
+        ) -> std::result::Result<(), VcpuResizeError> {
             // This is used to fix clippy warnings.
             use dbs_upcall::{DevMgrResponse, UpcallClientRequest, UpcallClientResponse};
 
@@ -946,9 +961,14 @@ mod hotplug {
                     Box::new(move |result| match result {
                         UpcallClientResponse::DevMgr(response) => {
                             if let DevMgrResponse::CpuDev(resp) = response {
+                                let result: VcpuResizeResult = if resp.result == 0 {
+                                    VcpuResizeResult::Success
+                                } else {
+                                    VcpuResizeResult::Failed
+                                };
                                 vcpu_state_sender
                                     .send(VcpuStateEvent::Hotplug((
-                                        resp.result,
+                                        result,
                                         resp.info.apic_id_index,
                                     )))
                                     .unwrap();
@@ -957,7 +977,7 @@ mod hotplug {
                         }
                         UpcallClientResponse::UpcallReset => {
                             vcpu_state_sender
-                                .send(VcpuStateEvent::Hotplug((0, 0)))
+                                .send(VcpuStateEvent::Hotplug((VcpuResizeResult::Success, 0)))
                                 .unwrap();
                             vcpu_state_event.write(1).unwrap();
                         }
@@ -968,7 +988,6 @@ mod hotplug {
                     }),
                 )
                 .map_err(VcpuResizeError::Upcall)
-                .map_err(VcpuManagerError::VcpuResize)
         }
 
         /// Get removable vcpus.
@@ -993,16 +1012,19 @@ impl VcpuEpollHandler {
         while let Ok(event) = self.rx.try_recv() {
             match event {
                 VcpuStateEvent::Hotplug((success, cpu_count)) => {
-                    info!("get vcpu event, cpu_index {}", cpu_count);
+                    info!(
-                    self.process_cpu_action(success != 0, cpu_count);
+                        "get vcpu event, cpu_index {} success {:?}",
+                        cpu_count, success
+                    );
+                    self.process_cpu_action(success, cpu_count);
                 }
             }
         }
     }
 
-    fn process_cpu_action(&self, success: bool, _cpu_index: u32) {
+    fn process_cpu_action(&self, result: VcpuResizeResult, _cpu_index: u32) {
         let mut vcpu_manager = self.vcpu_manager.lock().unwrap();
-        if success {
+        if result == VcpuResizeResult::Success {
             match vcpu_manager.get_vcpus_action() {
                 VcpuAction::Hotplug => {
                     // Notify hotplug success
@@ -1362,12 +1384,7 @@ mod tests {
 
         // vcpu is already in hotplug process
         let res = vcpu_manager.resize_vcpu(1, None);
-        assert!(matches!(
+        assert!(matches!(res, Err(VcpuResizeError::VcpuIsHotplugging)));
-            res,
-            Err(VcpuManagerError::VcpuResize(
-                VcpuResizeError::VcpuIsHotplugging
-            ))
-        ));
 
         // clear vcpus action
         let cpu_ids = vec![0];
@@ -1377,9 +1394,7 @@ mod tests {
         let res = vcpu_manager.resize_vcpu(1, None);
         assert!(matches!(
             res,
-            Err(VcpuManagerError::VcpuResize(
+            Err(VcpuResizeError::UpdateNotAllowedPostBoot)
-                VcpuResizeError::UpdateNotAllowedPostBoot
-            ))
         ));
 
         // init upcall channel
@@ -1397,20 +1412,10 @@ mod tests {
 
         // exceeed max vcpu count
         let res = vcpu_manager.resize_vcpu(4, None);
-        assert!(matches!(
+        assert!(matches!(res, Err(VcpuResizeError::ExpectedVcpuExceedMax)));
-            res,
-            Err(VcpuManagerError::VcpuResize(
-                VcpuResizeError::ExpectedVcpuExceedMax
-            ))
-        ));
 
         // remove vcpu 0
         let res = vcpu_manager.resize_vcpu(0, None);
-        assert!(matches!(
+        assert!(matches!(res, Err(VcpuResizeError::Vcpu0CanNotBeRemoved)));
-            res,
-            Err(VcpuManagerError::VcpuResize(
-                VcpuResizeError::Vcpu0CanNotBeRemoved
-            ))
-        ));
     }
 }

src/dragonball/src/vm/mod.rs

@@ -4,6 +4,7 @@
 use std::io::{self, Read, Seek, SeekFrom};
 use std::ops::Deref;
 use std::os::unix::io::RawFd;
+
 use std::sync::{Arc, Mutex, RwLock};
 
 use dbs_address_space::AddressSpace;
@@ -22,6 +23,8 @@ use vmm_sys_util::eventfd::EventFd;
 
 #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
 use dbs_upcall::{DevMgrService, UpcallClient};
+#[cfg(feature = "hotplug")]
+use std::sync::mpsc::Sender;
 
 use crate::address_space_manager::{
     AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl,
@@ -35,6 +38,8 @@ use crate::event_manager::EventManager;
 use crate::kvm_context::KvmContext;
 use crate::resource_manager::ResourceManager;
 use crate::vcpu::{VcpuManager, VcpuManagerError};
+#[cfg(feature = "hotplug")]
+use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
 #[cfg(target_arch = "aarch64")]
 use dbs_arch::gic::Error as GICError;
 
@@ -795,7 +800,30 @@ impl Vm {
         } else if self.is_upcall_client_ready() {
             Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr))
         } else {
-            Err(StartMicroVmError::UpcallNotReady)
+            Err(StartMicroVmError::UpcallServerNotReady)
+        }
+    }
+
+    /// Resize MicroVM vCPU number
+    #[cfg(feature = "hotplug")]
+    pub fn resize_vcpu(
+        &mut self,
+        config: VcpuResizeInfo,
+        sync_tx: Option<Sender<bool>>,
+    ) -> std::result::Result<(), VcpuResizeError> {
+        if self.upcall_client().is_none() {
+            Err(VcpuResizeError::UpcallClientMissing)
+        } else if self.is_upcall_client_ready() {
+            if let Some(vcpu_count) = config.vcpu_count {
+                self.vcpu_manager()
+                    .map_err(VcpuResizeError::Vcpu)?
+                    .resize_vcpu(vcpu_count, sync_tx)?;
+
+                self.vm_config.vcpu_count = vcpu_count;
+            }
+            Ok(())
+        } else {
+            Err(VcpuResizeError::UpcallServerNotReady)
         }
     }
 

src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs

@@ -314,7 +314,7 @@ impl VmmInstance {
                         return Ok(vmm_data);
                     }
                     Err(vmm_action_error) => {
-                        if let VmmActionError::UpcallNotReady = vmm_action_error {
+                        if let VmmActionError::UpcallServerNotReady = vmm_action_error {
                             std::thread::sleep(std::time::Duration::from_millis(10));
                             continue;
                         } else {