Merge pull request #7932 from Apokleos/wrap-virtiofs-in-dm

runtime-rs: bringing virtio-fs device in device-manager

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -102,7 +102,10 @@ impl ResourceManagerInner {
                     {
                         let share_fs = share_fs::new(&self.sid, &c).context("new share fs")?;
                         share_fs
-                            .setup_device_before_start_vm(self.hypervisor.as_ref())
+                            .setup_device_before_start_vm(
+                                self.hypervisor.as_ref(),
+                                &self.device_manager,
+                            )
                             .await
                             .context("setup share fs device before start vm")?;
 
@@ -212,7 +215,7 @@ impl ResourceManagerInner {
     pub async fn setup_after_start_vm(&mut self) -> Result<()> {
         if let Some(share_fs) = self.share_fs.as_ref() {
             share_fs
-                .setup_device_after_start_vm(self.hypervisor.as_ref())
+                .setup_device_after_start_vm(self.hypervisor.as_ref(), &self.device_manager)
                 .await
                 .context("setup share fs device after start vm")?;
         }
@@ -227,6 +230,7 @@ impl ResourceManagerInner {
                 .context("handle neighbors")?;
             self.handle_routes(network).await.context("handle routes")?;
         }
+
         Ok(())
     }
 

src/runtime-rs/crates/resource/src/rootfs/mod.rs

@@ -23,6 +23,7 @@ use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE};
 const ROOTFS: &str = "rootfs";
 const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower";
 const TYPE_OVERLAY_FS: &str = "overlay";
+
 #[async_trait]
 pub trait Rootfs: Send + Sync {
     async fn get_guest_rootfs_path(&self) -> Result<String>;
@@ -102,9 +103,16 @@ impl RootFsResource {
                     // handle nydus rootfs
                     let share_rootfs: Arc<dyn Rootfs> = if layer.fs_type == NYDUS_ROOTFS_TYPE {
                         Arc::new(
-                            nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer)
-                                .await
-                                .context("new nydus rootfs")?,
+                            nydus_rootfs::NydusRootfs::new(
+                                device_manager,
+                                share_fs,
+                                h,
+                                sid,
+                                cid,
+                                layer,
+                            )
+                            .await
+                            .context("new nydus rootfs")?,
                         )
                     }
                     // handle sharefs rootfs

src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs

@@ -39,6 +39,7 @@ pub(crate) struct NydusRootfs {
 
 impl NydusRootfs {
     pub async fn new(
+        d: &RwLock<DeviceManager>,
         share_fs: &Arc<dyn ShareFs>,
         h: &dyn Hypervisor,
         sid: &str,
@@ -61,7 +62,8 @@ impl NydusRootfs {
                 // rafs mount the metadata of nydus rootfs
                 let rafs_mnt = do_get_guest_share_path(HYBRID_ROOTFS_LOWER_DIR, cid, true);
                 rafs_mount(
-                    h,
+                    d,
+                    sid,
                     rafs_meta.to_string(),
                     rafs_mnt,
                     extra_options.config.clone(),

src/runtime-rs/crates/resource/src/share_fs/mod.rs

@@ -22,12 +22,15 @@ pub mod sandbox_bind_mounts;
 
 use std::{collections::HashMap, fmt::Debug, path::PathBuf, sync::Arc};
 
-use agent::Storage;
 use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
+use tokio::sync::RwLock;
+
+use agent::Storage;
 use kata_types::config::hypervisor::SharedFsInfo;
 
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
+
 const VIRTIO_FS: &str = "virtio-fs";
 const _VIRTIO_FS_NYDUS: &str = "virtio-fs-nydus";
 const INLINE_VIRTIO_FS: &str = "inline-virtio-fs";
@@ -45,8 +48,16 @@ const RAFS_DIR: &str = "rafs";
 #[async_trait]
 pub trait ShareFs: Send + Sync {
     fn get_share_fs_mount(&self) -> Arc<dyn ShareFsMount>;
-    async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()>;
-    async fn setup_device_after_start_vm(&self, h: &dyn Hypervisor) -> Result<()>;
+    async fn setup_device_before_start_vm(
+        &self,
+        h: &dyn Hypervisor,
+        d: &RwLock<DeviceManager>,
+    ) -> Result<()>;
+    async fn setup_device_after_start_vm(
+        &self,
+        h: &dyn Hypervisor,
+        d: &RwLock<DeviceManager>,
+    ) -> Result<()>;
     async fn get_storages(&self) -> Result<Vec<Storage>>;
     fn mounted_info_set(&self) -> Arc<Mutex<HashMap<String, MountedInfo>>>;
 }

src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs

@@ -7,18 +7,18 @@
 use std::path::Path;
 
 use anyhow::{Context, Result};
+use nix::mount::MsFlags;
+use tokio::sync::RwLock;
+
 use hypervisor::{
     device::{
-        driver::{
-            ShareFsDevice, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType,
-            ShareFsOperation,
-        },
-        DeviceType,
+        device_manager::{do_handle_device, do_update_device, DeviceManager},
+        driver::{ShareFsMountConfig, ShareFsMountOperation, ShareFsMountType},
+        DeviceConfig,
     },
-    Hypervisor, ShareFsDeviceConfig,
+    ShareFsConfig,
 };
 use kata_sys_util::mount;
-use nix::mount::MsFlags;
 
 use super::{utils, PASSTHROUGH_FS_DIR};
 
@@ -35,7 +35,7 @@ pub(crate) fn generate_sock_path(root: &str) -> String {
 }
 
 pub(crate) async fn prepare_virtiofs(
-    h: &dyn Hypervisor,
+    d: &RwLock<DeviceManager>,
     fs_type: &str,
     id: &str,
     root: &str,
@@ -49,24 +49,26 @@ pub(crate) async fn prepare_virtiofs(
     mount::bind_mount_unchecked(&host_rw_dest, &host_ro_dest, true, MsFlags::MS_SLAVE)
         .context("bind mount shared_fs directory")?;
 
-    let share_fs_device = ShareFsDevice {
-        config: ShareFsDeviceConfig {
-            sock_path: generate_sock_path(root),
-            mount_tag: String::from(MOUNT_GUEST_TAG),
-            host_path: String::from(host_ro_dest.to_str().unwrap()),
-            fs_type: fs_type.to_string(),
-            queue_size: 0,
-            queue_num: 0,
-            options: vec![],
-        },
+    let sharefs_config = ShareFsConfig {
+        host_shared_path: host_ro_dest.display().to_string(),
+        sock_path: generate_sock_path(root),
+        mount_tag: String::from(MOUNT_GUEST_TAG),
+        fs_type: fs_type.to_string(),
+        queue_size: 0,
+        queue_num: 0,
+        options: vec![],
+        mount_config: None,
     };
-    h.add_device(DeviceType::ShareFs(share_fs_device))
+
+    // create and insert virtio-fs device into Guest
+    do_handle_device(d, &DeviceConfig::ShareFsCfg(sharefs_config))
         .await
-        .context("add device")?;
+        .context("do add virtio-fs device failed.")?;
+
     Ok(())
 }
 
-pub(crate) async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Result<()> {
+pub(crate) async fn setup_inline_virtiofs(d: &RwLock<DeviceManager>, id: &str) -> Result<()> {
     // - source is the absolute path of PASSTHROUGH_FS_DIR on host, e.g.
     //   /run/kata-containers/shared/sandboxes/<sid>/passthrough
     // - mount point is the path relative to KATA_GUEST_SHARE_DIR in guest
@@ -75,34 +77,39 @@ pub(crate) async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Resul
     let rw_source = utils::get_host_rw_shared_path(id).join(PASSTHROUGH_FS_DIR);
     utils::ensure_dir_exist(&rw_source).context("ensure directory exist")?;
 
-    let ro_source = utils::get_host_ro_shared_path(id).join(PASSTHROUGH_FS_DIR);
-    let source = String::from(ro_source.to_str().unwrap());
+    let host_ro_shared_path = utils::get_host_ro_shared_path(id);
+    let source = host_ro_shared_path
+        .join(PASSTHROUGH_FS_DIR)
+        .display()
+        .to_string();
 
-    let virtio_fs = ShareFsMountDevice {
-        config: ShareFsMountConfig {
-            source: source.clone(),
-            fstype: ShareFsMountType::PASSTHROUGH,
-            mount_point: mnt,
-            config: None,
-            tag: String::from(MOUNT_GUEST_TAG),
-            op: ShareFsOperation::Mount,
-            prefetch_list_path: None,
-        },
+    let virtiofs_mount = ShareFsMountConfig {
+        source: source.clone(),
+        fstype: ShareFsMountType::PASSTHROUGH,
+        mount_point: mnt,
+        config: None,
+        tag: String::from(MOUNT_GUEST_TAG),
+        op: ShareFsMountOperation::Mount,
+        prefetch_list_path: None,
     };
 
-    let result = h
-        .add_device(DeviceType::ShareFsMount(virtio_fs))
-        .await
-        .with_context(|| format!("fail to attach passthrough fs {:?}", source));
+    let sharefs_config = ShareFsConfig {
+        host_shared_path: host_ro_shared_path.display().to_string(),
+        mount_config: Some(virtiofs_mount),
+        ..Default::default()
+    };
 
-    match result {
-        Ok(_) => Ok(()),
-        Err(e) => Err(e),
-    }
+    // update virtio-fs device with ShareFsMountConfig
+    do_update_device(d, &DeviceConfig::ShareFsCfg(sharefs_config))
+        .await
+        .context("fail to attach passthrough fs.")?;
+
+    Ok(())
 }
 
 pub async fn rafs_mount(
-    h: &dyn Hypervisor,
+    d: &RwLock<DeviceManager>,
+    sid: &str,
     rafs_meta: String,
     rafs_mnt: String,
     config_content: String,
@@ -112,19 +119,28 @@ pub async fn rafs_mount(
         sl!(),
         "Attaching rafs meta file {} to virtio-fs device, rafs mount point {}", rafs_meta, rafs_mnt
     );
-    let virtio_fs = ShareFsMountDevice {
-        config: ShareFsMountConfig {
-            source: rafs_meta.clone(),
-            fstype: ShareFsMountType::RAFS,
-            mount_point: rafs_mnt,
-            config: Some(config_content),
-            tag: String::from(MOUNT_GUEST_TAG),
-            op: ShareFsOperation::Mount,
-            prefetch_list_path,
-        },
+
+    let rafs_config = ShareFsMountConfig {
+        source: rafs_meta.clone(),
+        fstype: ShareFsMountType::RAFS,
+        mount_point: rafs_mnt,
+        config: Some(config_content),
+        tag: String::from(MOUNT_GUEST_TAG),
+        op: ShareFsMountOperation::Mount,
+        prefetch_list_path,
     };
-    h.add_device(DeviceType::ShareFsMount(virtio_fs))
+
+    let host_shared_path = utils::get_host_ro_shared_path(sid).display().to_string();
+    let sharefs_config = ShareFsConfig {
+        host_shared_path,
+        mount_config: Some(rafs_config),
+        ..Default::default()
+    };
+
+    // update virtio-fs device with ShareFsMountConfig
+    do_update_device(d, &DeviceConfig::ShareFsCfg(sharefs_config))
         .await
         .with_context(|| format!("fail to attach rafs {:?}", rafs_meta))?;
+
     Ok(())
 }

src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs

@@ -6,12 +6,13 @@
 
 use std::collections::HashMap;
 
-use agent::Storage;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
+use tokio::sync::{Mutex, RwLock};
+
+use agent::Storage;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use kata_types::config::hypervisor::SharedFsInfo;
-use tokio::sync::Mutex;
 
 use super::{
     share_virtio_fs::{
@@ -52,19 +53,30 @@ impl ShareFs for ShareVirtioFsInline {
         self.share_fs_mount.clone()
     }
 
-    async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()> {
-        prepare_virtiofs(h, INLINE_VIRTIO_FS, &self.config.id, "")
+    async fn setup_device_before_start_vm(
+        &self,
+        _h: &dyn Hypervisor,
+        d: &RwLock<DeviceManager>,
+    ) -> Result<()> {
+        prepare_virtiofs(d, INLINE_VIRTIO_FS, &self.config.id, "")
             .await
             .context("prepare virtiofs")?;
+
         Ok(())
     }
 
-    async fn setup_device_after_start_vm(&self, h: &dyn Hypervisor) -> Result<()> {
-        setup_inline_virtiofs(&self.config.id, h)
+    async fn setup_device_after_start_vm(
+        &self,
+        _h: &dyn Hypervisor,
+        d: &RwLock<DeviceManager>,
+    ) -> Result<()> {
+        setup_inline_virtiofs(d, &self.config.id)
             .await
             .context("setup inline virtiofs")?;
+
         Ok(())
     }
+
     async fn get_storages(&self) -> Result<Vec<Storage>> {
         // setup storage
         let mut storages: Vec<Storage> = Vec::new();

src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs

@@ -6,15 +6,8 @@
 
 use std::{collections::HashMap, process::Stdio, sync::Arc};
 
-use crate::share_fs::share_virtio_fs::{
-    prepare_virtiofs, FS_TYPE_VIRTIO_FS, KATA_VIRTIO_FS_DEV_TYPE, MOUNT_GUEST_TAG,
-};
-use crate::share_fs::{KATA_GUEST_SHARE_DIR, VIRTIO_FS};
-use agent::Storage;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
-use kata_types::config::hypervisor::SharedFsInfo;
 use tokio::{
     io::{AsyncBufReadExt, BufReader},
     process::{Child, Command},
@@ -24,10 +17,20 @@ use tokio::{
     },
 };
 
+use agent::Storage;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
+use kata_types::config::hypervisor::SharedFsInfo;
+
 use super::{
     share_virtio_fs::generate_sock_path, utils::ensure_dir_exist, utils::get_host_ro_shared_path,
     virtio_fs_share_mount::VirtiofsShareMount, MountedInfo, ShareFs, ShareFsMount,
 };
+use crate::share_fs::{
+    share_virtio_fs::{
+        prepare_virtiofs, FS_TYPE_VIRTIO_FS, KATA_VIRTIO_FS_DEV_TYPE, MOUNT_GUEST_TAG,
+    },
+    KATA_GUEST_SHARE_DIR, VIRTIO_FS,
+};
 
 #[derive(Debug, Clone)]
 pub struct ShareVirtioFsStandaloneConfig {
@@ -172,15 +175,24 @@ impl ShareFs for ShareVirtioFsStandalone {
         self.share_fs_mount.clone()
     }
 
-    async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()> {
-        prepare_virtiofs(h, VIRTIO_FS, &self.config.id, &h.get_jailer_root().await?)
+    async fn setup_device_before_start_vm(
+        &self,
+        h: &dyn Hypervisor,
+        d: &RwLock<DeviceManager>,
+    ) -> Result<()> {
+        prepare_virtiofs(d, VIRTIO_FS, &self.config.id, &h.get_jailer_root().await?)
             .await
             .context("prepare virtiofs")?;
         self.setup_virtiofsd(h).await.context("setup virtiofsd")?;
+
         Ok(())
     }
 
-    async fn setup_device_after_start_vm(&self, _h: &dyn Hypervisor) -> Result<()> {
+    async fn setup_device_after_start_vm(
+        &self,
+        _h: &dyn Hypervisor,
+        _d: &RwLock<DeviceManager>,
+    ) -> Result<()> {
         Ok(())
     }