From 0a21ad78b12d362912a826b8e22140f45e8d9f9f Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Tue, 7 Feb 2023 18:30:16 +0100
Subject: [PATCH 01/76] osbuilder: fix default build target in makefile

The .dracut_rootfs.done file is accidentally being picked up as the default
target, regardless of BUILD_METHOD. Move the 'all' target definition up, so
that it's the default (=first) target in the makefile. Additionally make the
.dracut_rootfs.done target conditional on the right BUILD_METHOD being
selected, as building it doesn't make sense with BUILD_METHOD=distro.

Fixes: #6235
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 tools/osbuilder/Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tools/osbuilder/Makefile b/tools/osbuilder/Makefile
index 25effc967..19dc3bd0e 100644
--- a/tools/osbuilder/Makefile
+++ b/tools/osbuilder/Makefile
@@ -85,6 +85,9 @@ endif
 
 ################################################################################
 
+.PHONY: all
+all: image initrd
+
 rootfs-%: $(ROOTFS_BUILD_DEST)/.%$(ROOTFS_MARKER_SUFFIX)
 	@ # DONT remove. This is not cancellation rule.
 
@@ -97,11 +100,13 @@ $(ROOTFS_BUILD_DEST)/.%$(ROOTFS_MARKER_SUFFIX):: rootfs-builder/%
 # extract it in a local folder.
 # Notes:
 # - assuming a not compressed initrd.
+ifeq (dracut,$(BUILD_METHOD))
 .PRECIOUS: $(ROOTFS_BUILD_DEST)/.dracut$(ROOTFS_MARKER_SUFFIX)
 $(ROOTFS_BUILD_DEST)/.dracut$(ROOTFS_MARKER_SUFFIX): $(TARGET_INITRD)
 	mkdir -p $(TARGET_ROOTFS)
 	(cd $(TARGET_ROOTFS); cat $< | cpio --extract --preserve-modification-time --make-directories)
 	@touch $@
+endif
 
 image-%: $(IMAGES_BUILD_DEST)/kata-containers-image-%.img
 	@ # DONT remove. This is not cancellation rule.
@@ -117,9 +122,6 @@ initrd-%: $(IMAGES_BUILD_DEST)/kata-containers-initrd-%.img
 $(IMAGES_BUILD_DEST)/kata-containers-initrd-%.img: rootfs-%
 	$(call silent_run,Creating initrd image for $*,$(INITRD_BUILDER) -o $@ $(ROOTFS_BUILD_DEST)/$*_rootfs)
 
-.PHONY: all
-all: image initrd
-
 .PHONY: rootfs
 rootfs: $(TARGET_ROOTFS_MARKER)
 

From 2dd2421ad0c7cda501e924624324a980e8353ac2 Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Mon, 16 Jan 2023 11:18:57 +0800
Subject: [PATCH 02/76] runtime-rs: cleanup kata host share path

cleanup the /run/kata-containers/shared/sandboxes/pid path

Fixes:#5975
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
---
 src/runtime-rs/crates/resource/src/manager.rs |  4 +--
 .../crates/resource/src/manager_inner.rs      | 20 +++++++++++--
 .../crates/resource/src/share_fs/mod.rs       |  2 ++
 .../crates/resource/src/share_fs/utils.rs     |  4 +++
 .../src/share_fs/virtio_fs_share_mount.rs     | 19 +++++++++++--
 .../crates/runtimes/common/src/sandbox.rs     |  2 +-
 src/runtime-rs/crates/runtimes/src/manager.rs |  2 +-
 .../runtimes/virt_container/src/sandbox.rs    | 28 +++++++++----------
 8 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs
index 26cc2a26f..0a0082512 100644
--- a/src/runtime-rs/crates/resource/src/manager.rs
+++ b/src/runtime-rs/crates/resource/src/manager.rs
@@ -101,9 +101,9 @@ impl ResourceManager {
         inner.update_cgroups(cid, linux_resources).await
     }
 
-    pub async fn delete_cgroups(&self) -> Result<()> {
+    pub async fn cleanup(&self) -> Result<()> {
         let inner = self.inner.read().await;
-        inner.delete_cgroups().await
+        inner.cleanup().await
     }
 }
 
diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs
index 7bdfddb6a..e8623e37c 100644
--- a/src/runtime-rs/crates/resource/src/manager_inner.rs
+++ b/src/runtime-rs/crates/resource/src/manager_inner.rs
@@ -8,7 +8,7 @@ use std::{sync::Arc, thread};
 
 use crate::resource_persist::ResourceState;
 use agent::{Agent, Storage};
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
 use hypervisor::Hypervisor;
 use kata_types::config::TomlConfig;
@@ -233,8 +233,22 @@ impl ResourceManagerInner {
             .await
     }
 
-    pub async fn delete_cgroups(&self) -> Result<()> {
-        self.cgroups_resource.delete().await
+    pub async fn cleanup(&self) -> Result<()> {
+        // clean up cgroup
+        self.cgroups_resource
+            .delete()
+            .await
+            .context("delete cgroup")?;
+        // clean up share fs mount
+        if let Some(share_fs) = &self.share_fs {
+            share_fs
+                .get_share_fs_mount()
+                .cleanup(&self.sid)
+                .await
+                .context("failed to cleanup host path")?;
+        }
+        // TODO cleanup other resources
+        Ok(())
     }
 
     pub async fn dump(&self) {
diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs
index 739fa0459..350c7ea71 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs
@@ -131,6 +131,8 @@ pub trait ShareFsMount: Send + Sync {
     async fn umount_volume(&self, file_name: &str) -> Result<()>;
     /// Umount the rootfs
     async fn umount_rootfs(&self, config: &ShareFsRootfsConfig) -> Result<()>;
+    /// Clean up share fs mount
+    async fn cleanup(&self, sid: &str) -> Result<()>;
 }
 
 pub fn new(id: &str, config: &SharedFsInfo) -> Result<Arc<dyn ShareFs>> {
diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs
index 6288e860e..47f3df533 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs
@@ -59,6 +59,10 @@ pub fn get_host_rw_shared_path(sid: &str) -> PathBuf {
     Path::new(KATA_HOST_SHARED_DIR).join(sid).join("rw")
 }
 
+pub fn get_host_shared_path(sid: &str) -> PathBuf {
+    Path::new(KATA_HOST_SHARED_DIR).join(sid)
+}
+
 fn do_get_guest_any_path(
     target: &str,
     cid: &str,
diff --git a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs
index 27fb47972..2cea9904d 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs
@@ -7,7 +7,7 @@
 use agent::Storage;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use kata_sys_util::mount::{bind_remount, umount_timeout};
+use kata_sys_util::mount::{bind_remount, umount_all, umount_timeout};
 use kata_types::k8s::is_watchable_mount;
 use kata_types::mount;
 use nix::sys::stat::stat;
@@ -20,7 +20,8 @@ const WATCHABLE_BIND_DEV_TYPE: &str = "watchable-bind";
 pub const EPHEMERAL_PATH: &str = "/run/kata-containers/sandbox/ephemeral";
 
 use super::{
-    utils::{self, do_get_host_path},
+    get_host_rw_shared_path,
+    utils::{self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path},
     ShareFsMount, ShareFsMountResult, ShareFsRootfsConfig, ShareFsVolumeConfig,
     KATA_GUEST_SHARE_DIR, PASSTHROUGH_FS_DIR,
 };
@@ -224,4 +225,18 @@ impl ShareFsMount for VirtiofsShareMount {
 
         Ok(())
     }
+
+    async fn cleanup(&self, sid: &str) -> Result<()> {
+        // Unmount ro path
+        let host_ro_dest = get_host_ro_shared_path(sid);
+        umount_all(host_ro_dest.clone(), true).context("failed to umount ro path")?;
+        fs::remove_dir_all(host_ro_dest).context("failed to remove ro path")?;
+        // As the rootfs and volume have been umounted before calling this function, so just remove the rw dir directly
+        let host_rw_dest = get_host_rw_shared_path(sid);
+        fs::remove_dir_all(host_rw_dest).context("failed to remove rw path")?;
+        // remove the host share directory
+        let host_path = get_host_shared_path(sid);
+        fs::remove_dir_all(host_path).context("failed to remove host shared path")?;
+        Ok(())
+    }
 }
diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
index 5ac028ddb..1f03f50a8 100644
--- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
@@ -11,7 +11,7 @@ use async_trait::async_trait;
 pub trait Sandbox: Send + Sync {
     async fn start(&self, netns: Option<String>) -> Result<()>;
     async fn stop(&self) -> Result<()>;
-    async fn cleanup(&self, container_id: &str) -> Result<()>;
+    async fn cleanup(&self) -> Result<()>;
     async fn shutdown(&self) -> Result<()>;
 
     // agent function
diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs
index f3163a3dd..78ab42941 100644
--- a/src/runtime-rs/crates/runtimes/src/manager.rs
+++ b/src/runtime-rs/crates/runtimes/src/manager.rs
@@ -174,7 +174,7 @@ impl RuntimeHandlerManager {
                     .await
                     .context("failed to restore the sandbox")?;
                 sandbox
-                    .cleanup(&inner.id)
+                    .cleanup()
                     .await
                     .context("failed to cleanup the resource")?;
             }
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
index 469d5ea42..56e93be75 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
@@ -242,17 +242,7 @@ impl Sandbox for VirtSandbox {
 
         self.stop().await.context("stop")?;
 
-        info!(sl!(), "delete cgroup");
-        self.resource_manager
-            .delete_cgroups()
-            .await
-            .context("delete cgroups")?;
-
-        info!(sl!(), "delete hypervisor");
-        self.hypervisor
-            .cleanup()
-            .await
-            .context("delete hypervisor")?;
+        self.cleanup().await.context("do the clean up")?;
 
         info!(sl!(), "stop monitor");
         self.monitor.stop().await;
@@ -269,9 +259,19 @@ impl Sandbox for VirtSandbox {
         Ok(())
     }
 
-    async fn cleanup(&self, _id: &str) -> Result<()> {
-        self.resource_manager.delete_cgroups().await?;
-        self.hypervisor.cleanup().await?;
+    async fn cleanup(&self) -> Result<()> {
+        info!(sl!(), "delete hypervisor");
+        self.hypervisor
+            .cleanup()
+            .await
+            .context("delete hypervisor")?;
+
+        info!(sl!(), "resource clean up");
+        self.resource_manager
+            .cleanup()
+            .await
+            .context("resource clean up")?;
+
         // TODO: cleanup other snadbox resource
         Ok(())
     }

From b582c0db86b305e750ddf330c81756960cc6ea22 Mon Sep 17 00:00:00 2001
From: "alex.lyn" <alex.lyn@antgroup.com>
Date: Thu, 16 Feb 2023 17:05:53 +0800
Subject: [PATCH 03/76] kata-ctl/exec: add new command exec to enter guest VM.

The patchset will help users to easily enter guest VM by debug
console sock.

In order to enter guest VM smoothly, users needs to do some
configuration, options as below:
(1) Set debug_console_enabled = true with default vport 1026.
(2) Or add agent.debug_console agent.debug_console_vport=<PORT>
into kernel_params, and the vport is <PORT> you set.

The detail of usage:
$ kata-ctl exec -h
kata-ctl-exec
Enter into guest VM by debug console

USAGE:
kata-ctl exec [OPTIONS] <SANDBOX_ID>

ARGS:
<SANDBOX_ID> pod sandbox ID

Fixes: #5340

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 src/tools/kata-ctl/Cargo.lock           | 492 ++++++++++++++----------
 src/tools/kata-ctl/Cargo.toml           |   9 +
 src/tools/kata-ctl/src/args.rs          |  11 +-
 src/tools/kata-ctl/src/check.rs         |   2 +-
 src/tools/kata-ctl/src/main.rs          |   6 +-
 src/tools/kata-ctl/src/ops.rs           |   1 +
 src/tools/kata-ctl/src/ops/check_ops.rs |   4 -
 src/tools/kata-ctl/src/ops/exec_ops.rs  | 444 +++++++++++++++++++++
 8 files changed, 762 insertions(+), 207 deletions(-)
 create mode 100644 src/tools/kata-ctl/src/ops/exec_ops.rs

diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock
index 85d4eb517..3edafa91d 100644
--- a/src/tools/kata-ctl/Cargo.lock
+++ b/src/tools/kata-ctl/Cargo.lock
@@ -11,7 +11,7 @@ dependencies = [
  "kata-types",
  "log",
  "logging",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "protobuf",
  "protocols",
@@ -35,9 +35,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.65"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
 
 [[package]]
 name = "arc-swap"
@@ -47,9 +47,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
 
 [[package]]
 name = "async-trait"
-version = "0.1.60"
+version = "0.1.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3"
+checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -62,7 +62,7 @@ version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.1.19",
  "libc",
  "winapi",
 ]
@@ -75,9 +75,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "base64"
-version = "0.13.0"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "base64"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
 
 [[package]]
 name = "bitflags"
@@ -97,9 +103,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.11.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
 [[package]]
 name = "byte-unit"
@@ -125,15 +131,15 @@ dependencies = [
 
 [[package]]
 name = "bytes"
-version = "1.2.1"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 
 [[package]]
 name = "cfg-if"
@@ -143,9 +149,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "3.2.22"
+version = "3.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750"
+checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
 dependencies = [
  "atty",
  "bitflags",
@@ -164,7 +170,7 @@ version = "3.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65"
 dependencies = [
- "heck 0.4.0",
+ "heck 0.4.1",
  "proc-macro-error",
  "proc-macro2",
  "quote",
@@ -228,19 +234,29 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.31"
+version = "0.8.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
+checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
 dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "epoll"
+version = "4.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.8.0"
@@ -288,9 +304,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
+checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -303,9 +319,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
+checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -313,15 +329,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
+checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"
+checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -330,15 +346,15 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
+checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
+checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -347,21 +363,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
+checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
 
 [[package]]
 name = "futures-task"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
+checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
 
 [[package]]
 name = "futures-util"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
+checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -377,17 +393,17 @@ dependencies = [
 
 [[package]]
 name = "glob"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "h2"
-version = "0.3.14"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be"
+checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "fnv",
  "futures-core",
  "futures-sink",
@@ -417,9 +433,9 @@ dependencies = [
 
 [[package]]
 name = "heck"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
 [[package]]
 name = "hermit-abi"
@@ -430,6 +446,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -442,7 +467,7 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "fnv",
  "itoa",
 ]
@@ -453,7 +478,7 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "http",
  "pin-project-lite",
 ]
@@ -472,11 +497,11 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
 [[package]]
 name = "hyper"
-version = "0.14.20"
+version = "0.14.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
+checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -496,9 +521,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.23.0"
+version = "0.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"
+checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
 dependencies = [
  "http",
  "hyper",
@@ -513,7 +538,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "hyper",
  "native-tls",
  "tokio",
@@ -545,9 +570,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.9.1"
+version = "1.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
+checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -573,9 +598,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.5.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
+checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
 
 [[package]]
 name = "itertools"
@@ -588,15 +613,15 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
 [[package]]
 name = "js-sys"
-version = "0.3.60"
+version = "0.3.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -607,11 +632,15 @@ version = "0.0.1"
 dependencies = [
  "agent",
  "anyhow",
- "base64",
+ "base64 0.13.1",
  "clap",
+ "epoll",
  "futures",
+ "hyper",
  "kata-types",
- "nix 0.25.0",
+ "libc",
+ "micro_http",
+ "nix 0.25.1",
  "privdrop",
  "reqwest",
  "safe-path",
@@ -620,12 +649,17 @@ dependencies = [
  "serde_json",
  "serial_test",
  "shim-interface",
+ "slog",
+ "slog-scope",
  "strum",
  "strum_macros",
  "tempfile",
  "test-utils",
  "thiserror",
+ "tokio",
+ "ttrpc",
  "url",
+ "vmm-sys-util",
 ]
 
 [[package]]
@@ -633,7 +667,7 @@ name = "kata-types"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "base64",
+ "base64 0.13.1",
  "bitmask-enum",
  "byte-unit",
  "glob",
@@ -657,9 +691,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.135"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
 [[package]]
 name = "lock_api"
@@ -706,6 +740,24 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "micro_http"
+version = "0.1.0"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#b538bf89e50be83b6fa9ab1896727ff61e02fa13"
+dependencies = [
+ "libc",
+ "vmm-sys-util",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -714,9 +766,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
 [[package]]
 name = "mio"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
+checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
 dependencies = [
  "libc",
  "log",
@@ -732,9 +784,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
 [[package]]
 name = "native-tls"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
 dependencies = [
  "lazy_static",
  "libc",
@@ -758,42 +810,56 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
 name = "nix"
-version = "0.24.2"
+version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
+checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
 dependencies = [
  "bitflags",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
 name = "nix"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb"
+checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
 dependencies = [
  "autocfg",
  "bitflags",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
  "pin-utils",
 ]
 
 [[package]]
-name = "num_cpus"
-version = "1.13.1"
+name = "nix"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
 dependencies = [
- "hermit-abi",
+ "bitflags",
+ "cfg-if",
+ "libc",
+ "memoffset 0.7.1",
+ "pin-utils",
+ "static_assertions",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+dependencies = [
+ "hermit-abi 0.2.6",
  "libc",
 ]
 
@@ -809,15 +875,15 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.15.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
 [[package]]
 name = "openssl"
-version = "0.10.42"
+version = "0.10.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13"
+checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -847,9 +913,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.77"
+version = "0.9.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a"
+checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
 dependencies = [
  "autocfg",
  "cc",
@@ -860,9 +926,9 @@ dependencies = [
 
 [[package]]
 name = "os_str_bytes"
-version = "6.3.0"
+version = "6.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff"
+checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
 
 [[package]]
 name = "parking_lot"
@@ -945,12 +1011,12 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
 [[package]]
 name = "privdrop"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad5b1f7e40f628a2f8f90e40d3f313be83066cc61997fdcb96cade6abf7cee93"
+checksum = "81ed9e5437d82d5f2cde999a21571474c5f09b3d76e33eab94bf0e8e42a4fd96"
 dependencies = [
  "libc",
- "nix 0.24.2",
+ "nix 0.26.2",
 ]
 
 [[package]]
@@ -979,9 +1045,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.46"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
 dependencies = [
  "unicode-ident",
 ]
@@ -992,7 +1058,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "prost-derive",
 ]
 
@@ -1002,7 +1068,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "heck 0.3.3",
  "itertools",
  "log",
@@ -1033,7 +1099,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "prost",
 ]
 
@@ -1089,9 +1155,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
@@ -1107,9 +1173,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.7.0"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1133,12 +1199,12 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.12"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc"
+checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"
 dependencies = [
- "base64",
- "bytes 1.2.1",
+ "base64 0.21.0",
+ "bytes 1.4.0",
  "encoding_rs",
  "futures-core",
  "futures-util",
@@ -1190,9 +1256,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.20.6"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033"
+checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
 dependencies = [
  "log",
  "ring",
@@ -1202,24 +1268,24 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55"
+checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
 dependencies = [
- "base64",
+ "base64 0.21.0",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70"
 
 [[package]]
 name = "ryu"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
 
 [[package]]
 name = "safe-path"
@@ -1230,11 +1296,10 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.20"
+version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
+checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"
 dependencies = [
- "lazy_static",
  "windows-sys",
 ]
 
@@ -1256,9 +1321,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.7.0"
+version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c"
+checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"
 dependencies = [
  "bitflags",
  "core-foundation",
@@ -1269,9 +1334,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.6.1"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556"
+checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -1279,24 +1344,24 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.14"
+version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
+checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
 
 [[package]]
 name = "serde"
-version = "1.0.149"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.149"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1305,9 +1370,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.86"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
 dependencies = [
  "itoa",
  "ryu",
@@ -1430,6 +1495,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1448,7 +1519,7 @@ version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
 dependencies = [
- "heck 0.4.0",
+ "heck 0.4.1",
  "proc-macro2",
  "quote",
  "rustversion",
@@ -1457,9 +1528,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.105"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1488,9 +1559,9 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]
@@ -1499,29 +1570,29 @@ dependencies = [
 name = "test-utils"
 version = "0.1.0"
 dependencies = [
- "nix 0.24.2",
+ "nix 0.24.3",
 ]
 
 [[package]]
 name = "textwrap"
-version = "0.15.1"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16"
+checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
 
 [[package]]
 name = "thiserror"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1530,10 +1601,11 @@ dependencies = [
 
 [[package]]
 name = "thread_local"
-version = "1.1.4"
+version = "1.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
 dependencies = [
+ "cfg-if",
  "once_cell",
 ]
 
@@ -1575,18 +1647,18 @@ dependencies = [
 
 [[package]]
 name = "tinyvec_macros"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.21.2"
+version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
+checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
 dependencies = [
  "autocfg",
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "libc",
  "memchr",
  "mio",
@@ -1594,7 +1666,7 @@ dependencies = [
  "pin-project-lite",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys",
 ]
 
 [[package]]
@@ -1610,9 +1682,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-native-tls"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
 dependencies = [
  "native-tls",
  "tokio",
@@ -1631,11 +1703,11 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.4"
+version = "0.7.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
+checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2"
 dependencies = [
- "bytes 1.2.1",
+ "bytes 1.4.0",
  "futures-core",
  "futures-sink",
  "pin-project-lite",
@@ -1645,9 +1717,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-vsock"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d2ad075b54bbb450ae2e3770211d7954362a488fcd386085c9fbb6d787ade8b"
+checksum = "9b33556828911d16e24d8b5d336446b0bf6b4b9bfda52cbdc2fa35b7a2862ebc"
 dependencies = [
  "bytes 0.4.12",
  "futures",
@@ -1658,9 +1730,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.5.10"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
 dependencies = [
  "serde",
 ]
@@ -1693,9 +1765,9 @@ dependencies = [
 
 [[package]]
 name = "try-lock"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
 [[package]]
 name = "ttrpc"
@@ -1745,15 +1817,15 @@ dependencies = [
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.8"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
 [[package]]
 name = "unicode-normalization"
@@ -1766,9 +1838,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.10.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
 
 [[package]]
 name = "untrusted"
@@ -1799,6 +1871,16 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "vmm-sys-util"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd64fe09d8e880e600c324e7d664760a17f56e9672b7495a86381b49e4f72f46"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
 [[package]]
 name = "vsock"
 version = "0.2.6"
@@ -1827,9 +1909,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.83"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -1837,9 +1919,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.83"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
 dependencies = [
  "bumpalo",
  "log",
@@ -1852,9 +1934,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -1864,9 +1946,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.83"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1874,9 +1956,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.83"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1887,15 +1969,15 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.83"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
 
 [[package]]
 name = "web-sys"
-version = "0.3.60"
+version = "0.3.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
+checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -1913,18 +1995,18 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.22.5"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be"
+checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
 dependencies = [
  "webpki",
 ]
 
 [[package]]
 name = "which"
-version = "4.3.0"
+version = "4.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b"
+checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
 dependencies = [
  "either",
  "libc",
@@ -1964,46 +2046,60 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-sys"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
+ "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
  "windows_i686_gnu",
  "windows_i686_msvc",
  "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
  "windows_x86_64_msvc",
 ]
 
 [[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
+name = "windows_aarch64_gnullvm"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
 [[package]]
 name = "winreg"
diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml
index d1f3c5637..664a19ebc 100644
--- a/src/tools/kata-ctl/Cargo.toml
+++ b/src/tools/kata-ctl/Cargo.toml
@@ -31,6 +31,14 @@ kata-types = { path = "../../libs/kata-types" }
 safe-path = { path = "../../libs/safe-path" }
 agent = { path = "../../runtime-rs/crates/agent"}
 serial_test = "0.5.1"
+vmm-sys-util = "0.11.0"
+epoll = "4.0.1"
+libc = "0.2.138"
+slog = "2.7.0"
+slog-scope = "4.4.0"
+hyper = "0.14.20"
+ttrpc = "0.6.0"
+tokio = "1.8.0"
 
 [target.'cfg(target_arch = "s390x")'.dependencies]
 reqwest = { version = "0.11", default-features = false, features = ["json", "blocking", "native-tls"] }
@@ -42,3 +50,4 @@ reqwest = { version = "0.11", default-features = false, features = ["json", "blo
 semver = "1.0.12"
 tempfile = "3.1.0"
 test-utils = { path = "../../libs/test-utils" }
+micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
diff --git a/src/tools/kata-ctl/src/args.rs b/src/tools/kata-ctl/src/args.rs
index f5a1cd4f2..ff489f82b 100644
--- a/src/tools/kata-ctl/src/args.rs
+++ b/src/tools/kata-ctl/src/args.rs
@@ -26,7 +26,7 @@ pub enum Commands {
     Env,
 
     /// Enter into guest VM by debug console
-    Exec,
+    Exec(ExecArguments),
 
     /// Manage VM factory
     Factory,
@@ -136,3 +136,12 @@ pub struct DirectVolResizeArgs {
     pub volume_path: String,
     pub resize_size: u64,
 }
+
+#[derive(Debug, Args)]
+pub struct ExecArguments {
+    /// pod sandbox ID.
+    pub sandbox_id: String,
+    #[clap(short = 'p', long = "kata-debug-port", default_value_t = 1026)]
+    /// kata debug console vport same as configuration, default is 1026.
+    pub vport: u32,
+}
diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs
index 8218a0f31..bda635515 100644
--- a/src/tools/kata-ctl/src/check.rs
+++ b/src/tools/kata-ctl/src/check.rs
@@ -69,7 +69,7 @@ pub fn get_cpu_flags(cpu_info: &str, cpu_flags_tag: &str) -> Result<String> {
     }
 
     if cpu_flags_tag.is_empty() {
-        return Err(anyhow!("cpu flags delimiter string is empty"))?;
+        return Err(anyhow!("cpu flags delimiter string is empty"));
     }
 
     let subcontents: Vec<&str> = cpu_info.split('\n').collect();
diff --git a/src/tools/kata-ctl/src/main.rs b/src/tools/kata-ctl/src/main.rs
index cb943e694..980a94771 100644
--- a/src/tools/kata-ctl/src/main.rs
+++ b/src/tools/kata-ctl/src/main.rs
@@ -17,9 +17,9 @@ use std::process::exit;
 use args::{Commands, KataCtlCli};
 
 use ops::check_ops::{
-    handle_check, handle_env, handle_exec, handle_factory, handle_iptables, handle_metrics,
-    handle_version,
+    handle_check, handle_env, handle_factory, handle_iptables, handle_metrics, handle_version,
 };
+use ops::exec_ops::handle_exec;
 use ops::volume_ops::handle_direct_volume;
 
 fn real_main() -> Result<()> {
@@ -28,8 +28,8 @@ fn real_main() -> Result<()> {
     match args.command {
         Commands::Check(args) => handle_check(args),
         Commands::DirectVolume(args) => handle_direct_volume(args),
+        Commands::Exec(args) => handle_exec(args),
         Commands::Env => handle_env(),
-        Commands::Exec => handle_exec(),
         Commands::Factory => handle_factory(),
         Commands::Iptables(args) => handle_iptables(args),
         Commands::Metrics(args) => handle_metrics(args),
diff --git a/src/tools/kata-ctl/src/ops.rs b/src/tools/kata-ctl/src/ops.rs
index e33539bce..d5d4fe162 100644
--- a/src/tools/kata-ctl/src/ops.rs
+++ b/src/tools/kata-ctl/src/ops.rs
@@ -4,5 +4,6 @@
 //
 
 pub mod check_ops;
+pub mod exec_ops;
 pub mod version;
 pub mod volume_ops;
diff --git a/src/tools/kata-ctl/src/ops/check_ops.rs b/src/tools/kata-ctl/src/ops/check_ops.rs
index bf3db94e3..fa3aa688b 100644
--- a/src/tools/kata-ctl/src/ops/check_ops.rs
+++ b/src/tools/kata-ctl/src/ops/check_ops.rs
@@ -108,10 +108,6 @@ pub fn handle_env() -> Result<()> {
     Ok(())
 }
 
-pub fn handle_exec() -> Result<()> {
-    Ok(())
-}
-
 pub fn handle_factory() -> Result<()> {
     Ok(())
 }
diff --git a/src/tools/kata-ctl/src/ops/exec_ops.rs b/src/tools/kata-ctl/src/ops/exec_ops.rs
new file mode 100644
index 000000000..2a6b970aa
--- /dev/null
+++ b/src/tools/kata-ctl/src/ops/exec_ops.rs
@@ -0,0 +1,444 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Description:
+// Implementation of entering into guest VM by debug console.
+// Ensure that `kata-debug-port` is consistent with the port
+// set in the configuration.
+
+use std::{
+    io::{self, BufRead, BufReader, Read, Write},
+    os::unix::{
+        io::{AsRawFd, FromRawFd, RawFd},
+        net::UnixStream,
+    },
+    time::Duration,
+};
+
+use anyhow::{anyhow, Context};
+use nix::sys::socket::{connect, socket, AddressFamily, SockFlag, SockType, VsockAddr};
+use reqwest::StatusCode;
+use slog::debug;
+use vmm_sys_util::terminal::Terminal;
+
+use crate::args::ExecArguments;
+use shim_interface::shim_mgmt::{client::MgmtClient, AGENT_URL};
+
+const CMD_CONNECT: &str = "CONNECT";
+const CMD_OK: &str = "OK";
+const SCHEME_VSOCK: &str = "VSOCK";
+const SCHEME_HYBRID_VSOCK: &str = "HVSOCK";
+
+const EPOLL_EVENTS_LEN: usize = 16;
+const KATA_AGENT_VSOCK_TIMEOUT: u64 = 5;
+const TIMEOUT: Duration = Duration::from_millis(2000);
+
+type Result<T> = std::result::Result<T, Error>;
+
+// Convenience macro to obtain the scope logger
+#[macro_export]
+macro_rules! sl {
+    () => {
+        slog_scope::logger()
+    };
+}
+
+#[derive(Debug)]
+pub enum Error {
+    EpollWait(io::Error),
+    EpollCreate(io::Error),
+    EpollAdd(io::Error),
+    SocketWrite(io::Error),
+    StdioErr(io::Error),
+}
+
+#[derive(Debug, PartialEq)]
+enum EpollDispatch {
+    Stdin,
+    ServerSock,
+}
+
+struct EpollContext {
+    epoll_raw_fd: RawFd,
+    stdin_index: u64,
+    dispatch_table: Vec<EpollDispatch>,
+    stdin_handle: io::Stdin,
+    debug_console_sock: Option<UnixStream>,
+}
+
+impl EpollContext {
+    fn new() -> Result<Self> {
+        let epoll_raw_fd = epoll::create(true).map_err(Error::EpollCreate)?;
+        let dispatch_table = Vec::new();
+        let stdin_index = 0;
+
+        Ok(EpollContext {
+            epoll_raw_fd,
+            stdin_index,
+            dispatch_table,
+            stdin_handle: io::stdin(),
+            debug_console_sock: None,
+        })
+    }
+
+    fn init_debug_console_sock(&mut self, sock: UnixStream) -> Result<()> {
+        let dispatch_index = self.dispatch_table.len() as u64;
+        epoll::ctl(
+            self.epoll_raw_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            sock.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, dispatch_index),
+        )
+        .map_err(Error::EpollAdd)?;
+
+        self.dispatch_table.push(EpollDispatch::ServerSock);
+        self.debug_console_sock = Some(sock);
+
+        Ok(())
+    }
+
+    fn enable_stdin_event(&mut self) -> Result<()> {
+        let stdin_index = self.dispatch_table.len() as u64;
+        epoll::ctl(
+            self.epoll_raw_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            libc::STDIN_FILENO,
+            epoll::Event::new(epoll::Events::EPOLLIN, stdin_index),
+        )
+        .map_err(Error::EpollAdd)?;
+
+        self.stdin_index = stdin_index;
+        self.dispatch_table.push(EpollDispatch::Stdin);
+
+        Ok(())
+    }
+
+    fn do_exit(&self) {
+        self.stdin_handle
+            .lock()
+            .set_canon_mode()
+            .expect("Fail to set stdin to RAW mode");
+    }
+
+    fn do_process_handler(&mut self) -> Result<()> {
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+
+        let epoll_raw_fd = self.epoll_raw_fd;
+        let debug_console_sock = self.debug_console_sock.as_mut().unwrap();
+
+        loop {
+            let num_events =
+                epoll::wait(epoll_raw_fd, -1, &mut events[..]).map_err(Error::EpollWait)?;
+
+            for event in events.iter().take(num_events) {
+                let dispatch_index = event.data as usize;
+                match self.dispatch_table[dispatch_index] {
+                    EpollDispatch::Stdin => {
+                        let mut out = [0u8; 128];
+                        let stdin_lock = self.stdin_handle.lock();
+                        match stdin_lock.read_raw(&mut out[..]) {
+                            Ok(0) => {
+                                return Ok(());
+                            }
+                            Err(e) => {
+                                println!("error with errno {:?} while reading stdin", e);
+                                return Ok(());
+                            }
+                            Ok(count) => {
+                                debug_console_sock
+                                    .write(&out[..count])
+                                    .map_err(Error::SocketWrite)?;
+                            }
+                        }
+                    }
+                    EpollDispatch::ServerSock => {
+                        let mut out = [0u8; 128];
+                        match debug_console_sock.read(&mut out[..]) {
+                            Ok(0) => {
+                                return Ok(());
+                            }
+                            Err(e) => {
+                                println!("error with errno {:?} while reading server", e);
+                                return Ok(());
+                            }
+                            Ok(count) => {
+                                io::stdout()
+                                    .write_all(&out[..count])
+                                    .map_err(Error::StdioErr)?;
+                                io::stdout().flush().map_err(Error::StdioErr)?;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+trait SockHandler {
+    fn setup_sock(&self) -> anyhow::Result<UnixStream>;
+}
+
+struct VsockConfig {
+    sock_cid: u32,
+    sock_port: u32,
+}
+
+impl VsockConfig {
+    fn new(sock_cid: u32, sock_port: u32) -> VsockConfig {
+        VsockConfig {
+            sock_cid,
+            sock_port,
+        }
+    }
+}
+
+impl SockHandler for VsockConfig {
+    fn setup_sock(&self) -> anyhow::Result<UnixStream> {
+        let sock_addr = VsockAddr::new(self.sock_cid, self.sock_port);
+
+        // Create socket fd
+        let vsock_fd = socket(
+            AddressFamily::Vsock,
+            SockType::Stream,
+            SockFlag::SOCK_CLOEXEC,
+            None,
+        )
+        .context("create vsock socket")?;
+
+        // Wrap the socket fd in UnixStream, so that it is closed
+        // when anything fails.
+        let stream = unsafe { UnixStream::from_raw_fd(vsock_fd) };
+        // Connect the socket to vsock server.
+        connect(stream.as_raw_fd(), &sock_addr)
+            .with_context(|| format!("failed to connect to server {:?}", &sock_addr))?;
+
+        Ok(stream)
+    }
+}
+
+struct HvsockConfig {
+    sock_addr: String,
+    sock_port: u32,
+}
+
+impl HvsockConfig {
+    fn new(sock_addr: String, sock_port: u32) -> Self {
+        HvsockConfig {
+            sock_addr,
+            sock_port,
+        }
+    }
+}
+
+impl SockHandler for HvsockConfig {
+    fn setup_sock(&self) -> anyhow::Result<UnixStream> {
+        let mut stream = match UnixStream::connect(self.sock_addr.clone()) {
+            Ok(s) => s,
+            Err(e) => return Err(anyhow!(e).context("failed to create UNIX Stream socket")),
+        };
+
+        // Ensure the Unix Stream directly connects to the real VSOCK server which
+        // the Kata agent is listening to in the VM.
+        {
+            let test_msg = format!("{} {}\n", CMD_CONNECT, self.sock_port);
+
+            stream.set_read_timeout(Some(Duration::new(KATA_AGENT_VSOCK_TIMEOUT, 0)))?;
+            stream.set_write_timeout(Some(Duration::new(KATA_AGENT_VSOCK_TIMEOUT, 0)))?;
+
+            stream.write_all(test_msg.as_bytes())?;
+            // Now, see if we get the expected response
+            let stream_reader = stream.try_clone()?;
+            let mut reader = BufReader::new(&stream_reader);
+            let mut msg = String::new();
+
+            reader.read_line(&mut msg)?;
+            if msg.is_empty() {
+                return Err(anyhow!(
+                    "stream reader get message is empty with port: {:?}",
+                    self.sock_port
+                ));
+            }
+
+            // Expected response message returned was successful.
+            if msg.starts_with(CMD_OK) {
+                let response = msg
+                    .strip_prefix(CMD_OK)
+                    .ok_or(format!("invalid response: {:?}", msg))
+                    .map_err(|e| anyhow!(e))?
+                    .trim();
+                debug!(sl!(), "Hybrid Vsock host-side port: {:?}", response);
+                // Unset the timeout in order to turn the sokect to bloking mode.
+                stream.set_read_timeout(None)?;
+                stream.set_write_timeout(None)?;
+            } else {
+                return Err(anyhow!(
+                    "failed to setup Hybrid Vsock connection: {:?}",
+                    msg
+                ));
+            }
+        }
+
+        Ok(stream)
+    }
+}
+
+fn setup_client(server_url: String, dbg_console_port: u32) -> anyhow::Result<UnixStream> {
+    // server address format: scheme://[cid|/x/domain.sock]:port
+    let url_fields: Vec<&str> = server_url.split("://").collect();
+    if url_fields.len() != 2 {
+        return Err(anyhow!("invalid URI"));
+    }
+
+    let scheme = url_fields[0].to_uppercase();
+    let sock_addr: Vec<&str> = url_fields[1].split(':').collect();
+    if sock_addr.len() != 2 {
+        return Err(anyhow!("invalid VSOCK server address URI"));
+    }
+
+    match scheme.as_str() {
+        // Hybrid Vsock: hvsock://<path>:<port>.
+        // Example: "hvsock:///x/y/z/kata.hvsock:port"
+        // Firecracker/Dragonball/CLH implements the hybrid vsock device model.
+        SCHEME_HYBRID_VSOCK => {
+            let hvsock_path = sock_addr[0].to_string();
+            if hvsock_path.is_empty() {
+                return Err(anyhow!("hvsock path cannot be empty"));
+            }
+
+            let hvsock = HvsockConfig::new(hvsock_path, dbg_console_port);
+            hvsock.setup_sock().context("set up hvsock")
+        }
+        // Vsock: vsock://<cid>:<port>
+        // Example: "vsock://31513974:1024"
+        // Qemu using the Vsock device model.
+        SCHEME_VSOCK => {
+            let sock_cid: u32 = match sock_addr[0] {
+                "-1" | "" => libc::VMADDR_CID_ANY,
+                _ => match sock_addr[0].parse::<u32>() {
+                    Ok(cid) => cid,
+                    Err(e) => return Err(anyhow!("vsock addr CID is INVALID: {:?}", e)),
+                },
+            };
+
+            let vsock = VsockConfig::new(sock_cid, dbg_console_port);
+            vsock.setup_sock().context("set up vsock")
+        }
+        // Others will be INVALID URI.
+        _ => {
+            return Err(anyhow!("invalid URI scheme: {:?}", scheme));
+        }
+    }
+}
+
+async fn get_agent_socket(sandbox_id: &str) -> anyhow::Result<String> {
+    let shim_client = MgmtClient::new(sandbox_id, Some(TIMEOUT))?;
+
+    // get agent sock from body when status code is OK.
+    let response = shim_client.get(AGENT_URL).await?;
+    let status = response.status();
+    if status != StatusCode::OK {
+        return Err(anyhow!("shim client get connection failed: {:?} ", status));
+    }
+
+    let body = hyper::body::to_bytes(response.into_body()).await?;
+    let agent_sock = String::from_utf8(body.to_vec())?;
+
+    Ok(agent_sock)
+}
+
+fn get_server_socket(sandbox_id: &str) -> anyhow::Result<String> {
+    let server_url = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?
+        .block_on(get_agent_socket(sandbox_id))
+        .context("get connection vsock")?;
+
+    Ok(server_url)
+}
+
+fn do_run_exec(sandbox_id: &str, dbg_console_vport: u32) -> anyhow::Result<()> {
+    // sandbox_id MUST be a long ID.
+    let server_url = get_server_socket(sandbox_id).context("get debug console socket URL")?;
+    if server_url.is_empty() {
+        return Err(anyhow!("server url is empty."));
+    }
+    let sock_stream = setup_client(server_url, dbg_console_vport)?;
+
+    let mut epoll_context = EpollContext::new().expect("create epoll context");
+    epoll_context
+        .enable_stdin_event()
+        .expect("enable stdin event");
+    epoll_context
+        .init_debug_console_sock(sock_stream)
+        .expect("enable debug console sock");
+
+    let stdin_handle = io::stdin();
+    stdin_handle.lock().set_raw_mode().expect("set raw mode");
+
+    epoll_context
+        .do_process_handler()
+        .expect("do process handler");
+    epoll_context.do_exit();
+
+    Ok(())
+}
+
+// kata-ctl handle exec command starts here.
+pub fn handle_exec(exec_args: ExecArguments) -> anyhow::Result<()> {
+    do_run_exec(exec_args.sandbox_id.as_str(), exec_args.vport)?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use micro_http::HttpServer;
+
+    #[test]
+    fn test_epoll_context_methods() {
+        let kata_hybrid_addr = "/tmp/kata_hybrid_vsock01.hvsock";
+        std::fs::remove_file(kata_hybrid_addr).unwrap_or_default();
+        let mut server = HttpServer::new(kata_hybrid_addr).unwrap();
+        server.start_server().unwrap();
+        let sock_addr: UnixStream = UnixStream::connect(kata_hybrid_addr).unwrap();
+        let mut epoll_ctx = EpollContext::new().expect("epoll context");
+        epoll_ctx
+            .init_debug_console_sock(sock_addr)
+            .expect("enable debug console sock");
+        assert_eq!(epoll_ctx.stdin_index, 0);
+        assert!(epoll_ctx.debug_console_sock.is_some());
+        assert_eq!(epoll_ctx.dispatch_table[0], EpollDispatch::ServerSock);
+        assert_eq!(epoll_ctx.dispatch_table.len(), 1);
+
+        epoll_ctx.enable_stdin_event().expect("enable stdin event");
+        assert_eq!(epoll_ctx.stdin_index, 1);
+        assert_eq!(epoll_ctx.dispatch_table[1], EpollDispatch::Stdin);
+        assert_eq!(epoll_ctx.dispatch_table.len(), 2);
+        std::fs::remove_file(kata_hybrid_addr).unwrap_or_default();
+    }
+
+    #[test]
+    fn test_setup_hvsock_failed() {
+        let kata_hybrid_addr = "/tmp/kata_hybrid_vsock02.hvsock";
+        let hybrid_sock_addr = "hvsock:///tmp/kata_hybrid_vsock02.hvsock:1024";
+        std::fs::remove_file(kata_hybrid_addr).unwrap_or_default();
+        let dbg_console_port: u32 = 1026;
+        let mut server = HttpServer::new(kata_hybrid_addr).unwrap();
+        server.start_server().unwrap();
+
+        let stream = setup_client(hybrid_sock_addr.to_string(), dbg_console_port);
+        assert!(stream.is_err());
+        std::fs::remove_file(kata_hybrid_addr).unwrap_or_default();
+    }
+
+    #[test]
+    fn test_setup_vsock_client_failed() {
+        let hybrid_sock_addr = "hvsock://8:1024";
+        let dbg_console_port: u32 = 1026;
+        let stream = setup_client(hybrid_sock_addr.to_string(), dbg_console_port);
+        assert!(stream.is_err());
+    }
+}

From a96ba992391865f38695dfa970af0ac49b5421a9 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 16 Feb 2023 17:27:58 -0800
Subject: [PATCH 04/76] actions: Use `git-diff` to get changes in kernel dir

Use `git-diff` instead of legacy `git-whatchanged` to get
differences in the packaging/kernel directory. This also fixes
a bug by grepping for the kernel directory in the output of the
git command.

Fixes: #6210

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 .github/workflows/static-checks.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml
index 0cb9d72d3..8d9858500 100644
--- a/.github/workflows/static-checks.yaml
+++ b/.github/workflows/static-checks.yaml
@@ -43,8 +43,7 @@ jobs:
         kernel_dir="tools/packaging/kernel/"
         kernel_version_file="${kernel_dir}kata_config_version"
         modified_files=$(git diff --name-only origin/main..HEAD)
-        result=$(git whatchanged origin/main..HEAD "${kernel_dir}" >>"/dev/null")
-        if git whatchanged origin/main..HEAD "${kernel_dir}" >>"/dev/null"; then
+        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
           echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
           if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
             echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)

From 1047840cf81a0e70717ab244de728c236ceb6ee5 Mon Sep 17 00:00:00 2001
From: Gabe Venberg <gabevenberg@gmail.com>
Date: Thu, 16 Feb 2023 23:00:19 -0600
Subject: [PATCH 05/76] utils: always check some dependencies.

Every dependency in check_deps is used inside the script (apart from
git, which may be a historical artifact), and therefore should be
checked even when the -f option is passed to the script. Simply changed
at what point check_deps is called in order to always run it.

Fixes #6302.

Signed-off-by: Gabe Venberg <gabevenberg@gmail.com>
---
 utils/kata-manager.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh
index 1cb65145f..d09a2e67b 100755
--- a/utils/kata-manager.sh
+++ b/utils/kata-manager.sh
@@ -327,7 +327,7 @@ check_deps()
 		debian|ubuntu) sudo apt-get -y install $packages ;;
 		fedora) sudo dnf -y install $packages ;;
 		opensuse*|sles) sudo zypper install -y $packages ;;
-		*) die "Unsupported distro: $ID"
+		*) die "Unsupported distro: $ID, install $packages manually and re-run"
 	esac
 }
 
@@ -343,10 +343,12 @@ setup()
 
 	source /etc/os-release || source /usr/lib/os-release
 
+	#these dependencies are needed inside this script, and should be checked regardless of the -f option.
+	check_deps
+
 	[ "$force" = "true" ] && return 0
 
 	pre_checks
-	check_deps
 }
 
 # Download the requested version of the specified project.

From 6c05e5c67a0bdab86db52874f05e9863ba552794 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 15:11:16 +0200
Subject: [PATCH 06/76] packaging: Add and export BUILDER_REGISTRY
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

BUILD_REGISTRY, which points to quay.io/kata-containers/builder, will be
used for storing the builder images used to build the artefacts via the
kata-deploy scripts.

The plan is to tag, whenever it's possible and makes sense, images like:
* ${BUILDER_REGISTRY}:${component}-${unique_identifier}

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/scripts/lib.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh
index d53159a8c..34fb29250 100644
--- a/tools/packaging/scripts/lib.sh
+++ b/tools/packaging/scripts/lib.sh
@@ -8,6 +8,7 @@
 export GOPATH=${GOPATH:-${HOME}/go}
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
+export BUILDER_REGISTRY="quay.io/kata-containers/builders"
 
 this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 

From e69a6f57493d286aca4711248263d17c0dacfca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:20:52 +0200
Subject: [PATCH 07/76] packaging: Add get_last_modification()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's add a function to get the hash of the last commit modifying a
specific file.

This will help to avoid writing `git rev-list ...` into every single
build script used by the kata-deploy.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/scripts/lib.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh
index 34fb29250..616c17b96 100644
--- a/tools/packaging/scripts/lib.sh
+++ b/tools/packaging/scripts/lib.sh
@@ -98,3 +98,18 @@ get_kata_hash() {
 	ref=$2
 	git ls-remote --heads --tags "https://github.com/${project}/${repo}.git" | grep "${ref}" | awk '{print $1}'
 }
+
+# $1 - Repo's root dir
+# $2 - The file we're looking for the last modification
+get_last_modification() {
+	local repo_root_dir="${1}"
+	local file="${2}"
+
+	# This is a workaround needed for when running this code on Jenkins
+	git config --global --add safe.directory ${repo_root_dir} &> /dev/null
+
+	dirty=""
+	[ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty"
+
+	echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}"
+}

From 869827d77f62b8337a84228db740e1244afe47ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:38:38 +0200
Subject: [PATCH 08/76] packaging: Add push_to_registry()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This function will push a specific tag to a registry, whenever the
PUSH_TO_REGISTRY environment variable is set, otherwise it's a no-op.

This will be used in the future to avoid replicating that logic in every
builder used by the kata-deploy scripts.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../kata-deploy-binaries-in-docker.sh            |  4 +++-
 tools/packaging/scripts/lib.sh                   | 16 ++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
index 164dbf864..412f1e87d 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
@@ -47,8 +47,10 @@ docker build -q -t build-kata-deploy \
 docker run \
 	-v $HOME/.docker:/root/.docker \
 	-v /var/run/docker.sock:/var/run/docker.sock \
+	-v "${kata_dir}:${kata_dir}" \
 	--env CI="${CI:-}" \
-	--env USER=${USER} -v "${kata_dir}:${kata_dir}" \
+	--env USER=${USER} \
+	--env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" \
 	--rm \
 	-w ${script_dir} \
 	build-kata-deploy "${kata_deploy_create}" $@
diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh
index 616c17b96..5ac31d0e3 100644
--- a/tools/packaging/scripts/lib.sh
+++ b/tools/packaging/scripts/lib.sh
@@ -9,6 +9,7 @@ export GOPATH=${GOPATH:-${HOME}/go}
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
 export BUILDER_REGISTRY="quay.io/kata-containers/builders"
+export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}"
 
 this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
@@ -113,3 +114,18 @@ get_last_modification() {
 
 	echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}"
 }
+
+# $1 - The tag to be pushed to the registry
+# $2 - "yes" to use sudo, "no" otherwise
+push_to_registry() {
+	local tag="${1}"
+	local use_sudo="${2:-"yes"}"
+
+	if [ "${PUSH_TO_REGISTRY}" == "yes" ]; then
+		if [ "${use_sudo}" == "yes" ]; then
+			sudo docker push ${tag}
+		else
+			docker push ${tag}
+		fi
+	fi
+}

From b9b23112bf6fb23f550b2d1c7d53dd9a4987459b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:04:11 +0200
Subject: [PATCH 09/76] packaging: Use existing image to build the kernel
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's first try to pull a pre-existing image, instead of building our
own, to be used as a builder image for the kernel.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/kernel/build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh
index 1aa6c1cd4..7e6f9a66e 100755
--- a/tools/packaging/static-build/kernel/build.sh
+++ b/tools/packaging/static-build/kernel/build.sh
@@ -12,12 +12,13 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
 
+source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="kata-kernel-builder"
+container_image="${BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(umame -m)"
 
-sudo docker build -t "${container_image}" "${script_dir}"
+sudo docker pull ${container_image} || sudo docker build -t "${container_image}" "${script_dir}"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From 6c3c771a52a66df1c31813339b87484aea1b028f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:08:05 +0200
Subject: [PATCH 10/76] packaging: Add infra to push the kernel builder image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's add the needed infra for only building and pushing the kernel
builder image to the Kata Containers' quay.io registry.

Fixes: #5476

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/kernel/build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh
index 7e6f9a66e..c04ee0c8e 100755
--- a/tools/packaging/static-build/kernel/build.sh
+++ b/tools/packaging/static-build/kernel/build.sh
@@ -18,7 +18,10 @@ DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
 container_image="${BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(umame -m)"
 
-sudo docker pull ${container_image} || sudo docker build -t "${container_image}" "${script_dir}"
+sudo docker pull ${container_image} || \
+	(sudo docker build -t "${container_image}" "${script_dir}" && \
+	 # No-op unless PUSH_TO_REGISTRY is exported as "yes"
+	 push_to_registry "${container_image}")
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From e84af6a6205e64dde194c91e993d7ed1dc5c800d Mon Sep 17 00:00:00 2001
From: Amulyam24 <amulmek1@in.ibm.com>
Date: Fri, 17 Feb 2023 16:18:58 +0530
Subject: [PATCH 11/76] virtiofsd: update to a valid path on ppc64le

Currently the symbolic link for virtiofsd which is used as
a valid path is not updated on every CI run. Fix it by
using the actual path of installation.

Fixes: #6311

Signed-off-by: Amulyam24 <amulmek1@in.ibm.com>
---
 src/runtime/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index 99dde7e2b..ab9611968 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -197,7 +197,7 @@ DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
 ifeq ($(ARCH),ppc64le)
-DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd
+DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/qemu/virtiofsd
 endif
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 # Default DAX mapping cache size in MiB

From 24767d82aa5bd2db8264a0973054dac3b541d302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:57:02 +0200
Subject: [PATCH 12/76] packaging: Use existing image to build the shim-v2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's try to pull a pre-existing image, instead of building our own, to
be used as a builder for the shim-v2.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/shim-v2/build.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh
index 6cbb0a41b..a2e4630bf 100755
--- a/tools/packaging/static-build/shim-v2/build.sh
+++ b/tools/packaging/static-build/shim-v2/build.sh
@@ -10,7 +10,8 @@ set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
-readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
+
+source "${script_dir}/../../scripts/lib.sh"
 
 VMM_CONFIGS="qemu fc"
 
@@ -19,9 +20,14 @@ RUST_VERSION=${RUST_VERSION}
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="shim-v2-builder"
+container_image="${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
 
-sudo docker build  --build-arg GO_VERSION="${GO_VERSION}"  --build-arg RUST_VERSION="${RUST_VERSION}" -t "${container_image}" "${script_dir}"
+sudo docker pull ${container_image} || \
+       	sudo docker build  \
+		--build-arg GO_VERSION="${GO_VERSION}" \
+		--build-arg RUST_VERSION="${RUST_VERSION}" \
+		-t "${container_image}" \
+		"${script_dir}"
 
 arch=$(uname -m)
 if [ ${arch} = "ppc64le" ]; then

From 7ee00d8e5764acde38ff332d62370e38ae05c6c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:59:13 +0200
Subject: [PATCH 13/76] packaging: Add infra to push the shim-v2 builder image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's add the needed infra for only building and pushing the shim-v2
builder image to the Kata Containers' quay.io registry.

Fixes: #5478

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/shim-v2/build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh
index a2e4630bf..a40f87f25 100755
--- a/tools/packaging/static-build/shim-v2/build.sh
+++ b/tools/packaging/static-build/shim-v2/build.sh
@@ -23,11 +23,12 @@ PREFIX=${PREFIX:-/opt/kata}
 container_image="${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
 
 sudo docker pull ${container_image} || \
-       	sudo docker build  \
+       	(sudo docker build  \
 		--build-arg GO_VERSION="${GO_VERSION}" \
 		--build-arg RUST_VERSION="${RUST_VERSION}" \
 		-t "${container_image}" \
-		"${script_dir}"
+		"${script_dir}" && \
+	 push_to_registry "${container_image}")
 
 arch=$(uname -m)
 if [ ${arch} = "ppc64le" ]; then

From 8c227e247185aa2a4496d26c6bc2d051a83a955a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 17:20:27 +0200
Subject: [PATCH 14/76] virtiofsd: Pass the expected toolchain to the build
 container
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's ensure we're building virtiofsd with a specific toolchain that's
known to not cause any issues, instead of always using the latest one.

On each bump of the virtiofsd, we'll make sure to adjust this according
to what's been used by the virtiofsd community.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/virtiofsd/build.sh        | 4 ++++
 tools/packaging/static-build/virtiofsd/gnu/Dockerfile  | 3 ++-
 tools/packaging/static-build/virtiofsd/musl/Dockerfile | 3 ++-
 versions.yaml                                          | 1 +
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh
index 64441d2ae..ec9d7e248 100755
--- a/tools/packaging/static-build/virtiofsd/build.sh
+++ b/tools/packaging/static-build/virtiofsd/build.sh
@@ -20,15 +20,18 @@ container_image="kata-virtiofsd-builder"
 kata_version="${kata_version:-}"
 virtiofsd_repo="${virtiofsd_repo:-}"
 virtiofsd_version="${virtiofsd_version:-}"
+virtiofsd_toolchain="${virtiofsd_toolchain:-}"
 virtiofsd_zip="${virtiofsd_zip:-}"
 package_output_dir="${package_output_dir:-}"
 
 [ -n "${virtiofsd_repo}" ] || virtiofsd_repo=$(get_from_kata_deps "externals.virtiofsd.url")
 [ -n "${virtiofsd_version}" ] || virtiofsd_version=$(get_from_kata_deps "externals.virtiofsd.version")
+[ -n "${virtiofsd_toolchain}" ] || virtiofsd_toolchain=$(get_from_kata_deps "externals.virtiofsd.toolchain")
 [ -n "${virtiofsd_zip}" ] || virtiofsd_zip=$(get_from_kata_deps "externals.virtiofsd.meta.binary")
 
 [ -n "${virtiofsd_repo}" ] || die "Failed to get virtiofsd repo"
 [ -n "${virtiofsd_version}" ] || die "Failed to get virtiofsd version or commit"
+[ -n "${virtiofsd_toolchain}" ] || die "Failed to get the rust toolchain to build virtiofsd"
 [ -n "${virtiofsd_zip}" ] || die "Failed to get virtiofsd binary URL"
 
 ARCH=$(uname -m)
@@ -48,6 +51,7 @@ case ${ARCH} in
 esac
 
 sudo docker build \
+	--build-arg RUST_TOOLCHAIN="${virtiofsd_toolchain}" \
 	-t "${container_image}" "${script_dir}/${libc}"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
diff --git a/tools/packaging/static-build/virtiofsd/gnu/Dockerfile b/tools/packaging/static-build/virtiofsd/gnu/Dockerfile
index c214dfc41..c10b8db49 100644
--- a/tools/packaging/static-build/virtiofsd/gnu/Dockerfile
+++ b/tools/packaging/static-build/virtiofsd/gnu/Dockerfile
@@ -4,6 +4,7 @@
 
 FROM ubuntu:20.04
 ENV DEBIAN_FRONTEND=noninteractive
+ARG RUST_TOOLCHAIN
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN apt-get update && \
@@ -16,4 +17,4 @@ RUN apt-get update && \
         libseccomp-dev \
         unzip && \
     apt-get clean && rm -rf /var/lib/lists/ && \
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_TOOLCHAIN}
diff --git a/tools/packaging/static-build/virtiofsd/musl/Dockerfile b/tools/packaging/static-build/virtiofsd/musl/Dockerfile
index 9b9bb93b9..1236010e0 100644
--- a/tools/packaging/static-build/virtiofsd/musl/Dockerfile
+++ b/tools/packaging/static-build/virtiofsd/musl/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 FROM alpine:3.16.2
+ARG RUST_TOOLCHAIN
 
 SHELL ["/bin/ash", "-o", "pipefail", "-c"]
 RUN apk --no-cache add \
@@ -13,4 +14,4 @@ RUN apk --no-cache add \
         libcap-ng-static \
         libseccomp-static \
         musl-dev && \
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_TOOLCHAIN}
diff --git a/versions.yaml b/versions.yaml
index 5f74ee8ba..36c0bb603 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -291,6 +291,7 @@ externals:
     description: "vhost-user virtio-fs device backend written in Rust"
     url: "https://gitlab.com/virtio-fs/virtiofsd"
     version: "v1.3.0"
+    toolchain: "1.62.0"
     meta:
       # From https://gitlab.com/virtio-fs/virtiofsd/-/releases/v1.3.0,
       # this is the link labelled virtiofsd-v1.3.0.zip

From 7c93428a1889a5a5e9fd63031dd2c91e408d4bee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 17:12:18 +0200
Subject: [PATCH 15/76] packaging: Use existing image to build virtiofsd
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's first try to pull a pre-existing image, instead of building our
own, to be used as a builder image for the virtiofsd.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/virtiofsd/build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh
index ec9d7e248..42e76a514 100755
--- a/tools/packaging/static-build/virtiofsd/build.sh
+++ b/tools/packaging/static-build/virtiofsd/build.sh
@@ -16,7 +16,6 @@ source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="kata-virtiofsd-builder"
 kata_version="${kata_version:-}"
 virtiofsd_repo="${virtiofsd_repo:-}"
 virtiofsd_version="${virtiofsd_version:-}"
@@ -50,7 +49,9 @@ case ${ARCH} in
 		;;
 esac
 
-sudo docker build \
+container_image="${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(umame -m)"
+
+sudo docker pull ${container_image} || sudo docker build \
 	--build-arg RUST_TOOLCHAIN="${virtiofsd_toolchain}" \
 	-t "${container_image}" "${script_dir}/${libc}"
 

From d0181bb26261b50e1d4a2ddc2526fd80fea1034a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 17:28:52 +0200
Subject: [PATCH 16/76] packaging: Add infra to push the virtiofsd builder
 image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's add the needed infra for only building and pushing the virtiofsd
builder image to the Kata Containers' quay.io registry.

Fixes: #5480

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/virtiofsd/build.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh
index 42e76a514..d3f8dca07 100755
--- a/tools/packaging/static-build/virtiofsd/build.sh
+++ b/tools/packaging/static-build/virtiofsd/build.sh
@@ -51,9 +51,12 @@ esac
 
 container_image="${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(umame -m)"
 
-sudo docker pull ${container_image} || sudo docker build \
-	--build-arg RUST_TOOLCHAIN="${virtiofsd_toolchain}" \
-	-t "${container_image}" "${script_dir}/${libc}"
+sudo docker pull ${container_image} || \
+	(sudo docker build \
+		--build-arg RUST_TOOLCHAIN="${virtiofsd_toolchain}" \
+		-t "${container_image}" "${script_dir}/${libc}" && \
+	 # No-op unless PUSH_TO_REGISTRY is exported as "yes"
+	 push_to_registry "${container_image}")
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From fb7b86b8e0e39cb5e47a6a89d110a516bc042da2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 27 Jan 2023 17:31:42 +0100
Subject: [PATCH 17/76] packaging: Use existing image to build QEMU
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's first try to pull a pre-existsing image, instead of building our
own, to be used as a builder image for QEMU.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/qemu/build-base-qemu.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh
index 4ad6e0f95..9584857c7 100755
--- a/tools/packaging/static-build/qemu/build-base-qemu.sh
+++ b/tools/packaging/static-build/qemu/build-base-qemu.sh
@@ -39,13 +39,15 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
 [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
 [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
 
-sudo "${container_engine}" build \
+container_image="${BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
+
+sudo docker pull ${container_image} || sudo "${container_engine}" build \
 	--build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \
 	--build-arg http_proxy="${http_proxy}" \
 	--build-arg https_proxy="${https_proxy}" \
 	"${packaging_dir}" \
 	-f "${script_dir}/Dockerfile" \
-	-t qemu-static
+	-t "${container_image}"
 
 sudo "${container_engine}" run \
 	--rm \
@@ -59,7 +61,7 @@ sudo "${container_engine}" run \
 	--env QEMU_TARBALL="${qemu_tar}" \
 	--env PREFIX="${prefix}" \
 	-v "${repo_root_dir}:/root/kata-containers" \
-	-v "${PWD}":/share qemu-static \
+	-v "${PWD}":/share "${container_image}" \
 	bash -c "/root/kata-containers/tools/packaging/static-build/qemu/build-qemu.sh"
 
 sudo chown ${USER}:$(id -gn ${USER}) "${PWD}/${qemu_tar}"

From c7f515172dc2a84a2c749e2d2c5c3667f5a65c5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 20 Oct 2022 09:41:34 +0200
Subject: [PATCH 18/76] packaging: Add infra to push the QEMU builder image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's add the needed infra for only building and pushing the QEMU
builder image to the Kata Containers' quay.io registry.

Fixes: #5481

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/qemu/build-base-qemu.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh
index 9584857c7..e1406006a 100755
--- a/tools/packaging/static-build/qemu/build-base-qemu.sh
+++ b/tools/packaging/static-build/qemu/build-base-qemu.sh
@@ -41,13 +41,15 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
 
 container_image="${BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
 
-sudo docker pull ${container_image} || sudo "${container_engine}" build \
+sudo docker pull ${container_image} || (sudo "${container_engine}" build \
 	--build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \
 	--build-arg http_proxy="${http_proxy}" \
 	--build-arg https_proxy="${https_proxy}" \
 	"${packaging_dir}" \
 	-f "${script_dir}/Dockerfile" \
-	-t "${container_image}"
+	-t "${container_image}" && \
+	 # No-op unless PUSH_TO_REGISTRY is exported as "yes"
+	 push_to_registry "${container_image}")
 
 sudo "${container_engine}" run \
 	--rm \

From f076fa4c770f09f98a06de207d2ccf3810bebc16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:12:34 +0200
Subject: [PATCH 19/76] packaging: Use existing image to build OVMF
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's first try to pull a pre-existing image, instead of buildinf our
own, to be used as a builder image for OVMF.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/ovmf/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh
index fcbbd9321..020416269 100755
--- a/tools/packaging/static-build/ovmf/build.sh
+++ b/tools/packaging/static-build/ovmf/build.sh
@@ -16,7 +16,7 @@ source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="kata-ovmf-builder"
+container_image="${BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
 ovmf_build="${ovmf_build:-x86_64}"
 kata_version="${kata_version:-}"
 ovmf_repo="${ovmf_repo:-}"
@@ -52,7 +52,7 @@ fi
 [ -n "$ovmf_package" ] || die "failed to get ovmf package or commit"
 [ -n "$package_output_dir" ] || die "failed to get ovmf package or commit"
 
-sudo docker build -t "${container_image}" "${script_dir}"
+sudo docker pull ${container_image} || sudo docker build -t "${container_image}" "${script_dir}"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From 3fa24f7acce5a89048be2287e75b94fe6178b2ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 16:42:50 +0200
Subject: [PATCH 20/76] packaging: Add infra to push the OVMF builder image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's add the needed infra for building and pushing the OVMF builder
image to the Kata Containers' quay.io registry.

Fixes: #5477

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/ovmf/build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh
index 020416269..96e989677 100755
--- a/tools/packaging/static-build/ovmf/build.sh
+++ b/tools/packaging/static-build/ovmf/build.sh
@@ -52,7 +52,10 @@ fi
 [ -n "$ovmf_package" ] || die "failed to get ovmf package or commit"
 [ -n "$package_output_dir" ] || die "failed to get ovmf package or commit"
 
-sudo docker pull ${container_image} || sudo docker build -t "${container_image}" "${script_dir}"
+sudo docker pull ${container_image} || \
+	(sudo docker build -t "${container_image}" "${script_dir}" && \
+	 # No-op unless PUSH_TO_REGISTRY is exported as "yes"
+	 push_to_registry "${container_image}")
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From e8c6bfbdeb8fcfc53ab5c177b1eabe08ed11720e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 17:08:30 +0200
Subject: [PATCH 21/76] packaging: Use existing image to build td-shim
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's first try to pull a pre-existing image, instead of building our
own, to be used as a builder image for the td-shim.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/td-shim/build.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh
index 580c4a337..ef62f47f4 100755
--- a/tools/packaging/static-build/td-shim/build.sh
+++ b/tools/packaging/static-build/td-shim/build.sh
@@ -16,7 +16,6 @@ source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="kata-td-shim-builder"
 kata_version="${kata_version:-}"
 tdshim_repo="${tdshim_repo:-}"
 tdshim_version="${tdshim_version:-}"
@@ -31,9 +30,12 @@ package_output_dir="${package_output_dir:-}"
 [ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit"
 [ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project"
 
-sudo docker build \
+container_image="${BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
+
+sudo docker pull ${container_image} || sudo docker build \
 	--build-arg RUST_TOOLCHAIN="${tdshim_toolchain}" \
-	-t "${container_image}" "${script_dir}"
+	-t "${container_image}" \
+	"${script_dir}"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From 45668fae15ac9f9d59543de656a616c139a30576 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 19 Oct 2022 17:08:30 +0200
Subject: [PATCH 22/76] packaging: Use existing image to build td-shim
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's first try to pull a pre-existing image, instead of building our
own, to be used as a builder image for the td-shim.

This will save us some CI time.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/td-shim/build.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh
index ef62f47f4..2afa4cebd 100755
--- a/tools/packaging/static-build/td-shim/build.sh
+++ b/tools/packaging/static-build/td-shim/build.sh
@@ -32,10 +32,13 @@ package_output_dir="${package_output_dir:-}"
 
 container_image="${BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
 
-sudo docker pull ${container_image} || sudo docker build \
+sudo docker pull ${container_image} || (sudo docker build \
 	--build-arg RUST_TOOLCHAIN="${tdshim_toolchain}" \
 	-t "${container_image}" \
-	"${script_dir}"
+	"${script_dir}" && \
+	# No-op unless PUSH_TO_REGISTRY is exported as "yes"
+	push_to_registry "${container_image}")
+
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \

From 781ed2986a25fb6a3e2c0e94a4fc8609c41ed5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 26 Oct 2022 14:09:47 +0200
Subject: [PATCH 23/76] packaging: Allow passing a container builder to the
 scripts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This, combined with the effort of caching builder images *and* only
performing the build itself inside the builder images, is the very first
step for reproducible builds for the project.

Reproducible builds are quite important when we talk about Confidential
Containers, as users may want to verify the content used / provided by
the CSPs, and this is the first step towards that direction.

Fixes: #5517

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../local-build/kata-deploy-binaries-in-docker.sh          | 7 +++++++
 tools/packaging/static-build/kernel/build.sh               | 2 +-
 tools/packaging/static-build/ovmf/build.sh                 | 2 +-
 tools/packaging/static-build/qemu/build-base-qemu.sh       | 2 +-
 tools/packaging/static-build/shim-v2/build.sh              | 2 +-
 tools/packaging/static-build/td-shim/build.sh              | 3 +--
 tools/packaging/static-build/virtiofsd/build.sh            | 2 +-
 7 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
index 412f1e87d..defb338d9 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
@@ -51,6 +51,13 @@ docker run \
 	--env CI="${CI:-}" \
 	--env USER=${USER} \
 	--env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" \
+	--env INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER:-}" \
+	--env KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER:-}" \
+	--env OVMF_CONTAINER_BUILDER="${OVMF_CONTAINER_BUILDER:-}" \
+	--env QEMU_CONTAINER_BUILDER="${QEMU_CONTAINER_BUILDER:-}" \
+	--env SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER:-}" \
+	--env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \
+	--env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \
 	--rm \
 	-w ${script_dir} \
 	build-kata-deploy "${kata_deploy_create}" $@
diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh
index c04ee0c8e..8235ba22f 100755
--- a/tools/packaging/static-build/kernel/build.sh
+++ b/tools/packaging/static-build/kernel/build.sh
@@ -16,7 +16,7 @@ source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(umame -m)"
+container_image="${KERNEL_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || \
 	(sudo docker build -t "${container_image}" "${script_dir}" && \
diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh
index 96e989677..256a89249 100755
--- a/tools/packaging/static-build/ovmf/build.sh
+++ b/tools/packaging/static-build/ovmf/build.sh
@@ -16,7 +16,7 @@ source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
+container_image="${OVMF_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
 ovmf_build="${ovmf_build:-x86_64}"
 kata_version="${kata_version:-}"
 ovmf_repo="${ovmf_repo:-}"
diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh
index e1406006a..d5ac3e259 100755
--- a/tools/packaging/static-build/qemu/build-base-qemu.sh
+++ b/tools/packaging/static-build/qemu/build-base-qemu.sh
@@ -39,7 +39,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
 [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
 [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
 
-container_image="${BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
+container_image="${QEMU_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || (sudo "${container_engine}" build \
 	--build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \
diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh
index a40f87f25..2164cf9d1 100755
--- a/tools/packaging/static-build/shim-v2/build.sh
+++ b/tools/packaging/static-build/shim-v2/build.sh
@@ -20,7 +20,7 @@ RUST_VERSION=${RUST_VERSION}
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
+container_image="${SHIM_V2_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || \
        	(sudo docker build  \
diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh
index 2afa4cebd..f822fd827 100755
--- a/tools/packaging/static-build/td-shim/build.sh
+++ b/tools/packaging/static-build/td-shim/build.sh
@@ -30,7 +30,7 @@ package_output_dir="${package_output_dir:-}"
 [ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit"
 [ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project"
 
-container_image="${BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)"
+container_image="${TDSHIM_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || (sudo docker build \
 	--build-arg RUST_TOOLCHAIN="${tdshim_toolchain}" \
@@ -39,7 +39,6 @@ sudo docker pull ${container_image} || (sudo docker build \
 	# No-op unless PUSH_TO_REGISTRY is exported as "yes"
 	push_to_registry "${container_image}")
 
-
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env DESTDIR="${DESTDIR}" \
diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh
index d3f8dca07..3a41c1211 100755
--- a/tools/packaging/static-build/virtiofsd/build.sh
+++ b/tools/packaging/static-build/virtiofsd/build.sh
@@ -49,7 +49,7 @@ case ${ARCH} in
 		;;
 esac
 
-container_image="${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(umame -m)"
+container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || \
 	(sudo docker build \

From b835c40bbdc126e97256c0342d10aa0b09ac14e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 27 Jan 2023 21:49:56 +0100
Subject: [PATCH 24/76] workflows: Push the builder image to quay.io
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's push the builder images to a registry, so we can take advantage of
those on each step of our building process.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/kata-deploy-push.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/kata-deploy-push.yaml b/.github/workflows/kata-deploy-push.yaml
index c7d7e8cb4..548391fb5 100644
--- a/.github/workflows/kata-deploy-push.yaml
+++ b/.github/workflows/kata-deploy-push.yaml
@@ -28,6 +28,13 @@ jobs:
           - virtiofsd
           - nydus
     steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
       - uses: actions/checkout@v2
       - name: Install docker
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
@@ -44,6 +51,7 @@ jobs:
           sudo cp -r --preserve=all "${build_dir}" "kata-build"
         env:
           KATA_ASSET: ${{ matrix.asset }}
+          PUSH_TO_REGISTRY: yes
 
       - name: store-artifact ${{ matrix.asset }}
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

From 919d19f41542970f9dc9ad914508c1f35d5ec512 Mon Sep 17 00:00:00 2001
From: Ji-Xinyou <jerryji0414@outlook.com>
Date: Fri, 17 Feb 2023 21:30:49 +0800
Subject: [PATCH 25/76] feat(runtime): make static resource management
 consistent with 2.0

* add doc in the configuration
* make entry consistent with 2.0

Fixes: #6313
Signed-off-by: Ji-Xinyou <jerryji0414@outlook.com>
---
 src/libs/kata-types/src/config/runtime.rs              | 3 ++-
 src/runtime-rs/config/configuration-dragonball.toml.in | 9 ++++++++-
 src/runtime-rs/crates/runtimes/src/manager.rs          | 3 ++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs
index bfbde60d0..1d7364368 100644
--- a/src/libs/kata-types/src/config/runtime.rs
+++ b/src/libs/kata-types/src/config/runtime.rs
@@ -103,8 +103,9 @@ pub struct Runtime {
     pub enable_pprof: bool,
 
     /// If enabled, static resource management will calculate the vcpu and memory for the sandbox/container
+    /// And pod configured this will not be able to further update its CPU/Memory resource
     #[serde(default)]
-    pub static_resource_mgmt: bool,
+    pub static_sandbox_resource_mgmt: bool,
 
     /// Determines whether container seccomp profiles are passed to the virtual machine and
     /// applied by the kata agent. If set to true, seccomp is not applied within the guest.
diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in
index 8131d0c68..8b963e12d 100644
--- a/src/runtime-rs/config/configuration-dragonball.toml.in
+++ b/src/runtime-rs/config/configuration-dragonball.toml.in
@@ -293,5 +293,12 @@ experimental=@DEFAULTEXPFEATURES@
 # (default: false)
 # enable_pprof = true
 
-static_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@
 
diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs
index 1fba4e522..c8e5f0f10 100644
--- a/src/runtime-rs/crates/runtimes/src/manager.rs
+++ b/src/runtime-rs/crates/runtimes/src/manager.rs
@@ -374,7 +374,7 @@ fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig>
     //   2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."),
     //	then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is
     //	a single container.
-    if toml_config.runtime.static_resource_mgmt {
+    if toml_config.runtime.static_sandbox_resource_mgmt {
         info!(sl!(), "static resource management enabled");
         let static_resource_manager = StaticResourceManager::new(spec)
             .context("failed to construct static resource manager")?;
@@ -382,6 +382,7 @@ fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig>
             .setup_config(&mut toml_config)
             .context("failed to setup static resource mgmt config")?;
     }
+
     info!(sl!(), "get config content {:?}", &toml_config);
     Ok(toml_config)
 }

From 3cfce5a7090f5a420def80e58fc7072e4f3cb90a Mon Sep 17 00:00:00 2001
From: Gabe Venberg <gabevenberg@gmail.com>
Date: Fri, 17 Feb 2023 09:06:26 -0600
Subject: [PATCH 26/76] utils: improved unsupported distro message.

previously, if installing on unkown distro, script would tell user that
their distro was unsupported. Changed error message prompting user to
install dependecies manually, then retry.

Signed-off-by: Gabe Venberg <gabevenberg@gmail.com>
---
 utils/kata-manager.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh
index d09a2e67b..e28c4a44b 100755
--- a/utils/kata-manager.sh
+++ b/utils/kata-manager.sh
@@ -327,7 +327,7 @@ check_deps()
 		debian|ubuntu) sudo apt-get -y install $packages ;;
 		fedora) sudo dnf -y install $packages ;;
 		opensuse*|sles) sudo zypper install -y $packages ;;
-		*) die "Unsupported distro: $ID, install $packages manually and re-run"
+		*) die "Cannot automatically install packages on $ID, install $packages manually and re-run"
 	esac
 }
 

From 9d78bf90861bcf2464aebf028b4cc233ec38ec26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 20 Feb 2023 07:06:38 +0100
Subject: [PATCH 27/76] shim-v2: Bump Ubuntu container image  to 22.04
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's bump the base container image to use the 22.04 version of Ubuntu,
as it does bring up-to-date package dependencies that we need to
statically build the runtime-rs on aarch64.

Fixes: #6320

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/shim-v2/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/static-build/shim-v2/Dockerfile b/tools/packaging/static-build/shim-v2/Dockerfile
index 045e4f2c9..5d4fe8f88 100644
--- a/tools/packaging/static-build/shim-v2/Dockerfile
+++ b/tools/packaging/static-build/shim-v2/Dockerfile
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \

From 0301194851c01aff30cba6850b78c62e7fb67961 Mon Sep 17 00:00:00 2001
From: Tim Zhang <tim@hyper.sh>
Date: Mon, 20 Feb 2023 10:20:56 +0800
Subject: [PATCH 28/76] dragonball: use crossbeam_channel in VmmService instead
 of mpsc::channel

Because crossbeam_channel has more features and better performance than
mpsc::channel and finally rust replace its channel implementation with
crossbeam_channel on version 1.67

Signed-off-by: Tim Zhang <tim@hyper.sh>
---
 src/dragonball/Cargo.lock               |  1 +
 src/dragonball/Cargo.toml               |  3 ++-
 src/dragonball/src/api/v1/vmm_action.rs | 16 ++++++++--------
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock
index 1352f2980..048d9c896 100644
--- a/src/dragonball/Cargo.lock
+++ b/src/dragonball/Cargo.lock
@@ -413,6 +413,7 @@ version = "0.1.0"
 dependencies = [
  "arc-swap",
  "bytes",
+ "crossbeam-channel",
  "dbs-address-space",
  "dbs-allocator",
  "dbs-arch",
diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml
index 9ef43a3e2..5b51dd68f 100644
--- a/src/dragonball/Cargo.toml
+++ b/src/dragonball/Cargo.toml
@@ -39,6 +39,7 @@ thiserror = "1"
 vmm-sys-util = "0.11.0"
 virtio-queue = { version = "0.4.0", optional = true }
 vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
+crossbeam-channel = "0.5.6"
 
 [dev-dependencies]
 slog-term = "2.9.0"
@@ -47,7 +48,7 @@ test-utils = { path = "../libs/test-utils" }
 
 [features]
 acpi = []
-atomic-guest-memory = [ "vm-memory/backend-atomic" ]
+atomic-guest-memory = ["vm-memory/backend-atomic"]
 hotplug = ["virtio-vsock"]
 virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs
index 886e9ec68..56affcacf 100644
--- a/src/dragonball/src/api/v1/vmm_action.rs
+++ b/src/dragonball/src/api/v1/vmm_action.rs
@@ -7,8 +7,8 @@
 // found in the THIRD-PARTY file.
 
 use std::fs::File;
-use std::sync::mpsc::{Receiver, Sender, TryRecvError};
 
+use crossbeam_channel::{Receiver, Sender, TryRecvError};
 use log::{debug, error, info, warn};
 
 use crate::error::{Result, StartMicroVmError, StopMicrovmError};
@@ -676,9 +676,9 @@ fn handle_cpu_topology(
 
 #[cfg(test)]
 mod tests {
-    use std::sync::mpsc::channel;
     use std::sync::{Arc, Mutex};
 
+    use crossbeam_channel::unbounded;
     use dbs_utils::epoll_manager::EpollManager;
     use test_utils::skip_if_not_root;
     use vmm_sys_util::tempfile::TempFile;
@@ -702,8 +702,8 @@ mod tests {
         }
 
         fn check_request(&mut self) {
-            let (to_vmm, from_api) = channel();
-            let (to_api, from_vmm) = channel();
+            let (to_vmm, from_api) = unbounded();
+            let (to_api, from_vmm) = unbounded();
 
             let epoll_mgr = EpollManager::default();
             let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
@@ -728,8 +728,8 @@ mod tests {
     fn test_vmm_action_receive_unknown() {
         skip_if_not_root!();
 
-        let (_to_vmm, from_api) = channel();
-        let (to_api, _from_vmm) = channel();
+        let (_to_vmm, from_api) = unbounded();
+        let (to_api, _from_vmm) = unbounded();
         let epoll_mgr = EpollManager::default();
         let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
         let mut vservice = VmmService::new(from_api, to_api);
@@ -742,8 +742,8 @@ mod tests {
     #[should_panic]
     #[test]
     fn test_vmm_action_disconnected() {
-        let (to_vmm, from_api) = channel();
-        let (to_api, _from_vmm) = channel();
+        let (to_vmm, from_api) = unbounded();
+        let (to_api, _from_vmm) = unbounded();
         let epoll_mgr = EpollManager::default();
         let vmm = Arc::new(Mutex::new(create_vmm_instance(epoll_mgr.clone())));
         let mut vservice = VmmService::new(from_api, to_api);

From da8a6417aa21baf7a1112c855e067a19cb90ad43 Mon Sep 17 00:00:00 2001
From: Tim Zhang <tim@hyper.sh>
Date: Mon, 20 Feb 2023 14:08:14 +0800
Subject: [PATCH 29/76] runtime-rs: remove all remaining unsafe impl

Fixes: #6307

Signed-off-by: Tim Zhang <tim@hyper.sh>
---
 src/runtime-rs/Cargo.lock                              |  6 ++++--
 src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs   |  3 ---
 src/runtime-rs/crates/agent/src/sock/vsock.rs          |  3 ---
 src/runtime-rs/crates/hypervisor/Cargo.toml            |  1 +
 src/runtime-rs/crates/hypervisor/src/ch/inner.rs       |  3 ---
 src/runtime-rs/crates/hypervisor/src/ch/mod.rs         |  3 ---
 .../crates/hypervisor/src/dragonball/inner.rs          |  1 -
 .../crates/hypervisor/src/dragonball/vmm_instance.rs   | 10 ++++------
 src/runtime-rs/crates/hypervisor/src/qemu/inner.rs     |  3 ---
 src/runtime-rs/crates/runtimes/common/src/message.rs   |  5 +----
 10 files changed, 10 insertions(+), 28 deletions(-)

diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock
index 598d1940a..24521f1de 100644
--- a/src/runtime-rs/Cargo.lock
+++ b/src/runtime-rs/Cargo.lock
@@ -534,9 +534,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.4"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
 dependencies = [
  "cfg-if 1.0.0",
  "crossbeam-utils",
@@ -780,6 +780,7 @@ version = "0.1.0"
 dependencies = [
  "arc-swap",
  "bytes 1.1.0",
+ "crossbeam-channel",
  "dbs-address-space",
  "dbs-allocator",
  "dbs-arch",
@@ -1256,6 +1257,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "ch-config",
+ "crossbeam-channel",
  "dbs-utils",
  "dragonball",
  "futures 0.3.26",
diff --git a/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs b/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs
index 59e93a64d..1b19a65b0 100644
--- a/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs
+++ b/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs
@@ -15,9 +15,6 @@ use tokio::{
 
 use super::{ConnectConfig, Sock, Stream};
 
-unsafe impl Send for HybridVsock {}
-unsafe impl Sync for HybridVsock {}
-
 #[derive(Debug, PartialEq)]
 pub struct HybridVsock {
     uds: String,
diff --git a/src/runtime-rs/crates/agent/src/sock/vsock.rs b/src/runtime-rs/crates/agent/src/sock/vsock.rs
index 52ec1eb0b..1fbac463d 100644
--- a/src/runtime-rs/crates/agent/src/sock/vsock.rs
+++ b/src/runtime-rs/crates/agent/src/sock/vsock.rs
@@ -16,9 +16,6 @@ use tokio::net::UnixStream;
 
 use super::{ConnectConfig, Sock, Stream};
 
-unsafe impl Send for Vsock {}
-unsafe impl Sync for Vsock {}
-
 #[derive(Debug, PartialEq)]
 pub struct Vsock {
     vsock_cid: u32,
diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml
index 7bd49dd8b..c06b6c404 100644
--- a/src/runtime-rs/crates/hypervisor/Cargo.toml
+++ b/src/runtime-rs/crates/hypervisor/Cargo.toml
@@ -36,6 +36,7 @@ ch-config = { path = "ch-config", optional = true }
 
 futures = "0.3.25"
 safe-path = "0.1.0"
+crossbeam-channel = "0.5.6"
 
 [features]
 default = []
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
index 7f65ac11b..b2bd32176 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
@@ -53,9 +53,6 @@ pub struct CloudHypervisorInner {
     pub(crate) tasks: Option<Vec<JoinHandle<Result<()>>>>,
 }
 
-unsafe impl Send for CloudHypervisorInner {}
-unsafe impl Sync for CloudHypervisorInner {}
-
 const CH_DEFAULT_TIMEOUT_SECS: u32 = 10;
 
 impl CloudHypervisorInner {
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
index d589c18df..8e754415d 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
@@ -33,9 +33,6 @@ pub struct CloudHypervisor {
     inner: Arc<RwLock<CloudHypervisorInner>>,
 }
 
-unsafe impl Send for CloudHypervisor {}
-unsafe impl Sync for CloudHypervisor {}
-
 impl CloudHypervisor {
     pub fn new() -> Self {
         Self {
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
index 849ec4a2b..d2d2cd86d 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
@@ -27,7 +27,6 @@ use std::{collections::HashSet, fs::create_dir_all, path::PathBuf};
 const DRAGONBALL_KERNEL: &str = "vmlinux";
 const DRAGONBALL_ROOT_FS: &str = "rootfs";
 
-unsafe impl Sync for DragonballInner {}
 pub struct DragonballInner {
     /// sandbox id
     pub(crate) id: String,
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
index 9837ea667..f0d613938 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
@@ -7,14 +7,12 @@
 use std::{
     fs::{File, OpenOptions},
     os::unix::{io::IntoRawFd, prelude::AsRawFd},
-    sync::{
-        mpsc::{channel, Receiver, Sender},
-        Arc, Mutex, RwLock,
-    },
+    sync::{Arc, Mutex, RwLock},
     thread,
 };
 
 use anyhow::{anyhow, Context, Result};
+use crossbeam_channel::{unbounded, Receiver, Sender};
 use dragonball::{
     api::v1::{
         BlockDeviceConfigInfo, BootSourceConfig, FsDeviceConfigInfo, FsMountConfigInfo,
@@ -86,8 +84,8 @@ impl VmmInstance {
     pub fn run_vmm_server(&mut self, id: &str, netns: Option<String>) -> Result<()> {
         let kvm = OpenOptions::new().read(true).write(true).open(KVM_DEVICE)?;
 
-        let (to_vmm, from_runtime) = channel();
-        let (to_runtime, from_vmm) = channel();
+        let (to_vmm, from_runtime) = unbounded();
+        let (to_runtime, from_vmm) = unbounded();
 
         self.set_instance_id(id);
 
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
index d985ba724..e298e801b 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
@@ -12,9 +12,6 @@ const VSOCK_SCHEME: &str = "vsock";
 const VSOCK_AGENT_CID: u32 = 3;
 const VSOCK_AGENT_PORT: u32 = 1024;
 
-unsafe impl Send for QemuInner {}
-unsafe impl Sync for QemuInner {}
-
 pub struct QemuInner {
     config: HypervisorConfig,
 }
diff --git a/src/runtime-rs/crates/runtimes/common/src/message.rs b/src/runtime-rs/crates/runtimes/common/src/message.rs
index 856a6e599..622d5a162 100644
--- a/src/runtime-rs/crates/runtimes/common/src/message.rs
+++ b/src/runtime-rs/crates/runtimes/common/src/message.rs
@@ -17,12 +17,9 @@ pub enum Action {
     Start,
     Stop,
     Shutdown,
-    Event(Arc<dyn Event>),
+    Event(Arc<dyn Event + Send + Sync>),
 }
 
-unsafe impl Send for Message {}
-unsafe impl Sync for Message {}
-
 #[derive(Debug)]
 pub struct Message {
     pub action: Action,

From ced3c99895597fce94b82a1a03c07a78ef3b4483 Mon Sep 17 00:00:00 2001
From: Helin Guo <helinguo@linux.alibaba.com>
Date: Mon, 13 Feb 2023 11:44:58 +0800
Subject: [PATCH 30/76] dragonball: config_manager: preserve device when update

DeviceConfigInfo contains config and device, so when we want to do
update we could simply update config part of the info, and device would
not be changed during update.

Fixes: #6324

Signed-off-by: Helin Guo <helinguo@linux.alibaba.com>
---
 src/dragonball/src/config_manager.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs
index 34a2af2e0..e1c89d2fa 100644
--- a/src/dragonball/src/config_manager.rs
+++ b/src/dragonball/src/config_manager.rs
@@ -231,7 +231,7 @@ where
                         info.config.check_conflicts(config)?;
                     }
                 }
-                self.info_list[index] = device_info;
+                self.info_list[index].config = config.clone();
                 index
             }
             None => {

From 697ec8e578f32c238adf7c14262f6bf1437eadd7 Mon Sep 17 00:00:00 2001
From: SinghWang <wangxin_0611@126.com>
Date: Mon, 20 Feb 2023 10:13:22 +0800
Subject: [PATCH 31/76] kata-deploy: Fix kata static firecracker arm64 package
 build error

When building the kata static arm64 package, the stages of firecracker report errors.

Fixes: #6318
Signed-off-by: SinghWang <wangxin_0611@126.com>
---
 .../static-build/firecracker/build-static-firecracker.sh    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh
index 2d98f4df7..159575410 100755
--- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh
+++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh
@@ -18,6 +18,8 @@ firecracker_repo="${firecracker_repo:-}"
 firecracker_dir="firecracker"
 firecracker_version="${firecracker_version:-}"
 
+arch=$(uname -m)
+
 if [ -z "$firecracker_repo" ]; then
 	info "Get firecracker information from runtime versions.yaml"
 	firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url")
@@ -37,5 +39,5 @@ git fetch
 git checkout ${firecracker_version}
 sudo ./tools/devtool --unattended build --release
 
-ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/firecracker ./firecracker-static
-ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/jailer ./jailer-static
+ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/firecracker ./firecracker-static
+ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/jailer ./jailer-static

From 76e926453a02bf52d62c9de65c549d7828ae4d65 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 10 Feb 2023 11:43:31 +0100
Subject: [PATCH 32/76] osbuilder: Include minimal set of device nodes in
 ubuntu initrd

When starting an initrd the kernel expects to find /dev/console in the initrd,
so that it can connect it as stdin/stdout/stderr to the /init process. If the
device node is missing the kernel will complain that it was unable to open an
initial console. If kata-agent is the initrd init process, it will also result
in log messages not being logged to console and thus not forwarded to host
syslog.

Add a set of standard device nodes for completeness, so that console logging
works. To do that we install the makedev packge which provides a MAKEDEV helper
that knows the major/minor numbers. Unfortunately the debian package tries to
create devnodes from postinst, which can be suppressed if systemd-detect-virt
is present. That's why we create a small dummy script that matches what
systemd-detect-virt would output (anything is enough to suppress mknod).

Fixes: #6261
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in | 3 +++
 tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in
index de3c31ed9..937ea89f4 100644
--- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in
@@ -6,6 +6,8 @@ ARG IMAGE_REGISTRY=docker.io
 FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@
 @SET_PROXY@
 
+# makedev tries to mknod from postinst
+RUN [ -x /usr/bin/systemd-detect-virt ] || ( echo "echo docker" >/usr/bin/systemd-detect-virt && chmod +x /usr/bin/systemd-detect-virt )
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive \
     apt-get --no-install-recommends -y install \
@@ -20,6 +22,7 @@ RUN apt-get update && \
          echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \
     git \
     make \
+    makedev \
     multistrap \
     musl-tools \
     protobuf-compiler
diff --git a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh
index daa158bf7..b53749edc 100644
--- a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh
+++ b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh
@@ -34,4 +34,10 @@ EOF
 
 	# Reduce image size and memory footprint by removing unnecessary files and directories.
 	rm -rf $rootfs_dir/usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zsh}
+
+	# Minimal set of device nodes needed when AGENT_INIT=yes so that the
+	# kernel can properly setup stdout/stdin/stderr for us
+	pushd $rootfs_dir/dev
+	MAKEDEV -v console tty ttyS null zero fd
+	popd
 }

From 4c39c4ef9f42fd397dc7f994e0f32e8c054f1319 Mon Sep 17 00:00:00 2001
From: Christophe de Dinechin <christophe@dinechin.org>
Date: Mon, 20 Feb 2023 15:23:37 +0100
Subject: [PATCH 33/76] devguide: Add link to the contribution guidelines

New developers are often confused by some of our requirements, notably porting
labels. While our CONTRIBUTING.md file points to the solution, the developer's
guide does not. Add a link there.

Fixes: #6329

Signed-off-by: Christophe de Dinechin <christophe@dinechin.org>
---
 docs/Developer-Guide.md | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md
index 5d0fdf6c5..7f0b3eaa7 100644
--- a/docs/Developer-Guide.md
+++ b/docs/Developer-Guide.md
@@ -2,6 +2,8 @@
 
 This document is written **specifically for developers**: it is not intended for end users.
 
+If you want to contribute changes that you have made, please read the [community guidelines](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md) for information about our processes.
+
 # Assumptions
 
 - You are working on a non-critical test or development system.
@@ -654,7 +656,7 @@ section when using rootfs, or when using initrd, complete the steps in the [Buil
 
 Install the image:
 
->**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
+>**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img`
 >with the initrd image name `kata-containers-initrd.img`.
 
 ```bash
@@ -688,25 +690,25 @@ $ sudo crictl run -r kata container.yaml pod.yaml
 
 The steps required to enable debug console for QEMU slightly differ with
 those for firecracker / cloud-hypervisor.
- 
+
 ##### Enabling debug console for QEMU
 
-Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
+Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console.
 
 ```bash
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
-Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml` 
+Here `kata_configuration_file` could point to `/etc/kata-containers/configuration.toml`
 or `/usr/share/defaults/kata-containers/configuration.toml`
 or `/opt/kata/share/defaults/kata-containers/configuration-{hypervisor}.toml`, if
 you installed Kata Containers using `kata-deploy`.
 
 ##### Enabling debug console for cloud-hypervisor / firecracker
 
-Slightly different configuration is required in case of firecracker and cloud hypervisor. 
-Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`. 
-Hence, the kernel command line option `agent.debug_console` will not work for them. 
+Slightly different configuration is required in case of firecracker and cloud hypervisor.
+Firecracker and cloud-hypervisor don't have a UNIX socket connected to `/dev/console`.
+Hence, the kernel command line option `agent.debug_console` will not work for them.
 These hypervisors support `hybrid vsocks`,  which can be used for communication
 between the host and the guest. The kernel command line option `agent.debug_console_vport`
  was added to allow developers specify on which `vsock` port the debugging console should be connected.
@@ -719,7 +721,7 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons
 ```
 
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
-> and gathering of agent logs respectively. 
+> and gathering of agent logs respectively.
 
 ##### Connecting to the debug console
 

From afaccf924d9363ba1a3b0e13efe014f612ef974b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 20 Feb 2023 18:37:28 +0100
Subject: [PATCH 34/76] Revert "workflows: Push the builder image to quay.io"

This reverts commit b835c40bbdc126e97256c0342d10aa0b09ac14e4.

Right now I'm reverting this one as this should only run *after* commits
get pushed to our repo, not on very PR.
---
 .github/workflows/kata-deploy-push.yaml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.github/workflows/kata-deploy-push.yaml b/.github/workflows/kata-deploy-push.yaml
index 548391fb5..c7d7e8cb4 100644
--- a/.github/workflows/kata-deploy-push.yaml
+++ b/.github/workflows/kata-deploy-push.yaml
@@ -28,13 +28,6 @@ jobs:
           - virtiofsd
           - nydus
     steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
       - uses: actions/checkout@v2
       - name: Install docker
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
@@ -51,7 +44,6 @@ jobs:
           sudo cp -r --preserve=all "${build_dir}" "kata-build"
         env:
           KATA_ASSET: ${{ matrix.asset }}
-          PUSH_TO_REGISTRY: yes
 
       - name: store-artifact ${{ matrix.asset }}
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

From 2c4f8077fd2ea0bfa8f274c3aa18952093037daf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 21 Feb 2023 09:25:31 +0100
Subject: [PATCH 35/76] Revert "shim-v2: Bump Ubuntu container image  to 22.04"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 9d78bf90861bcf2464aebf028b4cc233ec38ec26.

Golang binaries are built statically by default, unless linking against
CGO, which we do.  In this case we dynamically link against glibc,
causing us troubles when running a binary built with Ubuntu 22.04 on
Ubuntu 20.04 (which will still be supported for the next few years ...)

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/shim-v2/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/static-build/shim-v2/Dockerfile b/tools/packaging/static-build/shim-v2/Dockerfile
index 5d4fe8f88..045e4f2c9 100644
--- a/tools/packaging/static-build/shim-v2/Dockerfile
+++ b/tools/packaging/static-build/shim-v2/Dockerfile
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \

From b4a1527aa664ec54ce4fc2941bc5858622ba944f Mon Sep 17 00:00:00 2001
From: SinghWang <wangxin_0611@126.com>
Date: Mon, 20 Feb 2023 13:45:11 +0800
Subject: [PATCH 36/76] kata-deploy: Fix static shim-v2 build on arm64
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Following Jong Wu suggestion, let's link /usr/bin/musl-gcc to
/usr/bin/aarch64-linux-musl-gcc.

Fixes: #6320
Signed-off-by: SinghWang <wangxin_0611@126.com>
Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/shim-v2/install_go_rust.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/packaging/static-build/shim-v2/install_go_rust.sh b/tools/packaging/static-build/shim-v2/install_go_rust.sh
index 502d5f085..078f49d37 100755
--- a/tools/packaging/static-build/shim-v2/install_go_rust.sh
+++ b/tools/packaging/static-build/shim-v2/install_go_rust.sh
@@ -58,6 +58,10 @@ case "${ARCH}" in
 	aarch64)
 		goarch=arm64
 		LIBC=musl
+		# This is a hack needed as part of Ubuntu 20.04
+		if [ ! -f /usr/bin/aarch64-linux-musl-gcc ]; then
+			ln -sf /usr/bin/musl-gcc /usr/bin/aarch64-linux-musl-gcc
+		fi
 		;;
 	ppc64le) 
 		goarch=${ARCH}

From ad8968c8d99a6089a0fdcfae48eb55e6fd1a0504 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 10 Feb 2023 14:34:53 +0100
Subject: [PATCH 37/76] rustjail: print type of cgroup manager

Since the cgroup manager is wrapped in a dyn now, the print in
LinuxContainer::new has been useless and just says "CgroupManager". Extend the
Debug trait for 'dyn Manager' to print the type of the cgroup manager so that
it's easier to debug issues.

Fixes: #5779
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/rustjail/src/cgroups/fs/mod.rs          | 4 ++++
 src/agent/rustjail/src/cgroups/mock.rs            | 4 ++++
 src/agent/rustjail/src/cgroups/mod.rs             | 4 +++-
 src/agent/rustjail/src/cgroups/systemd/manager.rs | 4 ++++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs
index becc56036..6eaa9870d 100644
--- a/src/agent/rustjail/src/cgroups/fs/mod.rs
+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs
@@ -267,6 +267,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "cgroupfs"
+    }
 }
 
 fn set_network_resources(
diff --git a/src/agent/rustjail/src/cgroups/mock.rs b/src/agent/rustjail/src/cgroups/mock.rs
index fbad6d664..3bcc99955 100644
--- a/src/agent/rustjail/src/cgroups/mock.rs
+++ b/src/agent/rustjail/src/cgroups/mock.rs
@@ -66,6 +66,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "mock"
+    }
 }
 
 impl Manager {
diff --git a/src/agent/rustjail/src/cgroups/mod.rs b/src/agent/rustjail/src/cgroups/mod.rs
index 24275c314..c4e3b178b 100644
--- a/src/agent/rustjail/src/cgroups/mod.rs
+++ b/src/agent/rustjail/src/cgroups/mod.rs
@@ -52,10 +52,12 @@ pub trait Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Err(anyhow!("not supported!"))
     }
+
+    fn name(&self) -> &str;
 }
 
 impl Debug for dyn Manager + Send + Sync {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-        write!(f, "CgroupManager")
+        write!(f, "{}", self.name())
     }
 }
diff --git a/src/agent/rustjail/src/cgroups/systemd/manager.rs b/src/agent/rustjail/src/cgroups/systemd/manager.rs
index 481ef1e60..c52e727e4 100644
--- a/src/agent/rustjail/src/cgroups/systemd/manager.rs
+++ b/src/agent/rustjail/src/cgroups/systemd/manager.rs
@@ -101,6 +101,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "systemd"
+    }
 }
 
 impl Manager {

From 1bff1ca30adb2bd964a68f807d82e72afcfc4b49 Mon Sep 17 00:00:00 2001
From: Alex Carter <alex.carter@ibm.com>
Date: Tue, 24 Jan 2023 04:31:47 +0000
Subject: [PATCH 38/76] kernel: Add CONFIG_SEV_GUEST to SEV kernel config
 Adding kernel config to sev case since it is needed for SNP and SNP will use
 the SEV kernel. Incrementing kernel config version to reflect changes

Fixes: #6123
Signed-off-by: Alex Carter <Alex.Carter@ibm.com>
---
 tools/packaging/kernel/configs/fragments/x86_64/sev/sev.conf | 1 +
 tools/packaging/kernel/kata_config_version                   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/packaging/kernel/configs/fragments/x86_64/sev/sev.conf b/tools/packaging/kernel/configs/fragments/x86_64/sev/sev.conf
index b3b692739..46f694acc 100644
--- a/tools/packaging/kernel/configs/fragments/x86_64/sev/sev.conf
+++ b/tools/packaging/kernel/configs/fragments/x86_64/sev/sev.conf
@@ -7,4 +7,5 @@ CONFIG_EFI_STUB=y
 CONFIG_MODULE_SIG=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
+CONFIG_SEV_GUEST=y
 CONFIG_VIRT_DRIVERS=y
\ No newline at end of file
diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version
index 29d6383b5..398050c62 100644
--- a/tools/packaging/kernel/kata_config_version
+++ b/tools/packaging/kernel/kata_config_version
@@ -1 +1 @@
-100
+101

From fbee6c820e73cc686a452583046bb4dbbdb858cb Mon Sep 17 00:00:00 2001
From: "James O. D. Hunt" <james.o.hunt@intel.com>
Date: Tue, 14 Feb 2023 10:39:33 +0000
Subject: [PATCH 39/76] runtime-rs: Improve Cloud Hypervisor config handling

Replace `cloud_hypervisor_vm_create_cfg()` with a set of `TryFrom` trait
implementations in the new CH specific `convert.rs` to allow the generic
`Hypervisor` configuration to be converted into the CH specific
`VmConfig` type.

Note that device configuration is not currently handled in `convert.rs`
(it's handled in `inner_device.rs`).

This change removes the old hard-coded CH specific configuration.

Fixes: #6203.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
---
 src/libs/kata-types/src/config/default.rs     |   1 +
 src/runtime-rs/Cargo.lock                     |  37 ++-
 .../crates/hypervisor/ch-config/Cargo.toml    |   3 +
 .../crates/hypervisor/ch-config/src/ch_api.rs | 172 +-----------
 .../hypervisor/ch-config/src/convert.rs       | 251 ++++++++++++++++++
 .../crates/hypervisor/ch-config/src/lib.rs    |  15 ++
 .../hypervisor/src/ch/inner_hypervisor.rs     |  68 ++++-
 7 files changed, 371 insertions(+), 176 deletions(-)
 create mode 100644 src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs

diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs
index d2d922715..7108c4add 100644
--- a/src/libs/kata-types/src/config/default.rs
+++ b/src/libs/kata-types/src/config/default.rs
@@ -71,6 +71,7 @@ pub const MIN_QEMU_MEMORY_SIZE_MB: u32 = 64;
 
 // Default configuration for Cloud Hypervisor (CH)
 pub const DEFAULT_CH_BINARY_PATH: &str = "/usr/bin/cloud-hypervisor";
+pub const DEFAULT_CH_ROOTFS_TYPE: &str = "ext4";
 pub const DEFAULT_CH_CONTROL_PATH: &str = "";
 pub const DEFAULT_CH_ENTROPY_SOURCE: &str = "/dev/urandom";
 pub const DEFAULT_CH_GUEST_KERNEL_IMAGE: &str = "vmlinuz";
diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock
index 598d1940a..fb58de549 100644
--- a/src/runtime-rs/Cargo.lock
+++ b/src/runtime-rs/Cargo.lock
@@ -426,6 +426,8 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "api_client",
+ "kata-types",
+ "nix 0.26.2",
  "serde",
  "serde_json",
  "tokio",
@@ -614,7 +616,7 @@ dependencies = [
  "kvm-bindings",
  "kvm-ioctls",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
  "vm-memory",
  "vmm-sys-util 0.11.0",
 ]
@@ -1562,6 +1564,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.3"
@@ -1665,7 +1676,7 @@ dependencies = [
  "cc",
  "cfg-if 1.0.0",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
@@ -1677,7 +1688,7 @@ dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
@@ -1692,6 +1703,20 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nix"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
+dependencies = [
+ "bitflags",
+ "cfg-if 1.0.0",
+ "libc",
+ "memoffset 0.7.1",
+ "pin-utils",
+ "static_assertions",
+]
+
 [[package]]
 name = "no-std-compat"
 version = "0.4.1"
@@ -2720,6 +2745,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "02a8428da277a8e3a15271d79943e80ccc2ef254e78813a166a08d65e4c3ece5"
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strum"
 version = "0.24.0"
diff --git a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml
index 2fd58f9f3..a51370999 100644
--- a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml
+++ b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml
@@ -20,3 +20,6 @@ tokio = { version = "1.25.0", features = ["sync", "rt"] }
 # being used. This version is used to pin the CH config structure
 # which is relatively static.
 api_client = { git = "https://github.com/cloud-hypervisor/cloud-hypervisor", crate = "api_client", tag = "v27.0" }
+
+kata-types = { path = "../../../../libs/kata-types"}
+nix = "0.26.2"
diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs
index fe812c7ca..d332a154f 100644
--- a/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs
+++ b/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs
@@ -2,18 +2,11 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::net_util::MAC_ADDR_LEN;
-use crate::{
-    ConsoleConfig, ConsoleOutputMode, CpuTopology, CpusConfig, DeviceConfig, FsConfig, MacAddr,
-    MemoryConfig, NetConfig, PayloadConfig, PmemConfig, RngConfig, VmConfig, VsockConfig,
-};
-use anyhow::{anyhow, Context, Result};
+use crate::{DeviceConfig, FsConfig, VmConfig};
+use anyhow::{anyhow, Result};
 use api_client::simple_api_full_command_and_response;
 
-use std::fmt::Display;
-use std::net::Ipv4Addr;
 use std::os::unix::net::UnixStream;
-use std::path::PathBuf;
 use tokio::task;
 
 pub async fn cloud_hypervisor_vmm_ping(mut socket: UnixStream) -> Result<Option<String>> {
@@ -38,20 +31,9 @@ pub async fn cloud_hypervisor_vmm_shutdown(mut socket: UnixStream) -> Result<Opt
 }
 
 pub async fn cloud_hypervisor_vm_create(
-    sandbox_path: String,
-    vsock_socket_path: String,
     mut socket: UnixStream,
-    shared_fs_devices: Option<Vec<FsConfig>>,
-    pmem_devices: Option<Vec<PmemConfig>>,
+    cfg: VmConfig,
 ) -> Result<Option<String>> {
-    let cfg = cloud_hypervisor_vm_create_cfg(
-        sandbox_path,
-        vsock_socket_path,
-        shared_fs_devices,
-        pmem_devices,
-    )
-    .await?;
-
     let serialised = serde_json::to_string_pretty(&cfg)?;
 
     task::spawn_blocking(move || -> Result<Option<String>> {
@@ -124,151 +106,3 @@ pub async fn cloud_hypervisor_vm_fs_add(
 
     result
 }
-
-pub async fn cloud_hypervisor_vm_create_cfg(
-    // FIXME:
-    _sandbox_path: String,
-    vsock_socket_path: String,
-    shared_fs_devices: Option<Vec<FsConfig>>,
-    pmem_devices: Option<Vec<PmemConfig>>,
-) -> Result<VmConfig> {
-    let topology = CpuTopology {
-        threads_per_core: 1,
-        cores_per_die: 12,
-        dies_per_package: 1,
-        packages: 1,
-    };
-
-    let cpus = CpusConfig {
-        boot_vcpus: 1,
-        max_vcpus: 12,
-        max_phys_bits: 46,
-        topology: Some(topology),
-        ..Default::default()
-    };
-
-    let rng = RngConfig {
-        src: PathBuf::from("/dev/urandom"),
-        ..Default::default()
-    };
-
-    let kernel_args = vec![
-        "root=/dev/pmem0p1",
-        "rootflags=dax,data=ordered,errors=remount-ro",
-        "ro",
-        "rootfstype=ext4",
-        "panic=1",
-        "no_timer_check",
-        "noreplace-smp",
-        "console=ttyS0,115200n8",
-        "systemd.log_target=console",
-        "systemd.unit=kata-containers",
-        "systemd.mask=systemd-networkd.service",
-        "systemd.mask=systemd-networkd.socket",
-        "agent.log=debug",
-    ];
-
-    let cmdline = kernel_args.join(" ");
-
-    let kernel = PathBuf::from("/opt/kata/share/kata-containers/vmlinux.container");
-
-    // Note that PmemConfig replaces the PayloadConfig.initrd.
-    let payload = PayloadConfig {
-        kernel: Some(kernel),
-        cmdline: Some(cmdline),
-        ..Default::default()
-    };
-
-    let serial = ConsoleConfig {
-        mode: ConsoleOutputMode::Tty,
-        ..Default::default()
-    };
-
-    let ip = Ipv4Addr::new(192, 168, 10, 10);
-    let mask = Ipv4Addr::new(255, 255, 255, 0);
-
-    let mac_str = "12:34:56:78:90:01";
-
-    let mac = parse_mac(mac_str)?;
-
-    let network = NetConfig {
-        ip,
-        mask,
-        mac,
-        ..Default::default()
-    };
-
-    let memory = MemoryConfig {
-        size: (1024 * 1024 * 2048),
-
-        // Required
-        shared: true,
-
-        prefault: false,
-        hugepages: false,
-        mergeable: false,
-
-        // FIXME:
-        hotplug_size: Some(16475226112),
-
-        ..Default::default()
-    };
-
-    let fs = shared_fs_devices;
-    let pmem = pmem_devices;
-
-    let vsock = VsockConfig {
-        cid: 3,
-        socket: PathBuf::from(vsock_socket_path),
-        ..Default::default()
-    };
-
-    let cfg = VmConfig {
-        cpus,
-        memory,
-        fs,
-        serial,
-        pmem,
-        payload: Some(payload),
-        vsock: Some(vsock),
-        rng,
-        net: Some(vec![network]),
-        ..Default::default()
-    };
-
-    Ok(cfg)
-}
-
-fn parse_mac<S>(s: &S) -> Result<MacAddr>
-where
-    S: AsRef<str> + ?Sized + Display,
-{
-    let v: Vec<&str> = s.as_ref().split(':').collect();
-    let mut bytes = [0u8; MAC_ADDR_LEN];
-
-    if v.len() != MAC_ADDR_LEN {
-        return Err(anyhow!(
-            "invalid MAC {} (length {}, expected {})",
-            s,
-            v.len(),
-            MAC_ADDR_LEN
-        ));
-    }
-
-    for i in 0..MAC_ADDR_LEN {
-        if v[i].len() != 2 {
-            return Err(anyhow!(
-                "invalid MAC {} (segment {} length {}, expected {})",
-                s,
-                i,
-                v.len(),
-                2
-            ));
-        }
-
-        bytes[i] =
-            u8::from_str_radix(v[i], 16).context(format!("failed to parse MAC address: {}", s))?;
-    }
-
-    Ok(MacAddr { bytes })
-}
diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs
new file mode 100644
index 000000000..f1dc8574c
--- /dev/null
+++ b/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs
@@ -0,0 +1,251 @@
+// Copyright (c) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::net_util::MAC_ADDR_LEN;
+use crate::NamedHypervisorConfig;
+use crate::VmConfig;
+use crate::{
+    ConsoleConfig, ConsoleOutputMode, CpuFeatures, CpuTopology, CpusConfig, MacAddr, MemoryConfig,
+    PayloadConfig, RngConfig, VsockConfig,
+};
+use anyhow::{anyhow, Context, Result};
+use kata_types::config::default::DEFAULT_CH_ENTROPY_SOURCE;
+use kata_types::config::hypervisor::{CpuInfo, MachineInfo, MemoryInfo};
+use kata_types::config::BootInfo;
+use std::convert::TryFrom;
+use std::fmt::Display;
+use std::path::PathBuf;
+
+// 1 MiB
+const MIB: u64 = 1024 * 1024;
+
+const DEFAULT_CH_MAX_PHYS_BITS: u8 = 46;
+
+impl TryFrom<NamedHypervisorConfig> for VmConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(n: NamedHypervisorConfig) -> Result<Self, Self::Error> {
+        let kernel_params = n.kernel_params;
+        let cfg = n.cfg;
+        let vsock_socket_path = n.vsock_socket_path;
+        let sandbox_path = n.sandbox_path;
+        let fs = n.shared_fs_devices;
+        let pmem = n.pmem_devices;
+
+        let cpus = CpusConfig::try_from(cfg.cpu_info)?;
+
+        let rng = RngConfig::try_from(cfg.machine_info)?;
+
+        // Note that PmemConfig replaces the PayloadConfig.initrd.
+        let payload = PayloadConfig::try_from((cfg.boot_info, kernel_params))?;
+
+        let serial = get_serial_cfg()?;
+        let console = get_console_cfg()?;
+
+        let memory = MemoryConfig::try_from(cfg.memory_info)?;
+
+        std::fs::create_dir_all(sandbox_path).context("failed to create sandbox path")?;
+
+        let vsock = VsockConfig {
+            cid: 3,
+            socket: PathBuf::from(vsock_socket_path),
+            ..Default::default()
+        };
+
+        let cfg = VmConfig {
+            cpus,
+            memory,
+            serial,
+            console,
+            payload: Some(payload),
+            fs,
+            pmem,
+            vsock: Some(vsock),
+            rng,
+            ..Default::default()
+        };
+
+        Ok(cfg)
+    }
+}
+
+impl TryFrom<MemoryInfo> for MemoryConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(mem: MemoryInfo) -> Result<Self, Self::Error> {
+        let sysinfo = nix::sys::sysinfo::sysinfo()?;
+
+        let max_mem_bytes = sysinfo.ram_total();
+
+        let mem_bytes: u64 = MIB
+            .checked_mul(mem.default_memory as u64)
+            .ok_or("cannot convert default memory to bytes")
+            .map_err(|e| anyhow!(e))?;
+
+        // The amount of memory that can be hot-plugged is the total less the
+        // amount allocated at VM start.
+        let hotplug_size_bytes = max_mem_bytes
+            .checked_sub(mem_bytes)
+            .ok_or("failed to calculate max hotplug size for CH")
+            .map_err(|e| anyhow!(e))?;
+
+        let cfg = MemoryConfig {
+            size: mem_bytes,
+
+            // Required
+            shared: true,
+
+            hotplug_size: Some(hotplug_size_bytes),
+
+            ..Default::default()
+        };
+
+        Ok(cfg)
+    }
+}
+
+impl TryFrom<CpuInfo> for CpusConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(cpu: CpuInfo) -> Result<Self, Self::Error> {
+        let boot_vcpus = u8::try_from(cpu.default_vcpus)?;
+        let max_vcpus = u8::try_from(cpu.default_maxvcpus)?;
+
+        let topology = CpuTopology {
+            threads_per_core: 1,
+            cores_per_die: max_vcpus,
+            dies_per_package: 1,
+            packages: 1,
+        };
+
+        let max_phys_bits = DEFAULT_CH_MAX_PHYS_BITS;
+
+        let cfg = CpusConfig {
+            boot_vcpus,
+            max_vcpus,
+            max_phys_bits,
+            topology: Some(topology),
+
+            ..Default::default()
+        };
+
+        Ok(cfg)
+    }
+}
+
+impl TryFrom<String> for CpuFeatures {
+    type Error = anyhow::Error;
+
+    #[cfg(target_arch = "x86_64")]
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        let amx = s.split(',').any(|x| x == "amx");
+
+        let cpu_features = CpuFeatures { amx };
+
+        Ok(cpu_features)
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    fn try_from(_s: String) -> Result<Self, Self::Error> {
+        Ok(CpuFeatures::default())
+    }
+}
+
+// The 2nd tuple element is the space separated kernel parameters list.
+// This cannot be created only from BootInfo since that contains the
+// user-specified kernel parameters only.
+impl TryFrom<(BootInfo, String)> for PayloadConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(args: (BootInfo, String)) -> Result<Self, Self::Error> {
+        let b = args.0;
+        let cmdline = args.1;
+
+        let kernel = PathBuf::from(b.kernel);
+
+        let payload = PayloadConfig {
+            kernel: Some(kernel),
+            cmdline: Some(cmdline),
+
+            ..Default::default()
+        };
+
+        Ok(payload)
+    }
+}
+
+impl TryFrom<MachineInfo> for RngConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(m: MachineInfo) -> Result<Self, Self::Error> {
+        let entropy_source = if !m.entropy_source.is_empty() {
+            m.entropy_source
+        } else {
+            DEFAULT_CH_ENTROPY_SOURCE.to_string()
+        };
+
+        let rng = RngConfig {
+            src: PathBuf::from(entropy_source),
+
+            ..Default::default()
+        };
+
+        Ok(rng)
+    }
+}
+
+fn get_serial_cfg() -> Result<ConsoleConfig> {
+    let cfg = ConsoleConfig {
+        file: None,
+        mode: ConsoleOutputMode::Tty,
+        iommu: false,
+    };
+
+    Ok(cfg)
+}
+
+fn get_console_cfg() -> Result<ConsoleConfig> {
+    let cfg = ConsoleConfig {
+        file: None,
+        mode: ConsoleOutputMode::Off,
+        iommu: false,
+    };
+
+    Ok(cfg)
+}
+
+#[allow(dead_code)]
+fn parse_mac<S>(s: &S) -> Result<MacAddr>
+where
+    S: AsRef<str> + ?Sized + Display,
+{
+    let v: Vec<&str> = s.as_ref().split(':').collect();
+    let mut bytes = [0u8; MAC_ADDR_LEN];
+
+    if v.len() != MAC_ADDR_LEN {
+        return Err(anyhow!(
+            "invalid MAC {} (length {}, expected {})",
+            s,
+            v.len(),
+            MAC_ADDR_LEN
+        ));
+    }
+
+    for i in 0..MAC_ADDR_LEN {
+        if v[i].len() != 2 {
+            return Err(anyhow!(
+                "invalid MAC {} (segment {} length {}, expected {})",
+                s,
+                i,
+                v.len(),
+                2
+            ));
+        }
+
+        bytes[i] =
+            u8::from_str_radix(v[i], 16).context(format!("failed to parse MAC address: {}", s))?;
+    }
+
+    Ok(MacAddr { bytes })
+}
diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs
index 3e3fb3412..157ab8970 100644
--- a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs
+++ b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs
@@ -7,10 +7,12 @@ use std::net::Ipv4Addr;
 use std::path::PathBuf;
 
 pub mod ch_api;
+pub mod convert;
 pub mod net_util;
 mod virtio_devices;
 
 use crate::virtio_devices::RateLimiterConfig;
+use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 pub use net_util::MacAddr;
 
 pub const MAX_NUM_PCI_SEGMENTS: u16 = 16;
@@ -479,3 +481,16 @@ fn usize_is_zero(v: &usize) -> bool {
 fn u16_is_zero(v: &u16) -> bool {
     *v == 0
 }
+
+// Type used to simplify conversion from a generic Hypervisor config
+// to a CH specific VmConfig.
+#[derive(Debug, Clone)]
+pub struct NamedHypervisorConfig {
+    pub kernel_params: String,
+    pub sandbox_path: String,
+    pub vsock_socket_path: String,
+    pub cfg: HypervisorConfig,
+
+    pub shared_fs_devices: Option<Vec<FsConfig>>,
+    pub pmem_devices: Option<Vec<PmemConfig>>,
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
index b3271ee79..2ecc328ba 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
@@ -6,18 +6,23 @@
 use super::inner::CloudHypervisorInner;
 use crate::ch::utils::get_api_socket_path;
 use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path};
+use crate::kernel_param::KernelParams;
 use crate::Device;
 use crate::VsockConfig;
+use crate::VM_ROOTFS_DRIVER_PMEM;
 use crate::{VcpuThreadIds, VmmState};
 use anyhow::{anyhow, Context, Result};
 use ch_config::ch_api::{
     cloud_hypervisor_vm_create, cloud_hypervisor_vm_start, cloud_hypervisor_vmm_ping,
     cloud_hypervisor_vmm_shutdown,
 };
+use ch_config::{NamedHypervisorConfig, VmConfig};
 use core::future::poll_fn;
 use futures::executor::block_on;
 use futures::future::join_all;
 use kata_types::capabilities::{Capabilities, CapabilityBits};
+use kata_types::config::default::DEFAULT_CH_ROOTFS_TYPE;
+use std::convert::TryFrom;
 use std::fs::create_dir_all;
 use std::os::unix::net::UnixStream;
 use std::path::Path;
@@ -54,6 +59,40 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
+    async fn get_kernel_params(&self) -> Result<String> {
+        let cfg = self
+            .config
+            .as_ref()
+            .ok_or("no hypervisor config for CH")
+            .map_err(|e| anyhow!(e))?;
+
+        let enable_debug = cfg.debug_info.enable_debug;
+
+        // Note that the configuration option hypervisor.block_device_driver is not used.
+        let rootfs_driver = VM_ROOTFS_DRIVER_PMEM;
+
+        let rootfs_type = match cfg.boot_info.rootfs_type.is_empty() {
+            true => DEFAULT_CH_ROOTFS_TYPE,
+            false => &cfg.boot_info.rootfs_type,
+        };
+
+        // Start by adding the default set of kernel parameters.
+        let mut params = KernelParams::new(enable_debug);
+
+        let mut rootfs_param = KernelParams::new_rootfs_kernel_params(rootfs_driver, rootfs_type)?;
+
+        // Add the rootfs device
+        params.append(&mut rootfs_param);
+
+        // Finally, add the user-specified options at the end
+        // (so they will take priority).
+        params.append(&mut KernelParams::from_string(&cfg.boot_info.kernel_params));
+
+        let kernel_params = params.to_string()?;
+
+        Ok(kernel_params)
+    }
+
     async fn boot_vm(&mut self) -> Result<()> {
         let shared_fs_devices = self.get_shared_fs_devices().await?;
 
@@ -71,14 +110,35 @@ impl CloudHypervisorInner {
 
         let vsock_socket_path = get_vsock_path(&self.id)?;
 
-        let response = cloud_hypervisor_vm_create(
+        let hypervisor_config = self
+            .config
+            .as_ref()
+            .ok_or("no hypervisor config for CH")
+            .map_err(|e| anyhow!(e))?;
+
+        debug!(
+            sl!(),
+            "generic Hypervisor configuration: {:?}", hypervisor_config
+        );
+
+        let kernel_params = self.get_kernel_params().await?;
+
+        let named_cfg = NamedHypervisorConfig {
+            kernel_params,
             sandbox_path,
             vsock_socket_path,
-            socket.try_clone().context("failed to clone socket")?,
+            cfg: hypervisor_config.clone(),
             shared_fs_devices,
             pmem_devices,
-        )
-        .await?;
+        };
+
+        let cfg = VmConfig::try_from(named_cfg)?;
+
+        debug!(sl!(), "CH specific VmConfig configuration: {:?}", cfg);
+
+        let response =
+            cloud_hypervisor_vm_create(socket.try_clone().context("failed to clone socket")?, cfg)
+                .await?;
 
         if let Some(detail) = response {
             debug!(sl!(), "vm boot response: {:?}", detail);

From 3483272bbda5c586f5720e9ce417fd0bdbe6e31d Mon Sep 17 00:00:00 2001
From: "James O. D. Hunt" <james.o.hunt@intel.com>
Date: Tue, 21 Feb 2023 11:56:28 +0000
Subject: [PATCH 40/76] runtime-rs: ch: Enable initrd usage

Allow an initrd/initramfs image to be used with Cloud Hypervisor, which
is handled differently to the default rootfs image type.

Fixes: #6335.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
---
 .../hypervisor/ch-config/src/convert.rs       | 87 +++++++++++++++++--
 .../crates/hypervisor/ch-config/src/lib.rs    |  1 -
 .../crates/hypervisor/src/ch/inner_device.rs  | 37 +-------
 .../hypervisor/src/ch/inner_hypervisor.rs     |  3 -
 4 files changed, 81 insertions(+), 47 deletions(-)

diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs
index f1dc8574c..f0f5e88e8 100644
--- a/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs
+++ b/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs
@@ -7,7 +7,7 @@ use crate::NamedHypervisorConfig;
 use crate::VmConfig;
 use crate::{
     ConsoleConfig, ConsoleOutputMode, CpuFeatures, CpuTopology, CpusConfig, MacAddr, MemoryConfig,
-    PayloadConfig, RngConfig, VsockConfig,
+    PayloadConfig, PmemConfig, RngConfig, VsockConfig,
 };
 use anyhow::{anyhow, Context, Result};
 use kata_types::config::default::DEFAULT_CH_ENTROPY_SOURCE;
@@ -20,6 +20,8 @@ use std::path::PathBuf;
 // 1 MiB
 const MIB: u64 = 1024 * 1024;
 
+const PMEM_ALIGN_BYTES: u64 = 2 * MIB;
+
 const DEFAULT_CH_MAX_PHYS_BITS: u8 = 46;
 
 impl TryFrom<NamedHypervisorConfig> for VmConfig {
@@ -31,14 +33,42 @@ impl TryFrom<NamedHypervisorConfig> for VmConfig {
         let vsock_socket_path = n.vsock_socket_path;
         let sandbox_path = n.sandbox_path;
         let fs = n.shared_fs_devices;
-        let pmem = n.pmem_devices;
 
         let cpus = CpusConfig::try_from(cfg.cpu_info)?;
 
         let rng = RngConfig::try_from(cfg.machine_info)?;
 
-        // Note that PmemConfig replaces the PayloadConfig.initrd.
-        let payload = PayloadConfig::try_from((cfg.boot_info, kernel_params))?;
+        // Note how CH handles the different image types:
+        //
+        // - An image is specified in PmemConfig.
+        // - An initrd/initramfs is specified in PayloadConfig.
+        let boot_info = cfg.boot_info;
+
+        let use_initrd = !boot_info.initrd.is_empty();
+        let use_image = !boot_info.image.is_empty();
+
+        if use_initrd && use_image {
+            return Err(anyhow!("cannot specify image and initrd"));
+        }
+
+        if !use_initrd && !use_image {
+            return Err(anyhow!("missing boot file (no image or initrd)"));
+        }
+
+        let initrd = if use_initrd {
+            Some(PathBuf::from(boot_info.initrd.clone()))
+        } else {
+            None
+        };
+
+        let pmem = if use_initrd {
+            None
+        } else {
+            let pmem = PmemConfig::try_from(&boot_info)?;
+            Some(vec![pmem])
+        };
+
+        let payload = PayloadConfig::try_from((boot_info, kernel_params, initrd))?;
 
         let serial = get_serial_cfg()?;
         let console = get_console_cfg()?;
@@ -90,13 +120,18 @@ impl TryFrom<MemoryInfo> for MemoryConfig {
             .ok_or("failed to calculate max hotplug size for CH")
             .map_err(|e| anyhow!(e))?;
 
+        let aligned_hotplug_size_bytes =
+            checked_next_multiple_of(hotplug_size_bytes, PMEM_ALIGN_BYTES)
+                .ok_or("cannot handle pmem alignment for CH")
+                .map_err(|e| anyhow!(e))?;
+
         let cfg = MemoryConfig {
             size: mem_bytes,
 
             // Required
             shared: true,
 
-            hotplug_size: Some(hotplug_size_bytes),
+            hotplug_size: Some(aligned_hotplug_size_bytes),
 
             ..Default::default()
         };
@@ -105,6 +140,20 @@ impl TryFrom<MemoryInfo> for MemoryConfig {
     }
 }
 
+// Return the next multiple of 'multiple' starting from the specified value
+// (aka align value to multiple).
+//
+// This is a temporary solution until checked_next_multiple_of() integer
+// method is available in the rust language.
+//
+// See: https://github.com/rust-lang/rust/issues/88581
+fn checked_next_multiple_of(value: u64, multiple: u64) -> Option<u64> {
+    match value.checked_rem(multiple) {
+        None => Some(value),
+        Some(r) => value.checked_add(multiple - r),
+    }
+}
+
 impl TryFrom<CpuInfo> for CpusConfig {
     type Error = anyhow::Error;
 
@@ -153,20 +202,23 @@ impl TryFrom<String> for CpuFeatures {
 }
 
 // The 2nd tuple element is the space separated kernel parameters list.
+// The 3rd tuple element is an optional initramfs image to use.
 // This cannot be created only from BootInfo since that contains the
 // user-specified kernel parameters only.
-impl TryFrom<(BootInfo, String)> for PayloadConfig {
+impl TryFrom<(BootInfo, String, Option<PathBuf>)> for PayloadConfig {
     type Error = anyhow::Error;
 
-    fn try_from(args: (BootInfo, String)) -> Result<Self, Self::Error> {
+    fn try_from(args: (BootInfo, String, Option<PathBuf>)) -> Result<Self, Self::Error> {
         let b = args.0;
         let cmdline = args.1;
+        let initramfs = args.2;
 
         let kernel = PathBuf::from(b.kernel);
 
         let payload = PayloadConfig {
             kernel: Some(kernel),
             cmdline: Some(cmdline),
+            initramfs,
 
             ..Default::default()
         };
@@ -195,6 +247,27 @@ impl TryFrom<MachineInfo> for RngConfig {
     }
 }
 
+impl TryFrom<&BootInfo> for PmemConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(b: &BootInfo) -> Result<Self, Self::Error> {
+        let file = if b.image.is_empty() {
+            return Err(anyhow!("CH PmemConfig only used for images"));
+        } else {
+            b.image.clone()
+        };
+
+        let cfg = PmemConfig {
+            file: PathBuf::from(file),
+            discard_writes: true,
+
+            ..Default::default()
+        };
+
+        Ok(cfg)
+    }
+}
+
 fn get_serial_cfg() -> Result<ConsoleConfig> {
     let cfg = ConsoleConfig {
         file: None,
diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs
index 157ab8970..2969e6847 100644
--- a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs
+++ b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs
@@ -492,5 +492,4 @@ pub struct NamedHypervisorConfig {
     pub cfg: HypervisorConfig,
 
     pub shared_fs_devices: Option<Vec<FsConfig>>,
-    pub pmem_devices: Option<Vec<PmemConfig>>,
 }
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
index 03cf95daf..f4475f66e 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
@@ -10,7 +10,7 @@ use crate::HybridVsockConfig;
 use crate::VmmState;
 use anyhow::{anyhow, Context, Result};
 use ch_config::ch_api::cloud_hypervisor_vm_fs_add;
-use ch_config::{FsConfig, PmemConfig};
+use ch_config::FsConfig;
 use safe_path::scoped_join;
 use std::convert::TryFrom;
 use std::path::PathBuf;
@@ -148,41 +148,6 @@ impl CloudHypervisorInner {
             Ok(None)
         }
     }
-
-    pub(crate) async fn get_boot_file(&mut self) -> Result<PathBuf> {
-        if let Some(ref config) = self.config {
-            let boot_info = &config.boot_info;
-
-            let file = if !boot_info.initrd.is_empty() {
-                boot_info.initrd.clone()
-            } else if !boot_info.image.is_empty() {
-                boot_info.image.clone()
-            } else {
-                return Err(anyhow!("missing boot file (no image or initrd)"));
-            };
-
-            Ok(PathBuf::from(file))
-        } else {
-            Err(anyhow!("no hypervisor config"))
-        }
-    }
-
-    pub(crate) async fn get_pmem_devices(&mut self) -> Result<Option<Vec<PmemConfig>>> {
-        let file = self.get_boot_file().await?;
-
-        let pmem_cfg = PmemConfig {
-            file,
-            size: None,
-            iommu: false,
-            discard_writes: true,
-            id: None,
-            pci_segment: 0,
-        };
-
-        let pmem_devices = vec![pmem_cfg];
-
-        Ok(Some(pmem_devices))
-    }
 }
 
 #[derive(Debug)]
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
index 2ecc328ba..8eea6f466 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
@@ -96,8 +96,6 @@ impl CloudHypervisorInner {
     async fn boot_vm(&mut self) -> Result<()> {
         let shared_fs_devices = self.get_shared_fs_devices().await?;
 
-        let pmem_devices = self.get_pmem_devices().await?;
-
         let socket = self
             .api_socket
             .as_ref()
@@ -129,7 +127,6 @@ impl CloudHypervisorInner {
             vsock_socket_path,
             cfg: hypervisor_config.clone(),
             shared_fs_devices,
-            pmem_devices,
         };
 
         let cfg = VmConfig::try_from(named_cfg)?;

From c4ef5fd32551c879fe38aaaa6fa10473cd792723 Mon Sep 17 00:00:00 2001
From: Yohei Ueda <yohei@jp.ibm.com>
Date: Fri, 24 Feb 2023 16:43:59 +0900
Subject: [PATCH 41/76] agent: don't set permission of existing directory

This patch fixes the issue that do_copy_file changes
the directory permission of the parent directory of
a target file, even when the parent directory already
exists.

Fixes #6367

Signed-off-by: Yohei Ueda <yohei@jp.ibm.com>
---
 src/agent/src/rpc.rs | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
index c52d866d6..bb34c0f5e 100644
--- a/src/agent/src/rpc.rs
+++ b/src/agent/src/rpc.rs
@@ -1877,23 +1877,18 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         ));
     }
 
-    let parent = path.parent();
-
-    let dir = if let Some(parent) = parent {
-        parent.to_path_buf()
-    } else {
-        PathBuf::from("/")
-    };
-
-    fs::create_dir_all(&dir).or_else(|e| {
-        if e.kind() != std::io::ErrorKind::AlreadyExists {
-            return Err(e);
+    if let Some(parent) = path.parent() {
+        if !parent.exists() {
+            let dir = parent.to_path_buf();
+            if let Err(e) = fs::create_dir_all(&dir) {
+                if e.kind() != std::io::ErrorKind::AlreadyExists {
+                    return Err(e.into());
+                }
+            } else {
+                std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
+            }
         }
-
-        Ok(())
-    })?;
-
-    std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
+    }
 
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");

From dc86d6dac35f17911ebeaab9a612173b005b7ac8 Mon Sep 17 00:00:00 2001
From: XDTG <click1799@163.com>
Date: Fri, 24 Feb 2023 15:48:09 +0800
Subject: [PATCH 42/76] runtime: use filepath.Clean() to clean the mount path

Fix path check bypassed issuse introduced by #6082,
use filepath.Clean() to clean path before check

Fixes: #6082

Signed-off-by: XDTG <click1799@163.com>
---
 src/runtime/virtcontainers/mount.go            | 2 ++
 src/runtime/virtcontainers/mount_linux_test.go | 3 +++
 src/runtime/virtcontainers/mount_test.go       | 4 ++++
 3 files changed, 9 insertions(+)

diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go
index 243c13f33..6c2e20420 100644
--- a/src/runtime/virtcontainers/mount.go
+++ b/src/runtime/virtcontainers/mount.go
@@ -44,6 +44,7 @@ func mountLogger() *logrus.Entry {
 }
 
 func isSystemMount(m string) bool {
+	m = filepath.Clean(m)
 	for _, p := range systemMountPrefixes {
 		if m == p || strings.HasPrefix(m, p+"/") {
 			return true
@@ -54,6 +55,7 @@ func isSystemMount(m string) bool {
 }
 
 func isHostDevice(m string) bool {
+	m = filepath.Clean(m)
 	if m == "/dev" {
 		return true
 	}
diff --git a/src/runtime/virtcontainers/mount_linux_test.go b/src/runtime/virtcontainers/mount_linux_test.go
index a34f7c28f..e5019b401 100644
--- a/src/runtime/virtcontainers/mount_linux_test.go
+++ b/src/runtime/virtcontainers/mount_linux_test.go
@@ -249,6 +249,9 @@ func TestIsHostDevice(t *testing.T) {
 		{"/dev/zero", true},
 		{"/dev/block", true},
 		{"/mnt/dev/block", false},
+		{"/../dev", true},
+		{"/../dev/block", true},
+		{"/../mnt/dev/block", false},
 	}
 
 	for _, test := range tests {
diff --git a/src/runtime/virtcontainers/mount_test.go b/src/runtime/virtcontainers/mount_test.go
index 6d91d22a7..c21d00a19 100644
--- a/src/runtime/virtcontainers/mount_test.go
+++ b/src/runtime/virtcontainers/mount_test.go
@@ -41,6 +41,10 @@ func TestIsSystemMount(t *testing.T) {
 		{"/home", false},
 		{"/dev/block/", false},
 		{"/mnt/dev/foo", false},
+		{"/../sys", true},
+		{"/../sys/", true},
+		{"/../sys/fs/cgroup", true},
+		{"/../sysfoo", false},
 	}
 
 	for _, test := range tests {

From b0691806f1437951c38436e28aa086ae9c6f00d9 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 24 Feb 2023 13:43:26 +0100
Subject: [PATCH 43/76] agent: determine value of use_systemd_cgroup before
 LinuxContainer::new()

Right now LinuxContainer::new() gets passed a CreateOpts struct, but then
modifies the use_systemd_cgroup field inside that struct. Pull the cgroups path
parsing logic into do_create_container, so that CreateOpts can be immutable in
LinuxContainer::new. This is just moving things around, there should be no
functional changes.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/rustjail/src/container.rs | 11 ++---------
 src/agent/src/rpc.rs                | 10 ++++++++--
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
index f75b6e6d5..ed69ef60d 100644
--- a/src/agent/rustjail/src/container.rs
+++ b/src/agent/rustjail/src/container.rs
@@ -1449,7 +1449,7 @@ impl LinuxContainer {
     pub fn new<T: Into<String> + Display + Clone>(
         id: T,
         base: T,
-        mut config: Config,
+        config: Config,
         logger: &Logger,
     ) -> Result<Self> {
         let base = base.into();
@@ -1475,21 +1475,14 @@ impl LinuxContainer {
         .context(format!("Cannot change owner of container {} root", id))?;
 
         let spec = config.spec.as_ref().unwrap();
-
         let linux = spec.linux.as_ref().unwrap();
-
-        // determine which cgroup driver to take and then assign to config.use_systemd_cgroup
-        // systemd: "[slice]:[prefix]:[name]"
-        // fs: "/path_a/path_b"
-        let cpath = if SYSTEMD_CGROUP_PATH_FORMAT.is_match(linux.cgroups_path.as_str()) {
-            config.use_systemd_cgroup = true;
+        let cpath = if config.use_systemd_cgroup {
             if linux.cgroups_path.len() == 2 {
                 format!("system.slice:kata_agent:{}", id.as_str())
             } else {
                 linux.cgroups_path.clone()
             }
         } else {
-            config.use_systemd_cgroup = false;
             if linux.cgroups_path.is_empty() {
                 format!("/{}", id.as_str())
             } else {
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
index c52d866d6..008176541 100644
--- a/src/agent/src/rpc.rs
+++ b/src/agent/src/rpc.rs
@@ -36,7 +36,7 @@ use protocols::health::{
 use protocols::types::Interface;
 use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
 use rustjail::cgroups::notifier;
-use rustjail::container::{BaseContainer, Container, LinuxContainer};
+use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
 use rustjail::process::Process;
 use rustjail::specconv::CreateOpts;
 
@@ -210,9 +210,15 @@ impl AgentService {
         // restore the cwd for kata-agent process.
         defer!(unistd::chdir(&olddir).unwrap());
 
+        // determine which cgroup driver to take and then assign to use_systemd_cgroup
+        // systemd: "[slice]:[prefix]:[name]"
+        // fs: "/path_a/path_b"
+        let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path);
+        let use_systemd_cgroup = SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path);
+
         let opts = CreateOpts {
             cgroup_name: "".to_string(),
-            use_systemd_cgroup: false,
+            use_systemd_cgroup,
             no_pivot_root: s.no_pivot_root,
             no_new_keyring: false,
             spec: Some(oci.clone()),

From 192df845885f5dab27abf78751b3035f68e3d786 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 10 Feb 2023 14:50:54 +0100
Subject: [PATCH 44/76] agent: always use cgroupfs when running as init

The logic to decide which cgroup driver is used is currently based on the
cgroup path that the host provides. This requires host and guest to use the
same cgroup driver. If the guest uses kata-agent as init, then systemd can't be
used as the cgroup driver. If the host requests a systemd cgroup, this
currently results in a rustjail panic:

  thread 'tokio-runtime-worker' panicked at 'called `Result::unwrap()` on an `Err` value: I/O error: No such file or directory (os error 2)

  Caused by:
      No such file or directory (os error 2)', rustjail/src/cgroups/systemd/manager.rs:44:51
  stack backtrace:
     0:     0x7ff0fe77a793 - std::backtrace_rs::backtrace::libunwind::trace::h8c197fa9a679d134
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
     1:     0x7ff0fe77a793 - std::backtrace_rs::backtrace::trace_unsynchronized::h9ee19d58b6d5934a
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
     2:     0x7ff0fe77a793 - std::sys_common::backtrace::_print_fmt::h4badc450600fc417
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:65:5
     3:     0x7ff0fe77a793 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::had334ddb529a2169
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:44:22
     4:     0x7ff0fdce815e - core::fmt::write::h1aa7694f03e44db2
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/fmt/mod.rs:1209:17
     5:     0x7ff0fe74e0c4 - std::io::Write::write_fmt::h61b2bdc565be41b5
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/io/mod.rs:1682:15
     6:     0x7ff0fe77cd3f - std::sys_common::backtrace::_print::h4ec69798b72ff254
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:47:5
     7:     0x7ff0fe77cd3f - std::sys_common::backtrace::print::h0e6c02048dec3c77
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:34:9
     8:     0x7ff0fe77c93f - std::panicking::default_hook::{{closure}}::hcdb7e705dc37ea6e
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:267:22
     9:     0x7ff0fe77d9b8 - std::panicking::default_hook::he03a933a0f01790f
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:286:9
    10:     0x7ff0fe77d9b8 - std::panicking::rust_panic_with_hook::he26b680bfd953008
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:688:13
    11:     0x7ff0fe77d482 - std::panicking::begin_panic_handler::{{closure}}::h559120d2dd1c6180
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:579:13
    12:     0x7ff0fe77d3ec - std::sys_common::backtrace::__rust_end_short_backtrace::h36db621fc93b005a
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:137:18
    13:     0x7ff0fe77d3c1 - rust_begin_unwind
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:575:5
    14:     0x7ff0fda52ee2 - core::panicking::panic_fmt::he7679b415d25c5f4
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/panicking.rs:65:14
    15:     0x7ff0fda53182 - core::result::unwrap_failed::hb71caff146724b6b
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/result.rs:1791:5
    16:     0x7ff0fe5bd738 - <rustjail::cgroups::systemd::manager::Manager as rustjail::cgroups::Manager>::apply::hd46958d9d807d2ca
    17:     0x7ff0fe606d80 - <rustjail::container::LinuxContainer as rustjail::container::BaseContainer>::start::{{closure}}::h1de806d91fcb878f
    18:     0x7ff0fe604a76 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h1749c148adcc235f
    19:     0x7ff0fdc0c992 - kata_agent::rpc::AgentService::do_create_container::{{closure}}::{{closure}}::hc1b87a15dfdf2f64
    20:     0x7ff0fdb80ae4 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h846a8c9e4fb67707
    21:     0x7ff0fe3bb816 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h53de16ff66ed3972
    22:     0x7ff0fdb519cb - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h1cbece980286c0f4
    23:     0x7ff0fdf4019c - <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::hc8e72d155feb8d1f
    24:     0x7ff0fdfa5fd8 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h0a407ffe2559449a
    25:     0x7ff0fdf033a1 - tokio::runtime::task::raw::poll::h1045d9f1db9742de
    26:     0x7ff0fe7a8ce2 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h4924ae3464af7fbd
    27:     0x7ff0fe7afb85 - tokio::runtime::task::raw::poll::h5c843be39646b833
    28:     0x7ff0fe7a05ee - std::sys_common::backtrace::__rust_begin_short_backtrace::ha7777c55b98a9bd1
    29:     0x7ff0fe7a9bdb - core::ops::function::FnOnce::call_once{{vtable.shim}}::h27ec83c953360cdd
    30:     0x7ff0fe7801d5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hed812350c5aef7a8
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/alloc/src/boxed.rs:1987:9
    31:     0x7ff0fe7801d5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hc7df8e435a658960
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/alloc/src/boxed.rs:1987:9
    32:     0x7ff0fe7801d5 - std::sys::unix::thread::Thread::new::thread_start::h575491a8a17dbb33
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys/unix/thread.rs:108:17

Forward the value of "init_mode" to AgentService, so that we can force cgroupfs
when systemd is unavailable.

Fixes: #5779
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/rustjail/src/container.rs |  9 ++++-----
 src/agent/src/main.rs               |  2 +-
 src/agent/src/rpc.rs                | 21 +++++++++++++++++----
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
index ed69ef60d..15b7d39ce 100644
--- a/src/agent/rustjail/src/container.rs
+++ b/src/agent/rustjail/src/container.rs
@@ -1482,12 +1482,11 @@ impl LinuxContainer {
             } else {
                 linux.cgroups_path.clone()
             }
+        } else if linux.cgroups_path.is_empty() {
+            format!("/{}", id.as_str())
         } else {
-            if linux.cgroups_path.is_empty() {
-                format!("/{}", id.as_str())
-            } else {
-                linux.cgroups_path.clone()
-            }
+            // if we have a systemd cgroup path we need to convert it to a fs cgroup path
+            linux.cgroups_path.replace(':', "/")
         };
 
         let cgroup_manager: Box<dyn Manager + Send + Sync> = if config.use_systemd_cgroup {
diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs
index 2c1f41def..d8e9fc828 100644
--- a/src/agent/src/main.rs
+++ b/src/agent/src/main.rs
@@ -339,7 +339,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
     server.start().await?;
 
     rx.await?;
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
index 008176541..3be8fed5c 100644
--- a/src/agent/src/rpc.rs
+++ b/src/agent/src/rpc.rs
@@ -137,6 +137,7 @@ macro_rules! is_allowed {
 #[derive(Clone, Debug)]
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
+    init_mode: bool,
 }
 
 impl AgentService {
@@ -213,8 +214,13 @@ impl AgentService {
         // determine which cgroup driver to take and then assign to use_systemd_cgroup
         // systemd: "[slice]:[prefix]:[name]"
         // fs: "/path_a/path_b"
+        // If agent is init we can't use systemd cgroup mode, no matter what the host tells us
         let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path);
-        let use_systemd_cgroup = SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path);
+        let use_systemd_cgroup = if self.init_mode {
+            false
+        } else {
+            SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path)
+        };
 
         let opts = CreateOpts {
             cgroup_name: "".to_string(),
@@ -1679,9 +1685,11 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
     Ok(content)
 }
 
-pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> Result<TtrpcServer> {
-    let agent_service =
-        Box::new(AgentService { sandbox: s }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
+pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
+    let agent_service = Box::new(AgentService {
+        sandbox: s,
+        init_mode,
+    }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
 
     let agent_worker = Arc::new(agent_service);
 
@@ -2157,6 +2165,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::UpdateInterfaceRequest::default();
@@ -2174,6 +2183,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::UpdateRoutesRequest::default();
@@ -2191,6 +2201,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::AddARPNeighborsRequest::default();
@@ -2324,6 +2335,7 @@ mod tests {
 
             let agent_service = Box::new(AgentService {
                 sandbox: Arc::new(Mutex::new(sandbox)),
+                init_mode: true,
             });
 
             let result = agent_service
@@ -2804,6 +2816,7 @@ OtherField:other
         let sandbox = Sandbox::new(&logger).unwrap();
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let ctx = mk_ttrpc_context();

From 262daaa2eff4ab725df498ea841f2893e4cc94e3 Mon Sep 17 00:00:00 2001
From: Bo Chen <chen.bo@intel.com>
Date: Fri, 24 Feb 2023 10:19:46 -0800
Subject: [PATCH 45/76] versions: Upgrade to Cloud Hypervisor v30.0

Details of this release can be found in our new roadmap project as
iteration v30.0: https://github.com/orgs/cloud-hypervisor/projects/6.

Fixes: #6375

Signed-off-by: Bo Chen <chen.bo@intel.com>
---
 versions.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.yaml b/versions.yaml
index 0099bd3be..174162b99 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -75,7 +75,7 @@ assets:
       url: "https://github.com/cloud-hypervisor/cloud-hypervisor"
       uscan-url: >-
         https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz
-      version: "v29.0"
+      version: "v30.0"
 
     firecracker:
       description: "Firecracker micro-VMM"

From 3ac6f29e95446e318b682d96bcba8a392ff5f392 Mon Sep 17 00:00:00 2001
From: Bo Chen <chen.bo@intel.com>
Date: Fri, 24 Feb 2023 10:20:29 -0800
Subject: [PATCH 46/76] runtime: clh: Re-generate the client code

This patch re-generates the client code for Cloud Hypervisor v30.0.
Note: The client code of cloud-hypervisor's OpenAPI is automatically
generated by openapi-generator.

Fixes: #6375

Signed-off-by: Bo Chen <chen.bo@intel.com>
---
 .../cloud-hypervisor/client/api/openapi.yaml  |  3 +
 .../client/docs/CpuAffinity.md                | 16 +----
 .../client/model_cpu_affinity.go              | 64 ++++++++-----------
 .../cloud-hypervisor/cloud-hypervisor.yaml    |  3 +
 4 files changed, 34 insertions(+), 52 deletions(-)

diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml
index 750c1e725..d514ae7b8 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml
@@ -1131,6 +1131,9 @@ components:
           items:
             type: integer
           type: array
+      required:
+      - host_cpus
+      - vcpu
       type: object
     CpuFeatures:
       example:
diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuAffinity.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuAffinity.md
index 300c7af1d..52eb24a18 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuAffinity.md
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuAffinity.md
@@ -4,14 +4,14 @@
 
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**Vcpu** | Pointer to **int32** |  | [optional] 
-**HostCpus** | Pointer to **[]int32** |  | [optional] 
+**Vcpu** | **int32** |  | 
+**HostCpus** | **[]int32** |  | 
 
 ## Methods
 
 ### NewCpuAffinity
 
-`func NewCpuAffinity() *CpuAffinity`
+`func NewCpuAffinity(vcpu int32, hostCpus []int32, ) *CpuAffinity`
 
 NewCpuAffinity instantiates a new CpuAffinity object
 This constructor will assign default values to properties that have it defined,
@@ -45,11 +45,6 @@ and a boolean to check if the value has been set.
 
 SetVcpu sets Vcpu field to given value.
 
-### HasVcpu
-
-`func (o *CpuAffinity) HasVcpu() bool`
-
-HasVcpu returns a boolean if a field has been set.
 
 ### GetHostCpus
 
@@ -70,11 +65,6 @@ and a boolean to check if the value has been set.
 
 SetHostCpus sets HostCpus field to given value.
 
-### HasHostCpus
-
-`func (o *CpuAffinity) HasHostCpus() bool`
-
-HasHostCpus returns a boolean if a field has been set.
 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_affinity.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_affinity.go
index f0009699f..889c8364f 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_affinity.go
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_affinity.go
@@ -16,16 +16,18 @@ import (
 
 // CpuAffinity struct for CpuAffinity
 type CpuAffinity struct {
-	Vcpu     *int32   `json:"vcpu,omitempty"`
-	HostCpus *[]int32 `json:"host_cpus,omitempty"`
+	Vcpu     int32   `json:"vcpu"`
+	HostCpus []int32 `json:"host_cpus"`
 }
 
 // NewCpuAffinity instantiates a new CpuAffinity object
 // This constructor will assign default values to properties that have it defined,
 // and makes sure properties required by API are set, but the set of arguments
 // will change when the set of required properties is changed
-func NewCpuAffinity() *CpuAffinity {
+func NewCpuAffinity(vcpu int32, hostCpus []int32) *CpuAffinity {
 	this := CpuAffinity{}
+	this.Vcpu = vcpu
+	this.HostCpus = hostCpus
 	return &this
 }
 
@@ -37,76 +39,60 @@ func NewCpuAffinityWithDefaults() *CpuAffinity {
 	return &this
 }
 
-// GetVcpu returns the Vcpu field value if set, zero value otherwise.
+// GetVcpu returns the Vcpu field value
 func (o *CpuAffinity) GetVcpu() int32 {
-	if o == nil || o.Vcpu == nil {
+	if o == nil {
 		var ret int32
 		return ret
 	}
-	return *o.Vcpu
+
+	return o.Vcpu
 }
 
-// GetVcpuOk returns a tuple with the Vcpu field value if set, nil otherwise
+// GetVcpuOk returns a tuple with the Vcpu field value
 // and a boolean to check if the value has been set.
 func (o *CpuAffinity) GetVcpuOk() (*int32, bool) {
-	if o == nil || o.Vcpu == nil {
+	if o == nil {
 		return nil, false
 	}
-	return o.Vcpu, true
+	return &o.Vcpu, true
 }
 
-// HasVcpu returns a boolean if a field has been set.
-func (o *CpuAffinity) HasVcpu() bool {
-	if o != nil && o.Vcpu != nil {
-		return true
-	}
-
-	return false
-}
-
-// SetVcpu gets a reference to the given int32 and assigns it to the Vcpu field.
+// SetVcpu sets field value
 func (o *CpuAffinity) SetVcpu(v int32) {
-	o.Vcpu = &v
+	o.Vcpu = v
 }
 
-// GetHostCpus returns the HostCpus field value if set, zero value otherwise.
+// GetHostCpus returns the HostCpus field value
 func (o *CpuAffinity) GetHostCpus() []int32 {
-	if o == nil || o.HostCpus == nil {
+	if o == nil {
 		var ret []int32
 		return ret
 	}
-	return *o.HostCpus
+
+	return o.HostCpus
 }
 
-// GetHostCpusOk returns a tuple with the HostCpus field value if set, nil otherwise
+// GetHostCpusOk returns a tuple with the HostCpus field value
 // and a boolean to check if the value has been set.
 func (o *CpuAffinity) GetHostCpusOk() (*[]int32, bool) {
-	if o == nil || o.HostCpus == nil {
+	if o == nil {
 		return nil, false
 	}
-	return o.HostCpus, true
+	return &o.HostCpus, true
 }
 
-// HasHostCpus returns a boolean if a field has been set.
-func (o *CpuAffinity) HasHostCpus() bool {
-	if o != nil && o.HostCpus != nil {
-		return true
-	}
-
-	return false
-}
-
-// SetHostCpus gets a reference to the given []int32 and assigns it to the HostCpus field.
+// SetHostCpus sets field value
 func (o *CpuAffinity) SetHostCpus(v []int32) {
-	o.HostCpus = &v
+	o.HostCpus = v
 }
 
 func (o CpuAffinity) MarshalJSON() ([]byte, error) {
 	toSerialize := map[string]interface{}{}
-	if o.Vcpu != nil {
+	if true {
 		toSerialize["vcpu"] = o.Vcpu
 	}
-	if o.HostCpus != nil {
+	if true {
 		toSerialize["host_cpus"] = o.HostCpus
 	}
 	return json.Marshal(toSerialize)
diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml
index dee8bdbf0..a2392f988 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml
@@ -578,6 +578,9 @@ components:
       description: Virtual machine configuration
 
     CpuAffinity:
+      required:
+        - vcpu
+        - host_cpus
       type: object
       properties:
         vcpu:

From ecac3a9e104aaef841579eb7af9a7228e4c3ebd4 Mon Sep 17 00:00:00 2001
From: Yushuo <y-shuo@linux.alibaba.com>
Date: Tue, 27 Dec 2022 17:53:04 +0800
Subject: [PATCH 47/76] docs: add design doc for Hooks

Fixes: #5787

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>
---
 docs/design/README.md         |  1 +
 docs/design/hooks-handling.md | 62 +++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 docs/design/hooks-handling.md

diff --git a/docs/design/README.md b/docs/design/README.md
index d2a9414ef..0c732defd 100644
--- a/docs/design/README.md
+++ b/docs/design/README.md
@@ -11,6 +11,7 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
+- [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)
diff --git a/docs/design/hooks-handling.md b/docs/design/hooks-handling.md
new file mode 100644
index 000000000..9a5edd907
--- /dev/null
+++ b/docs/design/hooks-handling.md
@@ -0,0 +1,62 @@
+# Kata Containers support for `Hooks`
+
+## Introduction
+
+During container's lifecycle, different Hooks can be executed to do custom actions. In Kata Containers, we support two types of Hooks, `OCI Hooks` and `Kata Hooks`.
+
+### OCI Hooks
+
+The OCI Spec stipulates six hooks that can be executed at different time points and namespaces, including `Prestart Hooks`, `CreateRuntime Hooks`, `CreateContainer Hooks`, `StartContainer Hooks`, `Poststart Hooks` and `Poststop Hooks`. We support these types of Hooks as compatible as possible in Kata Containers.
+
+The path and arguments of these hooks will be passed to Kata for execution via `bundle/config.json`. For example:
+```
+...
+"hooks": {
+  "prestart": [
+    {
+      "path": "/usr/bin/prestart-hook",
+      "args": ["prestart-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ],
+  "createRuntime": [
+    {
+      "path": "/usr/bin/createRuntime-hook",
+      "args": ["createRuntime-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ]
+}
+...
+```
+
+### Kata Hooks
+
+In Kata, we support another three kinds of hooks executed in guest VM, including `Guest Prestart Hook`, `Guest Poststart Hook`, `Guest Poststop Hook`.
+
+The executable files for Kata Hooks must be packaged in the *guest rootfs*. The file path to those guest hooks should be specified in the configuration file, and guest hooks must be stored in a subdirectory of `guest_hook_path` according to their hook type. For example:
+
++ In configuration file:
+```
+guest_hook_path="/usr/share/hooks"
+```
++ In guest rootfs, prestart-hook is stored in `/usr/share/hooks/prestart/prestart-hook`.
+
+## Execution
+The table below summarized when and where those different hooks will be executed in Kata Containers:
+
+| Hook Name | Hook Type | Hook Path | Exec Place | Exec Time |
+|---|---|---|---|---|
+| `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
+| `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
+| `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
+| `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
+| `Guest Prestart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, before container command is executed. |
+| `Guest Poststart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, after container command is executed, before start operation returns. |
+| `Guest Poststop` | Kata hook | guest agent namespace | guest agent namespace | During delete operation, after container is deleted, before delete operation returns. |
+
++ `Hook Path` specifies where hook's path be resolved.
++ `Exec Place` specifies in which namespace those hooks can be executed.
++ `Exec Time` specifies at which time point those hooks can be executed.
\ No newline at end of file

From 875f2db5284bed78a31c738fadcf44e0565ad942 Mon Sep 17 00:00:00 2001
From: Yushuo <y-shuo@linux.alibaba.com>
Date: Fri, 18 Nov 2022 19:25:09 +0800
Subject: [PATCH 48/76] runtime-rs: add oci hook support

According to the runtime OCI Spec, there can be some hook
operations in the lifecycle of the container. In these hook
operations, the runtime can execute some commands. There are different
points in time in the container lifecycle  and different hook types
can be executed.

In this commit, we are now supporting 4 types of hooks(same in
runtime-go): Prestart hook, CreateRuntime hook, Poststart hook and
Poststop hook.

Fixes: #5787

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>
---
 src/agent/rustjail/src/lib.rs                 |  5 ++
 src/dragonball/src/api/v1/instance_info.rs    |  4 ++
 src/libs/oci/src/lib.rs                       |  3 ++
 src/libs/protocols/protos/oci.proto           |  3 ++
 src/libs/protocols/src/trans.rs               |  6 +++
 src/runtime-rs/Cargo.lock                     |  5 ++
 .../hypervisor/src/ch/inner_hypervisor.rs     |  4 ++
 .../crates/hypervisor/src/ch/mod.rs           |  5 ++
 .../src/dragonball/inner_hypervisor.rs        |  5 ++
 .../crates/hypervisor/src/dragonball/mod.rs   |  5 ++
 .../hypervisor/src/dragonball/vmm_instance.rs | 10 ++++
 src/runtime-rs/crates/hypervisor/src/lib.rs   |  1 +
 .../crates/hypervisor/src/qemu/inner.rs       |  5 ++
 .../crates/hypervisor/src/qemu/mod.rs         |  5 ++
 src/runtime-rs/crates/runtimes/Cargo.toml     |  3 ++
 .../crates/runtimes/common/Cargo.toml         |  1 +
 .../crates/runtimes/common/src/sandbox.rs     |  3 ++
 src/runtime-rs/crates/runtimes/src/manager.rs | 51 +++++++++++++++++--
 .../src/container_manager/container.rs        | 11 ++++
 .../src/container_manager/manager.rs          | 50 +++++++++++++++++-
 .../crates/runtimes/virt_container/src/lib.rs | 11 ++--
 .../runtimes/virt_container/src/sandbox.rs    |  4 ++
 22 files changed, 192 insertions(+), 8 deletions(-)

diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs
index 6f96d18c2..9ca124b1f 100644
--- a/src/agent/rustjail/src/lib.rs
+++ b/src/agent/rustjail/src/lib.rs
@@ -154,12 +154,15 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
 fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
 
+    let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
+
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
 
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
     oci::Hooks {
         prestart,
+        create_runtime,
         poststart,
         poststop,
     }
@@ -860,6 +863,7 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    ..Default::default()
                 },
             },
             TestData {
@@ -908,6 +912,7 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    ..Default::default()
                 },
             },
         ];
diff --git a/src/dragonball/src/api/v1/instance_info.rs b/src/dragonball/src/api/v1/instance_info.rs
index 86174a2fd..45d03b414 100644
--- a/src/dragonball/src/api/v1/instance_info.rs
+++ b/src/dragonball/src/api/v1/instance_info.rs
@@ -50,6 +50,8 @@ pub struct InstanceInfo {
     pub vmm_version: String,
     /// The pid of the current VMM process.
     pub pid: u32,
+    /// The tid of the current VMM master thread.
+    pub master_tid: u32,
     /// The state of async actions.
     pub async_state: AsyncState,
     /// List of tids of vcpu threads (vcpu index, tid)
@@ -66,6 +68,7 @@ impl InstanceInfo {
             state: InstanceState::Uninitialized,
             vmm_version,
             pid: std::process::id(),
+            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
@@ -80,6 +83,7 @@ impl Default for InstanceInfo {
             state: InstanceState::Uninitialized,
             vmm_version: env!("CARGO_PKG_VERSION").to_string(),
             pid: std::process::id(),
+            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
diff --git a/src/libs/oci/src/lib.rs b/src/libs/oci/src/lib.rs
index 3bcaefa16..7a31662a1 100644
--- a/src/libs/oci/src/lib.rs
+++ b/src/libs/oci/src/lib.rs
@@ -193,6 +193,8 @@ pub struct Hooks {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub prestart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub create_runtime: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststop: Vec<Hook>,
@@ -1401,6 +1403,7 @@ mod tests {
                     env: vec![],
                     timeout: None,
                 }],
+                ..Default::default()
             }),
             annotations: [
                 ("com.example.key1".to_string(), "value1".to_string()),
diff --git a/src/libs/protocols/protos/oci.proto b/src/libs/protocols/protos/oci.proto
index aa0db0123..43dea0657 100644
--- a/src/libs/protocols/protos/oci.proto
+++ b/src/libs/protocols/protos/oci.proto
@@ -166,6 +166,9 @@ message Hooks {
 
 	// Poststop is a list of hooks to be run after the container process exits.
 	repeated Hook Poststop = 3  [(gogoproto.nullable) = false];
+
+	// Createruntime is a list of hooks to be run during the creation of runtime(sandbox).
+	repeated Hook CreateRuntime = 4  [(gogoproto.nullable) = false];
 }
 
 message Hook {
diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs
index c6f0c64f0..3c1aa6f66 100644
--- a/src/libs/protocols/src/trans.rs
+++ b/src/libs/protocols/src/trans.rs
@@ -294,6 +294,7 @@ impl From<oci::Hooks> for crate::oci::Hooks {
     fn from(from: Hooks) -> Self {
         crate::oci::Hooks {
             Prestart: from_vec(from.prestart),
+            CreateRuntime: from_vec(from.create_runtime),
             Poststart: from_vec(from.poststart),
             Poststop: from_vec(from.poststop),
             unknown_fields: Default::default(),
@@ -974,6 +975,10 @@ impl From<crate::oci::Hooks> for oci::Hooks {
         for hook in from.take_Prestart().to_vec() {
             prestart.push(hook.into())
         }
+        let mut create_runtime = Vec::new();
+        for hook in from.take_CreateRuntime().to_vec() {
+            create_runtime.push(hook.into())
+        }
         let mut poststart = Vec::new();
         for hook in from.take_Poststart().to_vec() {
             poststart.push(hook.into());
@@ -984,6 +989,7 @@ impl From<crate::oci::Hooks> for oci::Hooks {
         }
         oci::Hooks {
             prestart,
+            create_runtime,
             poststart,
             poststop,
         }
diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock
index fbff64e87..9ba66e764 100644
--- a/src/runtime-rs/Cargo.lock
+++ b/src/runtime-rs/Cargo.lock
@@ -1703,6 +1703,8 @@ dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
  "libc",
+ "memoffset 0.6.5",
+ "pin-utils",
 ]
 
 [[package]]
@@ -2428,12 +2430,15 @@ dependencies = [
  "hyper",
  "hyperlocal",
  "hypervisor",
+ "kata-sys-util",
  "kata-types",
  "lazy_static",
  "linux_container",
  "logging",
+ "nix 0.25.1",
  "oci",
  "persist",
+ "serde_json",
  "shim-interface",
  "slog",
  "slog-scope",
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
index 8eea6f466..9cd05de91 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
@@ -472,6 +472,10 @@ impl CloudHypervisorInner {
         Ok(Vec::<u32>::new())
     }
 
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        todo!()
+    }
+
     pub(crate) async fn check(&self) -> Result<()> {
         Ok(())
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
index 8e754415d..7805d2601 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
@@ -118,6 +118,11 @@ impl Hypervisor for CloudHypervisor {
         inner.get_pids().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
     async fn check(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.check().await
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
index d4d75e6ef..fc3bea7a5 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
@@ -127,6 +127,11 @@ impl DragonballInner {
         Ok(Vec::from_iter(pids.into_iter()))
     }
 
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let master_tid = self.vmm_instance.get_vmm_master_tid();
+        Ok(master_tid)
+    }
+
     pub(crate) async fn check(&self) -> Result<()> {
         Ok(())
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
index 2886043c8..d096abf54 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
@@ -117,6 +117,11 @@ impl Hypervisor for Dragonball {
         inner.get_pids().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
     async fn check(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.check().await
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
index f0d613938..8e7d6332f 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
@@ -75,6 +75,12 @@ impl VmmInstance {
         share_info_lock.write().unwrap().id = String::from(id);
     }
 
+    pub fn get_vmm_master_tid(&self) -> u32 {
+        let info = self.vmm_shared_info.clone();
+        let result = info.read().unwrap().master_tid;
+        result
+    }
+
     pub fn get_vcpu_tids(&self) -> Vec<(u8, u32)> {
         let info = self.vmm_shared_info.clone();
         let result = info.read().unwrap().tids.clone();
@@ -103,6 +109,7 @@ impl VmmInstance {
             Some(kvm.into_raw_fd()),
         )
         .expect("Failed to start vmm");
+        let vmm_shared_info = self.get_shared_info();
 
         self.vmm_thread = Some(
             thread::Builder::new()
@@ -110,6 +117,9 @@ impl VmmInstance {
                 .spawn(move || {
                     || -> Result<i32> {
                         debug!(sl!(), "run vmm thread start");
+                        let cur_tid = nix::unistd::gettid().as_raw() as u32;
+                        vmm_shared_info.write().unwrap().master_tid = cur_tid;
+
                         if let Some(netns_path) = netns {
                             info!(sl!(), "set netns for vmm master {}", &netns_path);
                             let netns_fd = File::open(&netns_path)
diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs
index 3c417f195..e7f18e513 100644
--- a/src/runtime-rs/crates/hypervisor/src/lib.rs
+++ b/src/runtime-rs/crates/hypervisor/src/lib.rs
@@ -87,6 +87,7 @@ pub trait Hypervisor: Send + Sync {
     async fn hypervisor_config(&self) -> HypervisorConfig;
     async fn get_thread_ids(&self) -> Result<VcpuThreadIds>;
     async fn get_pids(&self) -> Result<Vec<u32>>;
+    async fn get_vmm_master_tid(&self) -> Result<u32>;
     async fn cleanup(&self) -> Result<()>;
     async fn check(&self) -> Result<()>;
     async fn get_jailer_root(&self) -> Result<String>;
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
index e298e801b..995a9c590 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
@@ -89,6 +89,11 @@ impl QemuInner {
         todo!()
     }
 
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        info!(sl!(), "QemuInner::get_vmm_master_tid()");
+        todo!()
+    }
+
     pub(crate) async fn cleanup(&self) -> Result<()> {
         info!(sl!(), "QemuInner::cleanup()");
         todo!()
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
index 6df386923..0192e2a8e 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
@@ -103,6 +103,11 @@ impl Hypervisor for Qemu {
         inner.get_thread_ids().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
     async fn cleanup(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.cleanup().await
diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml
index 6ff6b71a1..142c44ed0 100644
--- a/src/runtime-rs/crates/runtimes/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/Cargo.toml
@@ -13,9 +13,12 @@ slog-scope = "4.4.0"
 tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
 hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
 hyperlocal = "0.8"
+serde_json = "1.0.88"
+nix = "0.25.0"
 
 common = { path = "./common" }
 kata-types = { path = "../../../libs/kata-types" }
+kata-sys-util = { path = "../../../libs/kata-sys-util" }
 logging = { path = "../../../libs/logging"}
 oci = { path = "../../../libs/oci" }
 shim-interface = { path = "../../../libs/shim-interface" }
diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml
index 00eb64825..78a640e95 100644
--- a/src/runtime-rs/crates/runtimes/common/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml
@@ -26,3 +26,4 @@ agent = { path = "../../agent" }
 kata-sys-util = { path = "../../../../libs/kata-sys-util" }
 kata-types = { path = "../../../../libs/kata-types" }
 oci = { path = "../../../../libs/oci" }
+
diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
index 4a5e49cba..56dc95087 100644
--- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
@@ -17,6 +17,9 @@ pub trait Sandbox: Send + Sync {
     // agent function
     async fn agent_sock(&self) -> Result<String>;
 
+    // hypervisor function
+    async fn get_vmm_master_tid(&self) -> Result<u32>;
+
     // utils
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>>;
     async fn get_iptables(&self, is_ipv6: bool) -> Result<Vec<u8>>;
diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs
index 49193cb5c..ba2b0b0b0 100644
--- a/src/runtime-rs/crates/runtimes/src/manager.rs
+++ b/src/runtime-rs/crates/runtimes/src/manager.rs
@@ -18,6 +18,8 @@ use hypervisor::Param;
 use kata_types::{
     annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig,
 };
+use kata_sys_util::hooks::HookStates;
+
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
 use persist::sandbox_persist::Persist;
@@ -81,7 +83,12 @@ impl RuntimeHandlerManagerInner {
         Ok(())
     }
 
-    async fn try_init(&mut self, spec: &oci::Spec, options: &Option<Vec<u8>>) -> Result<()> {
+    async fn try_init(
+        &mut self,
+        spec: &oci::Spec,
+        state: &oci::State,
+        options: &Option<Vec<u8>>,
+    ) -> Result<()> {
         // return if runtime instance has init
         if self.runtime_instance.is_some() {
             return Ok(());
@@ -125,6 +132,35 @@ impl RuntimeHandlerManagerInner {
             .await
             .context("init runtime handler")?;
 
+        let mut st = state.clone();
+        if let Some(runtime_instance) = self.runtime_instance.clone() {
+            let vmm_master_tid = runtime_instance
+                .sandbox
+                .get_vmm_master_tid()
+                .await
+                .context("get vmm master tid")?;
+            st.pid = vmm_master_tid as i32;
+        }
+
+        // Prestart Hooks [DEPRECATED in newest oci spec]:
+        // * should be run in runtime namespace
+        // * should be run after vm is started, but before container is created
+        //      if Prestart Hook and CreateRuntime Hook are both supported
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart
+        if let Some(hooks) = spec.hooks.as_ref() {
+            let mut prestart_hook_states = HookStates::new();
+            prestart_hook_states.execute_hooks(&hooks.prestart, Some(st.clone()))?
+        }
+
+        // CreateRuntime Hooks:
+        // * should be run in runtime namespace
+        // * should be run when creating the runtime
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
+        if let Some(hooks) = spec.hooks.as_ref() {
+            let mut create_runtime_hook_states = HookStates::new();
+            create_runtime_hook_states.execute_hooks(&hooks.create_runtime, Some(st.clone()))?
+        }
+
         // the sandbox creation can reach here only once and the sandbox is created
         // so we can safely create the shim management socket right now
         // the unwrap here is safe because the runtime handler is correctly created
@@ -207,10 +243,11 @@ impl RuntimeHandlerManager {
     async fn try_init_runtime_instance(
         &self,
         spec: &oci::Spec,
+        state: &oci::State,
         options: &Option<Vec<u8>>,
     ) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.try_init(spec, options).await
+        inner.try_init(spec, state, options).await
     }
 
     pub async fn handler_message(&self, req: Request) -> Result<Response> {
@@ -222,8 +259,16 @@ impl RuntimeHandlerManager {
                 oci::OCI_SPEC_CONFIG_FILE_NAME
             );
             let spec = oci::Spec::load(&bundler_path).context("load spec")?;
+            let state = oci::State {
+                version: spec.version.clone(),
+                id: container_config.container_id.to_string(),
+                status: oci::ContainerState::Creating,
+                pid: 0,
+                bundle: bundler_path,
+                annotations: spec.annotations.clone(),
+            };
 
-            self.try_init_runtime_instance(&spec, &container_config.options)
+            self.try_init_runtime_instance(&spec, &state, &container_config.options)
                 .await
                 .context("try init runtime instance")?;
             let instance = self
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
index 93ff1a499..3289b4f29 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
@@ -37,6 +37,7 @@ pub struct Container {
     pid: u32,
     pub container_id: ContainerID,
     config: ContainerConfig,
+    spec: oci::Spec,
     inner: Arc<RwLock<ContainerInner>>,
     agent: Arc<dyn Agent>,
     resource_manager: Arc<ResourceManager>,
@@ -47,6 +48,7 @@ impl Container {
     pub fn new(
         pid: u32,
         config: ContainerConfig,
+        spec: oci::Spec,
         agent: Arc<dyn Agent>,
         resource_manager: Arc<ResourceManager>,
     ) -> Result<Self> {
@@ -67,6 +69,7 @@ impl Container {
             pid,
             container_id,
             config,
+            spec,
             inner: Arc::new(RwLock::new(ContainerInner::new(
                 agent.clone(),
                 init_process,
@@ -382,6 +385,14 @@ impl Container {
             .context("agent update container")?;
         Ok(())
     }
+
+    pub async fn config(&self) -> ContainerConfig {
+        self.config.clone()
+    }
+
+    pub async fn spec(&self) -> oci::Spec {
+        self.spec.clone()
+    }
 }
 
 fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> {
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs
index ba73c17d5..326614e9b 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs
@@ -5,11 +5,10 @@
 //
 
 use anyhow::{anyhow, Context, Result};
-
+use async_trait::async_trait;
 use std::{collections::HashMap, sync::Arc};
 
 use agent::Agent;
-use async_trait::async_trait;
 use common::{
     error::Error,
     types::{
@@ -19,10 +18,13 @@ use common::{
     },
     ContainerManager,
 };
+use hypervisor::Hypervisor;
 use oci::Process as OCIProcess;
 use resource::ResourceManager;
 use tokio::sync::RwLock;
 
+use kata_sys_util::hooks::HookStates;
+
 use super::{logger_with_process, Container};
 
 pub struct VirtContainerManager {
@@ -31,6 +33,7 @@ pub struct VirtContainerManager {
     containers: Arc<RwLock<HashMap<String, Container>>>,
     resource_manager: Arc<ResourceManager>,
     agent: Arc<dyn Agent>,
+    hypervisor: Arc<dyn Hypervisor>,
 }
 
 impl VirtContainerManager {
@@ -38,6 +41,7 @@ impl VirtContainerManager {
         sid: &str,
         pid: u32,
         agent: Arc<dyn Agent>,
+        hypervisor: Arc<dyn Hypervisor>,
         resource_manager: Arc<ResourceManager>,
     ) -> Self {
         Self {
@@ -46,6 +50,7 @@ impl VirtContainerManager {
             containers: Default::default(),
             resource_manager,
             agent,
+            hypervisor,
         }
     }
 }
@@ -56,6 +61,7 @@ impl ContainerManager for VirtContainerManager {
         let container = Container::new(
             self.pid,
             config,
+            spec.clone(),
             self.agent.clone(),
             self.resource_manager.clone(),
         )
@@ -87,6 +93,26 @@ impl ContainerManager for VirtContainerManager {
                 let c = containers
                     .remove(container_id)
                     .ok_or_else(|| Error::ContainerNotFound(container_id.to_string()))?;
+
+                // Poststop Hooks:
+                // * should be run in runtime namespace
+                // * should be run after the container is deleted but before delete operation returns
+                // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#poststop
+                let c_spec = c.spec().await;
+                let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
+                let state = oci::State {
+                    version: c_spec.version.clone(),
+                    id: c.container_id.to_string(),
+                    status: oci::ContainerState::Stopped,
+                    pid: vmm_master_tid as i32,
+                    bundle: c.config().await.bundle,
+                    annotations: c_spec.annotations.clone(),
+                };
+                if let Some(hooks) = c_spec.hooks.as_ref() {
+                    let mut poststop_hook_states = HookStates::new();
+                    poststop_hook_states.execute_hooks(&hooks.poststop, Some(state))?;
+                }
+
                 c.state_process(process).await.context("state process")
             }
             ProcessType::Exec => {
@@ -190,6 +216,26 @@ impl ContainerManager for VirtContainerManager {
             .get(container_id)
             .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?;
         c.start(process).await.context("start")?;
+
+        // Poststart Hooks:
+        // * should be run in runtime namespace
+        // * should be run after user-specific command is executed but before start operation returns
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#poststart
+        let c_spec = c.spec().await;
+        let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
+        let state = oci::State {
+            version: c_spec.version.clone(),
+            id: c.container_id.to_string(),
+            status: oci::ContainerState::Running,
+            pid: vmm_master_tid as i32,
+            bundle: c.config().await.bundle,
+            annotations: c_spec.annotations.clone(),
+        };
+        if let Some(hooks) = c_spec.hooks.as_ref() {
+            let mut poststart_hook_states = HookStates::new();
+            poststart_hook_states.execute_hooks(&hooks.poststart, Some(state))?;
+        }
+
         Ok(PID { pid: self.pid })
     }
 
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs
index b73caa849..3d1381c71 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs
@@ -86,13 +86,18 @@ impl RuntimeHandler for VirtContainer {
             sid,
             msg_sender,
             agent.clone(),
-            hypervisor,
+            hypervisor.clone(),
             resource_manager.clone(),
         )
         .await
         .context("new virt sandbox")?;
-        let container_manager =
-            container_manager::VirtContainerManager::new(sid, pid, agent, resource_manager);
+        let container_manager = container_manager::VirtContainerManager::new(
+            sid,
+            pid,
+            agent,
+            hypervisor,
+            resource_manager,
+        );
         Ok(RuntimeInstance {
             sandbox: Arc::new(sandbox),
             container_manager: Arc::new(container_manager),
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
index be7577ecc..91372c881 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
@@ -278,6 +278,10 @@ impl Sandbox for VirtSandbox {
         self.agent.agent_sock().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        self.hypervisor.get_vmm_master_tid().await
+    }
+
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>> {
         info!(sl!(), "sb: set_iptables invoked");
         let req = SetIPTablesRequest { is_ipv6, data };

From 977f281c5c083e6aa5f342b78c35c2783fbf5071 Mon Sep 17 00:00:00 2001
From: Yushuo <y-shuo@linux.alibaba.com>
Date: Mon, 5 Dec 2022 16:49:52 +0800
Subject: [PATCH 49/76] runtime-rs: add CreateContainer hook support

CreateContainer hook is one kind of OCI hook. In kata, it will be
executed after VM is started, before container is created, and after
CreateRuntime is executed.

The hook path of CreateContainer hook is in host runtime namespace, but
it will be executed in host vmm namespace.

Fixes: #5787

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>
---
 src/agent/rustjail/src/lib.rs                 |  3 +++
 src/libs/oci/src/lib.rs                       |  2 ++
 src/libs/protocols/protos/oci.proto           |  3 +++
 src/libs/protocols/src/trans.rs               |  6 +++++
 .../crates/resource/src/network/mod.rs        |  3 ++-
 .../resource/src/network/utils/netns.rs       |  4 +--
 .../src/container_manager/manager.rs          | 27 ++++++++++++++++++-
 7 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs
index 9ca124b1f..c791e001c 100644
--- a/src/agent/rustjail/src/lib.rs
+++ b/src/agent/rustjail/src/lib.rs
@@ -156,6 +156,8 @@ fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
 
     let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
 
+    let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref());
+
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
 
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
@@ -163,6 +165,7 @@ fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     oci::Hooks {
         prestart,
         create_runtime,
+        create_container,
         poststart,
         poststop,
     }
diff --git a/src/libs/oci/src/lib.rs b/src/libs/oci/src/lib.rs
index 7a31662a1..6b0e905d6 100644
--- a/src/libs/oci/src/lib.rs
+++ b/src/libs/oci/src/lib.rs
@@ -195,6 +195,8 @@ pub struct Hooks {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub create_runtime: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub create_container: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststop: Vec<Hook>,
diff --git a/src/libs/protocols/protos/oci.proto b/src/libs/protocols/protos/oci.proto
index 43dea0657..e0048beba 100644
--- a/src/libs/protocols/protos/oci.proto
+++ b/src/libs/protocols/protos/oci.proto
@@ -169,6 +169,9 @@ message Hooks {
 
 	// Createruntime is a list of hooks to be run during the creation of runtime(sandbox).
 	repeated Hook CreateRuntime = 4  [(gogoproto.nullable) = false];
+
+	// CreateContainer is a list of hooks to be run after VM is started, and before container is created.
+	repeated Hook CreateContainer = 5  [(gogoproto.nullable) = false];
 }
 
 message Hook {
diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs
index 3c1aa6f66..79f68ed05 100644
--- a/src/libs/protocols/src/trans.rs
+++ b/src/libs/protocols/src/trans.rs
@@ -295,6 +295,7 @@ impl From<oci::Hooks> for crate::oci::Hooks {
         crate::oci::Hooks {
             Prestart: from_vec(from.prestart),
             CreateRuntime: from_vec(from.create_runtime),
+            CreateContainer: from_vec(from.create_container),
             Poststart: from_vec(from.poststart),
             Poststop: from_vec(from.poststop),
             unknown_fields: Default::default(),
@@ -979,6 +980,10 @@ impl From<crate::oci::Hooks> for oci::Hooks {
         for hook in from.take_CreateRuntime().to_vec() {
             create_runtime.push(hook.into())
         }
+        let mut create_container = Vec::new();
+        for hook in from.take_CreateContainer().to_vec() {
+            create_container.push(hook.into())
+        }
         let mut poststart = Vec::new();
         for hook in from.take_Poststart().to_vec() {
             poststart.push(hook.into());
@@ -990,6 +995,7 @@ impl From<crate::oci::Hooks> for oci::Hooks {
         oci::Hooks {
             prestart,
             create_runtime,
+            create_container,
             poststart,
             poststop,
         }
diff --git a/src/runtime-rs/crates/resource/src/network/mod.rs b/src/runtime-rs/crates/resource/src/network/mod.rs
index e572ecc74..a85c2213d 100644
--- a/src/runtime-rs/crates/resource/src/network/mod.rs
+++ b/src/runtime-rs/crates/resource/src/network/mod.rs
@@ -5,6 +5,7 @@
 //
 
 mod endpoint;
+pub use endpoint::endpoint_persist::EndpointState;
 pub use endpoint::Endpoint;
 mod network_entity;
 mod network_info;
@@ -17,7 +18,7 @@ use network_with_netns::NetworkWithNetns;
 mod network_pair;
 use network_pair::NetworkPair;
 mod utils;
-pub use endpoint::endpoint_persist::EndpointState;
+pub use utils::netns::NetnsGuard;
 
 use std::sync::Arc;
 
diff --git a/src/runtime-rs/crates/resource/src/network/utils/netns.rs b/src/runtime-rs/crates/resource/src/network/utils/netns.rs
index 07584c641..bb0343dff 100644
--- a/src/runtime-rs/crates/resource/src/network/utils/netns.rs
+++ b/src/runtime-rs/crates/resource/src/network/utils/netns.rs
@@ -10,12 +10,12 @@ use anyhow::{Context, Result};
 use nix::sched::{setns, CloneFlags};
 use nix::unistd::{getpid, gettid};
 
-pub(crate) struct NetnsGuard {
+pub struct NetnsGuard {
     old_netns: Option<File>,
 }
 
 impl NetnsGuard {
-    pub(crate) fn new(new_netns_path: &str) -> Result<Self> {
+    pub fn new(new_netns_path: &str) -> Result<Self> {
         let old_netns = if !new_netns_path.is_empty() {
             let current_netns_path = format!("/proc/{}/task/{}/ns/{}", getpid(), gettid(), "net");
             let old_netns = File::open(&current_netns_path)
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs
index 326614e9b..53bd36894 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs
@@ -20,6 +20,7 @@ use common::{
 };
 use hypervisor::Hypervisor;
 use oci::Process as OCIProcess;
+use resource::network::NetnsGuard;
 use resource::ResourceManager;
 use tokio::sync::RwLock;
 
@@ -60,13 +61,37 @@ impl ContainerManager for VirtContainerManager {
     async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result<PID> {
         let container = Container::new(
             self.pid,
-            config,
+            config.clone(),
             spec.clone(),
             self.agent.clone(),
             self.resource_manager.clone(),
         )
         .context("new container")?;
 
+        // CreateContainer Hooks:
+        // * should be run in vmm namespace (hook path in runtime namespace)
+        // * should be run after the vm is started, before container is created, and after CreateRuntime Hooks
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createcontainer-hooks
+        let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
+        let vmm_netns_path = format!("/proc/{}/task/{}/ns/{}", self.pid, vmm_master_tid, "net");
+        let state = oci::State {
+            version: spec.version.clone(),
+            id: config.container_id.clone(),
+            status: oci::ContainerState::Creating,
+            pid: vmm_master_tid as i32,
+            bundle: config.bundle.clone(),
+            annotations: spec.annotations.clone(),
+        };
+
+        // new scope, CreateContainer hooks in which will execute in a new network namespace
+        {
+            let _netns_guard = NetnsGuard::new(&vmm_netns_path).context("vmm netns guard")?;
+            if let Some(hooks) = spec.hooks.as_ref() {
+                let mut create_container_hook_states = HookStates::new();
+                create_container_hook_states.execute_hooks(&hooks.create_container, Some(state))?;
+            }
+        }
+
         let mut containers = self.containers.write().await;
         container.create(spec).await.context("create")?;
         containers.insert(container.container_id.to_string(), container);

From e80c9f7b742df64437ab014055fe8e2e1b22fcbd Mon Sep 17 00:00:00 2001
From: Yushuo <y-shuo@linux.alibaba.com>
Date: Mon, 5 Dec 2022 17:20:18 +0800
Subject: [PATCH 50/76] runtime-rs: add StartContainer hook

StartContainer will be execute in guest container namespace in Kata.
The Hook Path of this kind of hook is also in guest container namespace.

StartContainer is executed after start operation is called, and it
should be executed before user-specific command is executed.

Fixes: #5787

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>
---
 src/agent/rustjail/src/container.rs           | 28 +++++++++++++++++--
 src/agent/rustjail/src/lib.rs                 |  6 ++--
 src/libs/oci/src/lib.rs                       |  2 ++
 src/libs/protocols/protos/oci.proto           |  3 ++
 src/libs/protocols/src/trans.rs               |  6 ++++
 src/runtime-rs/crates/agent/src/types.rs      |  1 -
 .../src/container_manager/container.rs        | 16 +++++++++--
 7 files changed, 53 insertions(+), 9 deletions(-)

diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
index f75b6e6d5..0ded5a093 100644
--- a/src/agent/rustjail/src/container.rs
+++ b/src/agent/rustjail/src/container.rs
@@ -374,13 +374,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
     let spec: oci::Spec = serde_json::from_str(spec_str)?;
-
     log_child!(cfd_log, "notify parent to send oci process");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
     let buf = read_sync(crfd)?;
     let process_str = std::str::from_utf8(&buf)?;
     let oci_process: oci::Process = serde_json::from_str(process_str)?;
+    log_child!(cfd_log, "notify parent to send oci state");
+    write_sync(cwfd, SYNC_SUCCESS, "")?;
+
+    let buf = read_sync(crfd)?;
+    let state_str = std::str::from_utf8(&buf)?;
+    let mut state: oci::State = serde_json::from_str(state_str)?;
     log_child!(cfd_log, "notify parent to send cgroup manager");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
@@ -743,6 +748,19 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, buf)?;
     }
 
+    if init {
+        // StartContainer Hooks:
+        // * should be run in container namespace
+        // * should be run after container is created and before container is started (before user-specific command is executed)
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#startcontainer-hooks
+        state.pid = std::process::id() as i32;
+        state.status = oci::ContainerState::Created;
+        if let Some(hooks) = spec.hooks.as_ref() {
+            let mut start_container_states = HookStates::new();
+            start_container_states.execute_hooks(&hooks.start_container, Some(state))?;
+        }
+    }
+
     // With NoNewPrivileges, we should set seccomp as close to
     // do_exec as possible in order to reduce the amount of
     // system calls in the seccomp profiles.
@@ -1323,7 +1341,6 @@ async fn join_namespaces(
     write_async(pipe_w, SYNC_DATA, spec_str.as_str()).await?;
 
     info!(logger, "wait child received oci spec");
-
     read_async(pipe_r).await?;
 
     info!(logger, "send oci process from parent to child");
@@ -1333,6 +1350,13 @@ async fn join_namespaces(
     info!(logger, "wait child received oci process");
     read_async(pipe_r).await?;
 
+    info!(logger, "try to send state from parent to child");
+    let state_str = serde_json::to_string(st)?;
+    write_async(pipe_w, SYNC_DATA, state_str.as_str()).await?;
+
+    info!(logger, "wait child received oci state");
+    read_async(pipe_r).await?;
+
     let cm_str = if use_systemd_cgroup {
         serde_json::to_string(cm.as_any()?.downcast_ref::<SystemdManager>().unwrap())
     } else {
diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs
index c791e001c..8af5fe929 100644
--- a/src/agent/rustjail/src/lib.rs
+++ b/src/agent/rustjail/src/lib.rs
@@ -153,19 +153,17 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
 
 fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
-
     let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
-
     let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref());
-
+    let start_container = hook_grpc_to_oci(h.StartContainer.as_ref());
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
-
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
     oci::Hooks {
         prestart,
         create_runtime,
         create_container,
+        start_container,
         poststart,
         poststop,
     }
diff --git a/src/libs/oci/src/lib.rs b/src/libs/oci/src/lib.rs
index 6b0e905d6..1c70410cf 100644
--- a/src/libs/oci/src/lib.rs
+++ b/src/libs/oci/src/lib.rs
@@ -197,6 +197,8 @@ pub struct Hooks {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub create_container: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub start_container: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststop: Vec<Hook>,
diff --git a/src/libs/protocols/protos/oci.proto b/src/libs/protocols/protos/oci.proto
index e0048beba..fcaef487f 100644
--- a/src/libs/protocols/protos/oci.proto
+++ b/src/libs/protocols/protos/oci.proto
@@ -172,6 +172,9 @@ message Hooks {
 
 	// CreateContainer is a list of hooks to be run after VM is started, and before container is created.
 	repeated Hook CreateContainer = 5  [(gogoproto.nullable) = false];
+
+	// StartContainer is a list of hooks to be run after container is created, but before it is started.
+	repeated Hook StartContainer = 6  [(gogoproto.nullable) = false];
 }
 
 message Hook {
diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs
index 79f68ed05..abd89f924 100644
--- a/src/libs/protocols/src/trans.rs
+++ b/src/libs/protocols/src/trans.rs
@@ -296,6 +296,7 @@ impl From<oci::Hooks> for crate::oci::Hooks {
             Prestart: from_vec(from.prestart),
             CreateRuntime: from_vec(from.create_runtime),
             CreateContainer: from_vec(from.create_container),
+            StartContainer: from_vec(from.start_container),
             Poststart: from_vec(from.poststart),
             Poststop: from_vec(from.poststop),
             unknown_fields: Default::default(),
@@ -984,6 +985,10 @@ impl From<crate::oci::Hooks> for oci::Hooks {
         for hook in from.take_CreateContainer().to_vec() {
             create_container.push(hook.into())
         }
+        let mut start_container = Vec::new();
+        for hook in from.take_StartContainer().to_vec() {
+            start_container.push(hook.into())
+        }
         let mut poststart = Vec::new();
         for hook in from.take_Poststart().to_vec() {
             poststart.push(hook.into());
@@ -996,6 +1001,7 @@ impl From<crate::oci::Hooks> for oci::Hooks {
             prestart,
             create_runtime,
             create_container,
+            start_container,
             poststart,
             poststop,
         }
diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs
index b8e79589a..0cd509ff5 100644
--- a/src/runtime-rs/crates/agent/src/types.rs
+++ b/src/runtime-rs/crates/agent/src/types.rs
@@ -124,7 +124,6 @@ pub struct CreateContainerRequest {
     pub devices: Vec<Device>,
     pub storages: Vec<Storage>,
     pub oci: Option<oci::Spec>,
-    pub guest_hooks: Option<oci::Hooks>,
     pub sandbox_pidns: bool,
     pub rootfs_mounts: Vec<oci::Mount>,
 }
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
index 3289b4f29..501002e42 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
@@ -396,8 +396,20 @@ impl Container {
 }
 
 fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> {
-    // hook should be done on host
-    spec.hooks = None;
+    // Only the StartContainer hook needs to be reserved for execution in the guest
+    let start_container_hooks = match spec.hooks.as_ref() {
+        Some(hooks) => hooks.start_container.clone(),
+        None => Vec::new(),
+    };
+
+    spec.hooks = if start_container_hooks.is_empty() {
+        None
+    } else {
+        Some(oci::Hooks {
+            start_container: start_container_hooks,
+            ..Default::default()
+        })
+    };
 
     // special process K8s ephemeral volumes.
     update_ephemeral_storage_type(spec);

From 2c4428ee02479076dadb702c8d19d39a1401f8e1 Mon Sep 17 00:00:00 2001
From: Yushuo <y-shuo@linux.alibaba.com>
Date: Thu, 5 Jan 2023 16:17:28 +0800
Subject: [PATCH 51/76] runtime-rs: move pre-start hooks to sandbox_start

In some cases, network endpoints will be configured through Prestart
Hook. So network endpoints may need to be added(hotpluged) after vm
is started and also Prestart Hook is executed.

We move pre-start hook functions' execution to sandbox_start to allow
hooks running between vm_start and netns_scan easily, so that the
lifecycle API can be cleaner.

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>
---
 docs/design/hooks-handling.md                 |  3 +-
 .../crates/runtimes/common/src/sandbox.rs     | 11 ++--
 src/runtime-rs/crates/runtimes/src/manager.rs | 36 ++----------
 .../runtimes/virt_container/src/sandbox.rs    | 57 +++++++++++++++++--
 4 files changed, 65 insertions(+), 42 deletions(-)

diff --git a/docs/design/hooks-handling.md b/docs/design/hooks-handling.md
index 9a5edd907..535c70cc4 100644
--- a/docs/design/hooks-handling.md
+++ b/docs/design/hooks-handling.md
@@ -49,7 +49,7 @@ The table below summarized when and where those different hooks will be executed
 |---|---|---|---|---|
 | `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
 | `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
-| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
 | `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
 | `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
 | `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
@@ -59,4 +59,5 @@ The table below summarized when and where those different hooks will be executed
 
 + `Hook Path` specifies where hook's path be resolved.
 + `Exec Place` specifies in which namespace those hooks can be executed.
+  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
 + `Exec Time` specifies at which time point those hooks can be executed.
\ No newline at end of file
diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
index 56dc95087..3fee8165d 100644
--- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs
@@ -9,7 +9,13 @@ use async_trait::async_trait;
 
 #[async_trait]
 pub trait Sandbox: Send + Sync {
-    async fn start(&self, netns: Option<String>, dns: Vec<String>) -> Result<()>;
+    async fn start(
+        &self,
+        netns: Option<String>,
+        dns: Vec<String>,
+        spec: &oci::Spec,
+        state: &oci::State,
+    ) -> Result<()>;
     async fn stop(&self) -> Result<()>;
     async fn cleanup(&self) -> Result<()>;
     async fn shutdown(&self) -> Result<()>;
@@ -17,9 +23,6 @@ pub trait Sandbox: Send + Sync {
     // agent function
     async fn agent_sock(&self) -> Result<String>;
 
-    // hypervisor function
-    async fn get_vmm_master_tid(&self) -> Result<u32>;
-
     // utils
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>>;
     async fn get_iptables(&self, is_ipv6: bool) -> Result<Vec<u8>>;
diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs
index ba2b0b0b0..e904718b9 100644
--- a/src/runtime-rs/crates/runtimes/src/manager.rs
+++ b/src/runtime-rs/crates/runtimes/src/manager.rs
@@ -18,7 +18,6 @@ use hypervisor::Param;
 use kata_types::{
     annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig,
 };
-use kata_sys_util::hooks::HookStates;
 
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
@@ -52,6 +51,8 @@ impl RuntimeHandlerManagerInner {
 
     async fn init_runtime_handler(
         &mut self,
+        spec: &oci::Spec,
+        state: &oci::State,
         netns: Option<String>,
         dns: Vec<String>,
         config: Arc<TomlConfig>,
@@ -76,7 +77,7 @@ impl RuntimeHandlerManagerInner {
         // start sandbox
         runtime_instance
             .sandbox
-            .start(netns, dns)
+            .start(netns, dns, spec, state)
             .await
             .context("start sandbox")?;
         self.runtime_instance = Some(Arc::new(runtime_instance));
@@ -128,39 +129,10 @@ impl RuntimeHandlerManagerInner {
         }
 
         let config = load_config(spec, options).context("load config")?;
-        self.init_runtime_handler(netns, dns, Arc::new(config))
+        self.init_runtime_handler(spec, state, netns, dns, Arc::new(config))
             .await
             .context("init runtime handler")?;
 
-        let mut st = state.clone();
-        if let Some(runtime_instance) = self.runtime_instance.clone() {
-            let vmm_master_tid = runtime_instance
-                .sandbox
-                .get_vmm_master_tid()
-                .await
-                .context("get vmm master tid")?;
-            st.pid = vmm_master_tid as i32;
-        }
-
-        // Prestart Hooks [DEPRECATED in newest oci spec]:
-        // * should be run in runtime namespace
-        // * should be run after vm is started, but before container is created
-        //      if Prestart Hook and CreateRuntime Hook are both supported
-        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart
-        if let Some(hooks) = spec.hooks.as_ref() {
-            let mut prestart_hook_states = HookStates::new();
-            prestart_hook_states.execute_hooks(&hooks.prestart, Some(st.clone()))?
-        }
-
-        // CreateRuntime Hooks:
-        // * should be run in runtime namespace
-        // * should be run when creating the runtime
-        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
-        if let Some(hooks) = spec.hooks.as_ref() {
-            let mut create_runtime_hook_states = HookStates::new();
-            create_runtime_hook_states.execute_hooks(&hooks.create_runtime, Some(st.clone()))?
-        }
-
         // the sandbox creation can reach here only once and the sandbox is created
         // so we can safely create the shim management socket right now
         // the unwrap here is safe because the runtime handler is correctly created
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
index 91372c881..0d6e4765e 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
@@ -17,6 +17,7 @@ use common::{
 };
 use containerd_shim_protos::events::task::TaskOOM;
 use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL};
+use kata_sys_util::hooks::HookStates;
 use kata_types::config::{
     default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT},
     TomlConfig,
@@ -117,11 +118,50 @@ impl VirtSandbox {
 
         Ok(resource_configs)
     }
+
+    async fn execute_oci_hook_functions(
+        &self,
+        prestart_hooks: &[oci::Hook],
+        create_runtime_hooks: &[oci::Hook],
+        state: &oci::State,
+    ) -> Result<()> {
+        let mut st = state.clone();
+        // for dragonball, we use vmm_master_tid
+        let vmm_pid = self
+            .hypervisor
+            .get_vmm_master_tid()
+            .await
+            .context("get vmm master tid")?;
+        st.pid = vmm_pid as i32;
+
+        // Prestart Hooks [DEPRECATED in newest oci spec]:
+        // * should be run in runtime namespace
+        // * should be run after vm is started, but before container is created
+        //      if Prestart Hook and CreateRuntime Hook are both supported
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart
+        let mut prestart_hook_states = HookStates::new();
+        prestart_hook_states.execute_hooks(prestart_hooks, Some(st.clone()))?;
+
+        // CreateRuntime Hooks:
+        // * should be run in runtime namespace
+        // * should be run when creating the runtime
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
+        let mut create_runtime_hook_states = HookStates::new();
+        create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?;
+
+        Ok(())
+    }
 }
 
 #[async_trait]
 impl Sandbox for VirtSandbox {
-    async fn start(&self, netns: Option<String>, dns: Vec<String>) -> Result<()> {
+    async fn start(
+        &self,
+        netns: Option<String>,
+        dns: Vec<String>,
+        spec: &oci::Spec,
+        state: &oci::State,
+    ) -> Result<()> {
         let id = &self.sid;
 
         // if sandbox running, return
@@ -149,6 +189,17 @@ impl Sandbox for VirtSandbox {
         self.hypervisor.start_vm(10_000).await.context("start vm")?;
         info!(sl!(), "start vm");
 
+        // execute pre-start hook functions, including Prestart Hooks and CreateRuntime Hooks
+        let (prestart_hooks, create_runtime_hooks) = match spec.hooks.as_ref() {
+            Some(hooks) => (hooks.prestart.clone(), hooks.create_runtime.clone()),
+            None => (Vec::new(), Vec::new()),
+        };
+        self.execute_oci_hook_functions(&prestart_hooks, &create_runtime_hooks, state)
+            .await?;
+
+        // TODO: if prestart_hooks is not empty, rescan the network endpoints(rely on hotplug endpoints).
+        // see: https://github.com/kata-containers/kata-containers/issues/6378
+
         // connect agent
         // set agent socket
         let address = self
@@ -278,10 +329,6 @@ impl Sandbox for VirtSandbox {
         self.agent.agent_sock().await
     }
 
-    async fn get_vmm_master_tid(&self) -> Result<u32> {
-        self.hypervisor.get_vmm_master_tid().await
-    }
-
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>> {
         info!(sl!(), "sb: set_iptables invoked");
         let req = SetIPTablesRequest { is_ipv6, data };

From 4b8a5a1a3df6e460add53577ac301ebb37c2a5fe Mon Sep 17 00:00:00 2001
From: Alec Pemberton <pembek1901@gmail.com>
Date: Fri, 24 Feb 2023 12:33:14 -0600
Subject: [PATCH 52/76] utils: Remove kata-manager.sh cgroups v2 check

Removed the part in the `kata-manager.sh` script that checks if the host system only runs cgroups v2.

Fixes: #6259.

Signed-off-by: Alec Pemberton <pembek1901@gmail.com>
---
 utils/kata-manager.sh | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh
index e28c4a44b..8d78aa447 100755
--- a/utils/kata-manager.sh
+++ b/utils/kata-manager.sh
@@ -238,23 +238,6 @@ Advice:
 EOF
 }
 
-# Determine if the system only supports cgroups v2.
-#
-# - Writes "true" to stdout if only cgroups v2 are supported.
-# - Writes "false" to stdout if cgroups v1 or v1+v2 are available.
-# - Writes a blank string to stdout if cgroups are not available.
-only_supports_cgroups_v2()
-{
-	local v1=$(mount|awk '$5 ~ /^cgroup$/ { print; }' || true)
-	local v2=$(mount|awk '$5 ~ /^cgroup2$/ { print; }' || true)
-
-	[ -n "$v1" ] && [ -n "$v2" ] && { echo "false"; return 0; } || true
-	[ -n "$v1" ] && { echo "false"; return 0; } || true
-	[ -n "$v2" ] && { echo "true"; return 0; } || true
-
-	return 0
-}
-
 # Return 0 if containerd is already installed, else return 1.
 containerd_installed()
 {
@@ -280,13 +263,6 @@ pre_checks()
 
 	[ "$ret" -eq 0 ] && die "$containerd_project already installed"
 
-	local cgroups_v2_only=$(only_supports_cgroups_v2 || true)
-
-	local url="https://github.com/kata-containers/kata-containers/issues/927"
-
-	[ "$cgroups_v2_only" = "true" ] && \
-		die "$kata_project does not yet fully support cgroups v2 - see $url"
-
 	return 0
 }
 

From 828d467222d4c1b161dcf8bf9acba0634e7ccf08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 28 Feb 2023 23:48:37 +0100
Subject: [PATCH 53/76] workflows: Do not install docker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The latest ubuntu runners already have docker installed and trying to
install it manually will cause the following issue:
```
Run curl -fsSL https://test.docker.com/ -o test-docker.sh
Warning: the "docker" command appears to already exist on this system.

If you already have Docker installed, this script can cause trouble, which is
why we're displaying this warning and provide the opportunity to cancel the
installation.

If you installed the current Docker package using this script and are using it
again to update Docker, you can safely ignore this message.

You may press Ctrl+C now to abort this script.
+ sleep 20
+ sudo -E sh -c apt-get update -qq >/dev/null
E: The repository 'https://packages.microsoft.com/ubuntu/22.04/prod jammy Release' is no longer signed.
```

Fixes: #6390

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/kata-deploy-push.yaml | 6 ------
 .github/workflows/kata-deploy-test.yaml | 5 -----
 .github/workflows/release.yaml          | 5 -----
 3 files changed, 16 deletions(-)

diff --git a/.github/workflows/kata-deploy-push.yaml b/.github/workflows/kata-deploy-push.yaml
index c7d7e8cb4..ce45ab5ab 100644
--- a/.github/workflows/kata-deploy-push.yaml
+++ b/.github/workflows/kata-deploy-push.yaml
@@ -29,12 +29,6 @@ jobs:
           - nydus
     steps:
       - uses: actions/checkout@v2
-      - name: Install docker
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |
diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml
index c19c7bb4a..6b30109c2 100644
--- a/.github/workflows/kata-deploy-test.yaml
+++ b/.github/workflows/kata-deploy-test.yaml
@@ -72,11 +72,6 @@ jobs:
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
 
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index dd48494f1..87a5992c1 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -22,11 +22,6 @@ jobs:
           - virtiofsd
     steps:
       - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh

From 12cfad4858539450f237c91133cf3941c5179204 Mon Sep 17 00:00:00 2001
From: Yushuo <y-shuo@linux.alibaba.com>
Date: Wed, 1 Mar 2023 10:35:10 +0800
Subject: [PATCH 54/76] runtime-rs: modify the transfer to oci::Hooks

In this commit, we have done:
    * modify the tranfer process from grpc::Hooks to oci::Hooks, so the code
      can be more clean
    * add more tests for create_runtime, create_container, start_container hooks

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>
---
 src/agent/rustjail/src/lib.rs   | 116 +++++++++++++++++++++++++++++++-
 src/libs/protocols/src/trans.rs |  47 +++++++------
 2 files changed, 137 insertions(+), 26 deletions(-)

diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs
index 8af5fe929..18ee0c333 100644
--- a/src/agent/rustjail/src/lib.rs
+++ b/src/agent/rustjail/src/lib.rs
@@ -835,6 +835,45 @@ mod tests {
                         Timeout: 10,
                         ..Default::default()
                     }])),
+                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -864,7 +903,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
-                    ..Default::default()
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
             TestData {
@@ -897,6 +953,45 @@ mod tests {
                         Timeout: 10,
                         ..Default::default()
                     }])),
+                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -913,7 +1008,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
-                    ..Default::default()
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
         ];
diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs
index abd89f924..1e1514788 100644
--- a/src/libs/protocols/src/trans.rs
+++ b/src/libs/protocols/src/trans.rs
@@ -973,30 +973,29 @@ impl From<crate::oci::Hook> for oci::Hook {
 
 impl From<crate::oci::Hooks> for oci::Hooks {
     fn from(mut from: crate::oci::Hooks) -> Self {
-        let mut prestart = Vec::new();
-        for hook in from.take_Prestart().to_vec() {
-            prestart.push(hook.into())
-        }
-        let mut create_runtime = Vec::new();
-        for hook in from.take_CreateRuntime().to_vec() {
-            create_runtime.push(hook.into())
-        }
-        let mut create_container = Vec::new();
-        for hook in from.take_CreateContainer().to_vec() {
-            create_container.push(hook.into())
-        }
-        let mut start_container = Vec::new();
-        for hook in from.take_StartContainer().to_vec() {
-            start_container.push(hook.into())
-        }
-        let mut poststart = Vec::new();
-        for hook in from.take_Poststart().to_vec() {
-            poststart.push(hook.into());
-        }
-        let mut poststop = Vec::new();
-        for hook in from.take_Poststop().to_vec() {
-            poststop.push(hook.into());
-        }
+        let prestart = from.take_Prestart().into_iter().map(|i| i.into()).collect();
+        let create_runtime = from
+            .take_CreateRuntime()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let create_container = from
+            .take_CreateContainer()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let start_container = from
+            .take_StartContainer()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let poststart = from
+            .take_Poststart()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let poststop = from.take_Poststop().into_iter().map(|i| i.into()).collect();
+
         oci::Hooks {
             prestart,
             create_runtime,

From b6880c60d38e19fec32655dedcd0a45fd9cb4037 Mon Sep 17 00:00:00 2001
From: yanggang <gang.yang@daocloud.io>
Date: Thu, 23 Feb 2023 16:41:38 +0800
Subject: [PATCH 55/76] logging: Correct the code notes

Fix wrong notes for func GetSandboxesStoragePathRust()

Fixes: #6394

Signed-off-by: yanggang <gang.yang@daocloud.io>
---
 src/runtime/pkg/containerd-shim-v2/shim_management.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime/pkg/containerd-shim-v2/shim_management.go b/src/runtime/pkg/containerd-shim-v2/shim_management.go
index 74c750918..f9c31b8b2 100644
--- a/src/runtime/pkg/containerd-shim-v2/shim_management.go
+++ b/src/runtime/pkg/containerd-shim-v2/shim_management.go
@@ -307,7 +307,7 @@ func GetSandboxesStoragePath() string {
 	return "/run/vc/sbs"
 }
 
-// GetSandboxesStoragePath returns the storage path where sandboxes info are stored in runtime-rs
+// GetSandboxesStoragePathRust returns the storage path where sandboxes info are stored in runtime-rs
 func GetSandboxesStoragePathRust() string {
 	return "/run/kata"
 }

From a9e2fc86786e954fe71fb491fd027f900bb52c25 Mon Sep 17 00:00:00 2001
From: "Eduardo Lima (Etrunko)" <etrunko@redhat.com>
Date: Tue, 24 Jan 2023 14:42:31 -0300
Subject: [PATCH 56/76] runtime/Makefile: Fix install-containerd-shim-v2
 dependency

$ make install
make: *** No rule to make target 'containerd-shim-kata-v2', needed by 'install-containerd-shim-v2'.  Stop.

Spotted when building kata-runtime with a different name for
SHIMV2_OUTPUT. For instance, trying to keep different runtime binaries
installed at the same time, one from master and another from lets say,
the CCv0 branch, with the following small change applied.

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index 95efaff78..2bab9eb75 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -231,7 +231,7 @@ SED = sed

 CLI_DIR = cmd
 SHIMV2 = containerd-shim-kata-v2
-SHIMV2_OUTPUT = $(bCURDIR)/$(SHIMV2)
+SHIMV2_OUTPUT = $(CURDIR)/$(SHIMV2)-ccv0
 SHIMV2_DIR = $(CLI_DIR)/$(SHIMV2)

 MONITOR = kata-monitor

Fixes: #6398

Signed-off-by: Eduardo Lima (Etrunko) <etrunko@redhat.com>
---
 src/runtime/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index ab9611968..95efaff78 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -667,7 +667,7 @@ install-bin: $(BINLIST)
 
 install-runtime: runtime install-scripts install-completions install-configs install-bin
 
-install-containerd-shim-v2: $(SHIMV2)
+install-containerd-shim-v2: $(SHIMV2_OUTPUT)
 	$(QUIET_INST)$(call INSTALL_EXEC,$<,$(BINDIR))
 
 install-monitor: $(MONITOR)

From fea7e8816fa5590f16bc97c344b38eea22db6391 Mon Sep 17 00:00:00 2001
From: Domesticcadiz <christopher.cadiz.robinson@gmail.com>
Date: Wed, 1 Mar 2023 09:37:17 -0600
Subject: [PATCH 57/76] runtime-rs: Fixed typo mod.rs

Fixed the typo in comment in the delete method located in mod.rs file.

Fixes: #6256.

Signed-off-by: Domesticcadiz <christopher.cadiz.robinson@gmail.com>
---
 src/runtime-rs/crates/resource/src/cgroups/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs
index 7787d2ad0..b7f515d7f 100644
--- a/src/runtime-rs/crates/resource/src/cgroups/mod.rs
+++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs
@@ -114,7 +114,7 @@ impl CgroupsResource {
     pub async fn delete(&self) -> Result<()> {
         for cg_pid in self.cgroup_manager.tasks() {
             // For now, we can't guarantee that the thread in cgroup_manager does still
-            // exist. Once it exit, we should ignor that error returned by remove_task
+            // exist. Once it exit, we should ignore that error returned by remove_task
             // to let it go.
             if let Err(error) = self.cgroup_manager.remove_task(cg_pid) {
                 match error.source() {

From bd1ed26c8d0ef8da99bc2ffffb099f93f73a3880 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 21 Feb 2023 12:19:21 +0100
Subject: [PATCH 58/76] workflows: Publish kata-deploy payload after a merge
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For the architectures we know that `make kata-tarball` works as
expected, let's start publishing the kata-deploy payload after each
merge.

This will help to:
* Easily test the content of current `main` or `stable-*` branch
* Easily bisect issues
* Start providing some sort of CI/CD content pipeline for those who
  need that

This is a forward-port work from the `CCv0` and groups together patches
that I've worked on, with the work that Choi did in order to support
different architectures.

Fixes: #6343

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../workflows/payload-after-push-amd64.yaml   |  96 ++++++++++++++++
 .../workflows/payload-after-push-arm64.yaml   | 108 ++++++++++++++++++
 .../workflows/payload-after-push-s390x.yaml   | 107 +++++++++++++++++
 .github/workflows/payload-after-push.yaml     |  47 ++++++++
 .../kata-deploy-build-and-upload-payload.sh   |  50 ++++++++
 5 files changed, 408 insertions(+)
 create mode 100644 .github/workflows/payload-after-push-amd64.yaml
 create mode 100644 .github/workflows/payload-after-push-arm64.yaml
 create mode 100644 .github/workflows/payload-after-push-s390x.yaml
 create mode 100644 .github/workflows/payload-after-push.yaml
 create mode 100755 tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh

diff --git a/.github/workflows/payload-after-push-amd64.yaml b/.github/workflows/payload-after-push-amd64.yaml
new file mode 100644
index 000000000..30b4303c9
--- /dev/null
+++ b/.github/workflows/payload-after-push-amd64.yaml
@@ -0,0 +1,96 @@
+name: CI | Publish kata-deploy payload for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-latest
+    needs: build-asset
+    steps:
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+          "kata-containers-${{ inputs.target-arch }}"
diff --git a/.github/workflows/payload-after-push-arm64.yaml b/.github/workflows/payload-after-push-arm64.yaml
new file mode 100644
index 000000000..0dc11a665
--- /dev/null
+++ b/.github/workflows/payload-after-push-arm64.yaml
@@ -0,0 +1,108 @@
+name: CI | Publish kata-deploy payload for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: arm64
+    strategy:
+      matrix:
+        asset:
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - nydus
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: arm64
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: arm64
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+          "kata-containers-${{ inputs.target-arch }}"
diff --git a/.github/workflows/payload-after-push-s390x.yaml b/.github/workflows/payload-after-push-s390x.yaml
new file mode 100644
index 000000000..0919da2a0
--- /dev/null
+++ b/.github/workflows/payload-after-push-s390x.yaml
@@ -0,0 +1,107 @@
+name: CI | Publish kata-deploy payload for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    strategy:
+      matrix:
+        asset:
+          - kernel
+          - shim-v2
+          - qemu
+          - rootfs-image
+          - rootfs-initrd
+          - virtiofsd
+    steps:
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R $(id -u):$(id -g) "kata-build"
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: yes
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs: build-asset
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-artifacts-s390x
+          path: kata-artifacts
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
+      - name: store-artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  kata-payload:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+          "kata-containers-${{ inputs.target-arch }}"
diff --git a/.github/workflows/payload-after-push.yaml b/.github/workflows/payload-after-push.yaml
new file mode 100644
index 000000000..c121f8d9f
--- /dev/null
+++ b/.github/workflows/payload-after-push.yaml
@@ -0,0 +1,47 @@
+name: CI | Publish Kata Containers payload
+on:
+  push:
+    branches:
+      - main
+      - stable-*
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
+
+  build-assets-arm64:
+    uses: ./.github/workflows/cc-payload-after-push-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [build-assets-amd64, build-assets-arm64, build-assets-s390x]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Confidential Containers quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          docker manifest create quay.io/kata-containers/kata-deploy-ci:kata-containers-latest \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-amd64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-arm64 \
+          --amend quay.io/kata-containers/kata-deploy-ci:kata-containers-s390x
+          docker manifest push quay.io/kata-containers/kata-deploy-ci:kata-containers-latest
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
new file mode 100755
index 000000000..29ea59b17
--- /dev/null
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Intel
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+KATA_DEPLOY_DIR="`dirname ${0}`/../../kata-deploy-cc"
+KATA_DEPLOY_ARTIFACT="${1:-"kata-static.tar.xz"}"
+REGISTRY="${2:-"quay.io/kata-containers/kata-deploy"}"
+TAG="${3:-}"
+
+echo "Copying ${KATA_DEPLOY_ARTIFACT} to ${KATA_DEPLOY_DIR}"
+cp ${KATA_DEPLOY_ARTIFACT} ${KATA_DEPLOY_DIR}
+
+pushd ${KATA_DEPLOY_DIR}
+
+IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-$(uname -m)"
+
+echo "Building the image"
+if [ "$(uname -m)" = "s390x" ]; then
+	docker build \
+		--build-arg IMG_NAME=clefos \
+		--build-arg IMG_TAG=7 \
+		--tag ${IMAGE_TAG} .
+else
+	docker build --tag ${IMAGE_TAG} .
+fi
+
+echo "Pushing the image to quay.io"
+docker push ${IMAGE_TAG}
+
+if [ -n "${TAG}" ]; then
+	ADDITIONAL_TAG="${REGISTRY}:${TAG}"
+
+	echo "Building the ${ADDITIONAL_TAG} image"
+	if [ "$(uname -m)" = "s390x" ]; then
+		docker build \
+			--build-arg IMG_NAME=clefos \
+			--build-arg IMG_TAG=7 \
+			--tag ${ADDITIONAL_TAG} .
+	else
+		docker build --tag ${ADDITIONAL_TAG} .
+	fi
+
+	echo "Pushing the image ${ADDITIONAL_TAG} to quay.io"
+	docker push ${ADDITIONAL_TAG}
+fi
+
+popd

From dd2713521e3a8aac782e3163b0dd56e1479cce58 Mon Sep 17 00:00:00 2001
From: Chao Wu <chaowu@linux.alibaba.com>
Date: Mon, 13 Feb 2023 11:45:53 +0800
Subject: [PATCH 59/76] Dragonball: update dependencies

Since rust-vmm and dragonball-sandbox has introduced several updates
such as vPMU support for aarch64, we also need to update Dragonball
dependencies to include those changes.

Update:
virtio-queue to v0.6.0
kvm-ioctls to v0.12.0
dbs-upcall to v0.2.0
dbs-virtio-devices to v0.2.0
kvm-bindings to v0.6.0

Also, several aarch64 features are updated because of dependencies
changes:
1. update vcpu hotplug API.
2. update vpmu related API.
3. adjust unit test cases for aarch64 Dragonball.

fixes: #6268

Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
---
 src/dragonball/Cargo.lock                |  863 +++++++---------
 src/dragonball/Cargo.toml                |   10 +-
 src/dragonball/src/device_manager/mod.rs |    8 +-
 src/dragonball/src/vcpu/vcpu_impl.rs     |  140 +--
 src/dragonball/src/vcpu/vcpu_manager.rs  |    7 +
 src/dragonball/src/vm/aarch64.rs         |    4 +-
 src/dragonball/src/vm/mod.rs             |    1 +
 src/runtime-rs/Cargo.lock                | 1201 ++++++++++++----------
 8 files changed, 1101 insertions(+), 1133 deletions(-)

diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock
index 048d9c896..c5be35bfb 100644
--- a/src/dragonball/Cargo.lock
+++ b/src/dragonball/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.17.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
 dependencies = [
  "gimli",
 ]
@@ -19,15 +19,15 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
 name = "anyhow"
-version = "1.0.66"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
 
 [[package]]
 name = "arc-swap"
-version = "1.5.1"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"
+checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
 
 [[package]]
 name = "arrayref"
@@ -47,7 +47,7 @@ version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.1.19",
  "libc",
  "winapi",
 ]
@@ -60,9 +60,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "backtrace"
-version = "0.3.66"
+version = "0.3.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
+checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
 dependencies = [
  "addr2line",
  "cc",
@@ -81,9 +81,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "blake3"
-version = "1.3.1"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
+checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -102,12 +102,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "bumpalo"
-version = "3.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
-
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -116,9 +110,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
-version = "1.2.1"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
 [[package]]
 name = "caps"
@@ -132,9 +126,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.76"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 dependencies = [
  "jobserver",
 ]
@@ -146,10 +140,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
-name = "constant_time_eq"
-version = "0.1.5"
+name = "cmake"
+version = "0.1.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "constant_time_eq"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279"
 
 [[package]]
 name = "core-foundation-sys"
@@ -177,9 +180,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -187,9 +190,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.12"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac"
+checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
 dependencies = [
  "cfg-if",
 ]
@@ -204,16 +207,6 @@ dependencies = [
  "typenum",
 ]
 
-[[package]]
-name = "dashmap"
-version = "4.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
-dependencies = [
- "cfg-if",
- "num_cpus",
-]
-
 [[package]]
 name = "dbs-address-space"
 version = "0.2.2"
@@ -222,10 +215,10 @@ checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc"
 dependencies = [
  "arc-swap",
  "libc",
- "nix 0.23.1",
+ "nix 0.23.2",
  "thiserror",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -239,23 +232,24 @@ dependencies = [
 
 [[package]]
 name = "dbs-arch"
-version = "0.2.1"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f89357fc97fb3608473073be037ea0b22787b1fa4c68b8eb3dd51f3c5fd6b41"
+checksum = "194c844946cd1d13f7a9eb29b84afbc5354578eee2b06fea96226bc3872e7424"
 dependencies = [
  "kvm-bindings",
  "kvm-ioctls",
  "libc",
  "memoffset",
+ "thiserror",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-boot"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e6930547e688d8527705d1b7c4163c090c8535b8dd526d8251aa4dfdcbf2f82"
+checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108"
 dependencies = [
  "dbs-arch",
  "kvm-bindings",
@@ -278,15 +272,16 @@ dependencies = [
 
 [[package]]
 name = "dbs-interrupt"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f217820329cea9d8d2870f9cdda426c5ca4379e33283c39338841a86bdc36c"
+checksum = "1eb2c5bb9f8f123ace33b1b2e8d53dd2d87331ee770ad1f82e56c3382c6bed6d"
 dependencies = [
+ "dbs-arch",
  "dbs-device",
  "kvm-bindings",
  "kvm-ioctls",
  "libc",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -300,24 +295,14 @@ dependencies = [
  "log",
  "serde",
  "vm-superio",
- "vmm-sys-util 0.11.0",
-]
-
-[[package]]
-name = "dbs-uhttp"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fd0544fe7ba81fa8deb8800843836d279a81b051e2e8ab046fe1b0cb096c1cc"
-dependencies = [
- "libc",
- "mio",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-upcall"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2fa8b67657cd71779eaceea1b5fa989b62a1be629a07be8498417772e5a8d35"
+checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f"
 dependencies = [
  "anyhow",
  "dbs-utils",
@@ -340,14 +325,14 @@ dependencies = [
  "serde",
  "thiserror",
  "timerfd",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-virtio-devices"
-version = "0.1.1"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f70cc3a62fa1c169beca6921ef0d3cf38fdfe7cd732ac76c8517bc8a3df9338"
+checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668"
 dependencies = [
  "byteorder",
  "caps",
@@ -361,7 +346,8 @@ dependencies = [
  "kvm-ioctls",
  "libc",
  "log",
- "nix 0.23.1",
+ "nix 0.23.2",
+ "nydus-api",
  "nydus-blobfs",
  "nydus-rafs",
  "rlimit",
@@ -372,14 +358,14 @@ dependencies = [
  "virtio-bindings",
  "virtio-queue",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "digest"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c"
+checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
 dependencies = [
  "block-buffer",
  "crypto-common",
@@ -430,7 +416,7 @@ dependencies = [
  "libc",
  "linux-loader",
  "log",
- "nix 0.24.2",
+ "nix 0.24.3",
  "seccompiler",
  "serde",
  "serde_derive",
@@ -443,7 +429,7 @@ dependencies = [
  "thiserror",
  "virtio-queue",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -484,53 +470,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "377fa591135fbe23396a18e2655a6d5481bf7c5823cdfa3cc81b01a229cbe640"
 dependencies = [
  "libc",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
+]
+
+[[package]]
+name = "filetime"
+version = "0.2.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "windows-sys",
 ]
 
 [[package]]
 name = "flate2"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
 dependencies = [
  "crc32fast",
+ "libz-sys",
  "miniz_oxide",
 ]
 
-[[package]]
-name = "fnv"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-
-[[package]]
-name = "form_urlencoded"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
-dependencies = [
- "percent-encoding",
-]
-
 [[package]]
 name = "fuse-backend-rs"
-version = "0.9.6"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "994a3bfb694ee52bf8f3bca80d784b723f150810998219337e429cc5dbe92717"
+checksum = "08af89cb80a7c5693bd63a2b1ee7ac31a307670977c18fda036b3aa94be8c47f"
 dependencies = [
  "arc-swap",
  "bitflags",
  "caps",
  "core-foundation-sys",
- "io-uring",
  "lazy_static",
  "libc",
  "log",
  "mio",
- "nix 0.24.2",
- "scoped-tls",
- "slab",
- "socket2",
+ "nix 0.24.3",
  "tokio-uring",
  "virtio-queue",
  "vm-memory",
@@ -539,9 +519,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
+checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -554,9 +534,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
+checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -564,15 +544,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
+checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"
+checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -581,15 +561,15 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
+checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
+checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -598,27 +578,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
+checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
 
 [[package]]
 name = "futures-task"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
-
-[[package]]
-name = "futures-timer"
-version = "3.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
+checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
 
 [[package]]
 name = "futures-util"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
+checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -650,31 +624,14 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
 ]
 
 [[package]]
 name = "gimli"
-version = "0.26.2"
+version = "0.27.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
-
-[[package]]
-name = "governor"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7df0ee4b237afb71e99f7e2fbd840ffec2d6c4bb569f69b2af18aa1f63077d38"
-dependencies = [
- "dashmap",
- "futures",
- "futures-timer",
- "no-std-compat",
- "nonzero_ext",
- "parking_lot",
- "quanta",
- "rand",
- "smallvec",
-]
+checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
 
 [[package]]
 name = "hermit-abi"
@@ -686,52 +643,41 @@ dependencies = [
 ]
 
 [[package]]
-name = "http"
-version = "0.2.8"
+name = "hermit-abi"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
 dependencies = [
- "bytes",
- "fnv",
- "itoa",
+ "libc",
 ]
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
 [[package]]
 name = "httpdate"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
-[[package]]
-name = "idna"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
-]
-
-[[package]]
-name = "instant"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
-dependencies = [
- "cfg-if",
-]
-
 [[package]]
 name = "io-lifetimes"
-version = "0.6.1"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504"
+checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
 
 [[package]]
 name = "io-uring"
-version = "0.5.9"
+version = "0.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ba34abb5175052fc1a2227a10d2275b7386c9990167de9786c0b88d8b062330"
+checksum = "dd1e1a01cfb924fd8c5c43b6827965db394f5a3a16c599ce03452266e1cf984c"
 dependencies = [
  "bitflags",
  "libc",
@@ -739,46 +685,37 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
 [[package]]
 name = "jobserver"
-version = "0.1.25"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b"
+checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
 dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "js-sys"
-version = "0.3.60"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
-dependencies = [
- "wasm-bindgen",
-]
-
 [[package]]
 name = "kvm-bindings"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a78c049190826fff959994b7c1d8a2930d0a348f1b8f3aa4f9bb34cd5d7f2952"
+checksum = "efe70e65a5b092161d17f5005b66e5eefe7a94a70c332e755036fc4af78c4e79"
 dependencies = [
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "kvm-ioctls"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97422ba48d7ffb66fd4d18130f72ab66f9bbbf791fb7a87b9291cdcfec437593"
+checksum = "c3a321cabd827642499c77e27314f388dd83a717a5ca716b86476fb947f73ae4"
 dependencies = [
  "kvm-bindings",
  "libc",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -788,10 +725,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
-name = "libc"
-version = "0.2.137"
+name = "leaky-bucket"
+version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
+checksum = "79e0d52231827974ba682f6257fd42a2f79749689f7ca0d763e198a0f7051c91"
+dependencies = [
+ "parking_lot",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.139"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+
+[[package]]
+name = "libz-sys"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
+dependencies = [
+ "cc",
+ "cmake",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
 
 [[package]]
 name = "linux-loader"
@@ -804,9 +765,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.0.46"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
+checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
 [[package]]
 name = "lock_api"
@@ -827,6 +788,16 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "lz4"
+version = "1.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1"
+dependencies = [
+ "libc",
+ "lz4-sys",
+]
+
 [[package]]
 name = "lz4-sys"
 version = "1.9.4"
@@ -837,15 +808,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "mach"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -863,30 +825,30 @@ dependencies = [
 
 [[package]]
 name = "miniz_oxide"
-version = "0.5.4"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
 dependencies = [
  "adler",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
+checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
 dependencies = [
  "libc",
  "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
  "windows-sys",
 ]
 
 [[package]]
 name = "nix"
-version = "0.23.1"
+version = "0.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6"
+checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
 dependencies = [
  "bitflags",
  "cc",
@@ -897,9 +859,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.24.2"
+version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
+checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -907,25 +869,13 @@ dependencies = [
  "memoffset",
 ]
 
-[[package]]
-name = "no-std-compat"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
-
-[[package]]
-name = "nonzero_ext"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
-
 [[package]]
 name = "num_cpus"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.2.6",
  "libc",
 ]
 
@@ -940,34 +890,28 @@ dependencies = [
 
 [[package]]
 name = "nydus-api"
-version = "0.1.3"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61fbfbdb58ff07bed50b412d4315b3c5808979bb5decb56706ac66d53daf2cf3"
+checksum = "1899def1a22ed32b1d60de4e444f525c4023a208ee0d1136a65399cff82837ce"
 dependencies = [
- "dbs-uhttp",
- "http",
- "lazy_static",
  "libc",
  "log",
- "mio",
  "nydus-error",
- "nydus-utils",
  "serde",
- "serde_derive",
  "serde_json",
- "url",
- "vmm-sys-util 0.10.0",
+ "toml",
 ]
 
 [[package]]
 name = "nydus-blobfs"
-version = "0.1.1"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ef818ecadc217f49ce8d48506b885d8d26f877d26b0108d90d8b82547663d95"
+checksum = "784cf6e1319da7a94734987dcc71d2940f74231256922431a505c832fc778dd3"
 dependencies = [
  "fuse-backend-rs",
  "libc",
  "log",
+ "nydus-api",
  "nydus-error",
  "nydus-rafs",
  "nydus-storage",
@@ -978,9 +922,9 @@ dependencies = [
 
 [[package]]
 name = "nydus-error"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90960fb7268286328d11f18e747bed58d8e3bbea6f401bd316e91fe39f4f7213"
+checksum = "ae2ec1efd1589377dbefca6b1047294c71b2fbab164d93319f97b20faae92001"
 dependencies = [
  "backtrace",
  "httpdate",
@@ -992,9 +936,9 @@ dependencies = [
 
 [[package]]
 name = "nydus-rafs"
-version = "0.1.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a06e8b0b4a90acc2d128d2f3b1ab6ae5d325116f1f69754bd3628dbd4499f4"
+checksum = "e0ace6945daa16842e72e9fe7647e2b8715856f50f07350cce82bd68db1ed02c"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -1006,59 +950,57 @@ dependencies = [
  "libc",
  "log",
  "lz4-sys",
- "nix 0.24.2",
+ "nix 0.24.3",
  "nydus-api",
  "nydus-error",
  "nydus-storage",
  "nydus-utils",
  "serde",
  "serde_json",
- "sha2",
  "spmc",
  "vm-memory",
 ]
 
 [[package]]
 name = "nydus-storage"
-version = "0.5.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5dd10c443f47a0ac7d71021f7658a605c2be5b46576a91f3238babbaf3f459e"
+checksum = "e08bc5ea9054fca2ec8b19dcce25ea600679b7fbf035aad86cfe4a659002c88b"
 dependencies = [
- "anyhow",
  "arc-swap",
  "bitflags",
- "dbs-uhttp",
  "fuse-backend-rs",
- "futures",
- "governor",
+ "hex",
  "lazy_static",
+ "leaky-bucket",
  "libc",
  "log",
- "nix 0.24.2",
+ "nix 0.24.3",
  "nydus-api",
  "nydus-error",
  "nydus-utils",
  "serde",
  "serde_json",
- "sha2",
+ "tar",
  "tokio",
  "vm-memory",
- "vmm-sys-util 0.10.0",
 ]
 
 [[package]]
 name = "nydus-utils"
-version = "0.3.2"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7e976c67052c3ff63372e2a07701923796d25a77eac605824b26d406ab0918"
+checksum = "d1e681d7207a1ec500323d5ca39ebb7e381fc4f14db5ff0c532c18ff1226a81f"
 dependencies = [
  "blake3",
  "flate2",
  "lazy_static",
  "libc",
+ "libz-sys",
  "log",
+ "lz4",
  "lz4-sys",
- "nix 0.24.2",
+ "nix 0.24.3",
  "nydus-error",
  "serde",
  "serde_json",
@@ -1069,50 +1011,42 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.29.0"
+version = "0.30.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
+checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.16.0"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
 [[package]]
 name = "parking_lot"
-version = "0.11.2"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
- "instant",
  "lock_api",
  "parking_lot_core",
 ]
 
 [[package]]
 name = "parking_lot_core"
-version = "0.8.5"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
+checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
 dependencies = [
  "cfg-if",
- "instant",
  "libc",
  "redox_syscall",
  "smallvec",
- "winapi",
+ "windows-sys",
 ]
 
-[[package]]
-name = "percent-encoding"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -1126,84 +1060,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
-name = "ppv-lite86"
-version = "0.2.17"
+name = "pkg-config"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
 dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "quanta"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
-dependencies = [
- "crossbeam-utils",
- "libc",
- "mach",
- "once_cell",
- "raw-cpuid",
- "wasi 0.10.2+wasi-snapshot-preview1",
- "web-sys",
- "winapi",
-]
-
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "raw-cpuid"
-version = "10.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -1241,29 +1120,29 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
 
 [[package]]
 name = "rustix"
-version = "0.34.8"
+version = "0.36.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2079c267b8394eb529872c3cf92e181c378b41fea36e68130357b52493701d2e"
+checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
 dependencies = [
  "bitflags",
  "errno",
  "io-lifetimes",
  "libc",
  "linux-raw-sys",
- "winapi",
+ "windows-sys",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70"
 
 [[package]]
 name = "ryu"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
 
 [[package]]
 name = "scoped-tls"
@@ -1288,18 +1167,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.147"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.147"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1308,9 +1187,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.87"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
 dependencies = [
  "itoa",
  "ryu",
@@ -1330,9 +1209,9 @@ dependencies = [
 
 [[package]]
 name = "slab"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"
+checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
 dependencies = [
  "autocfg",
 ]
@@ -1409,9 +1288,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.103"
+version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1424,6 +1303,17 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
 
+[[package]]
+name = "tar"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
 [[package]]
 name = "term"
 version = "0.7.0"
@@ -1439,23 +1329,23 @@ dependencies = [
 name = "test-utils"
 version = "0.1.0"
 dependencies = [
- "nix 0.24.2",
+ "nix 0.24.3",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1464,10 +1354,11 @@ dependencies = [
 
 [[package]]
 name = "thread_local"
-version = "1.1.4"
+version = "1.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
 dependencies = [
+ "cfg-if",
  "once_cell",
 ]
 
@@ -1482,9 +1373,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.17"
+version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
+checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
 dependencies = [
  "itoa",
  "libc",
@@ -1502,42 +1393,27 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
 
 [[package]]
 name = "time-macros"
-version = "0.2.6"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2"
+checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
 dependencies = [
  "time-core",
 ]
 
 [[package]]
 name = "timerfd"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29f85a7c965b8e7136952f59f2a359694c78f105b2d2ff99cf6c2c404bf7e33f"
+checksum = "0664936efa25f2bbe03ca25b62c50f5f492abec07e59d6dcf45131014b33483f"
 dependencies = [
  "rustix",
 ]
 
-[[package]]
-name = "tinyvec"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
-
 [[package]]
 name = "tokio"
-version = "1.21.2"
+version = "1.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
+checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64"
 dependencies = [
  "autocfg",
  "libc",
@@ -1545,14 +1421,26 @@ dependencies = [
  "num_cpus",
  "pin-project-lite",
  "socket2",
- "winapi",
+ "tokio-macros",
+ "windows-sys",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
 name = "tokio-uring"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3ad494f39874984d990ade7f6319dafbcd3301ff0b1841f8a55a1ebb3e742c8"
+checksum = "0d5e02bb137e030b3a547c65a3bd2f1836d66a97369fdcc69034002b10e155ef"
 dependencies = [
  "io-uring",
  "libc",
@@ -1563,42 +1451,63 @@ dependencies = [
 ]
 
 [[package]]
-name = "typenum"
-version = "1.15.0"
+name = "toml"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
+dependencies = [
+ "serde",
+]
 
 [[package]]
-name = "unicode-bidi"
-version = "0.3.8"
+name = "tracing"
+version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+dependencies = [
+ "cfg-if",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "typenum"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
 [[package]]
-name = "unicode-normalization"
-version = "0.1.22"
+name = "vcpkg"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "url"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
-]
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "version_check"
@@ -1614,13 +1523,14 @@ checksum = "3ff512178285488516ed85f15b5d0113a7cdb89e9e8a760b269ae4f02b84bd6b"
 
 [[package]]
 name = "virtio-queue"
-version = "0.4.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "519c0a333c871650269cba303bc108075d52a0c0d64f9b91fae61829b53725af"
+checksum = "435dd49c7b38419729afd43675850c7b5dc4728f2fabd70c7a9079a331e4f8c6"
 dependencies = [
  "log",
+ "virtio-bindings",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.10.0",
 ]
 
 [[package]]
@@ -1658,90 +1568,20 @@ dependencies = [
 
 [[package]]
 name = "vmm-sys-util"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc06a16ee8ebf0d9269aed304030b0d20a866b8b3dd3d4ce532596ac567a0d24"
+checksum = "dd64fe09d8e880e600c324e7d664760a17f56e9672b7495a86381b49e4f72f46"
 dependencies = [
  "bitflags",
  "libc",
 ]
 
-[[package]]
-name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
-dependencies = [
- "cfg-if",
- "wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
-dependencies = [
- "bumpalo",
- "log",
- "once_cell",
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
-
-[[package]]
-name = "web-sys"
-version = "0.3.60"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
-dependencies = [
- "js-sys",
- "wasm-bindgen",
-]
-
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -1766,9 +1606,18 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-sys"
-version = "0.42.0"
+version = "0.45.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
 dependencies = [
  "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
@@ -1781,45 +1630,54 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
+
+[[package]]
+name = "xattr"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "zstd"
@@ -1842,10 +1700,11 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.1+zstd.1.5.2"
+version = "2.0.7+zstd.1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
+checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5"
 dependencies = [
  "cc",
  "libc",
+ "pkg-config",
 ]
diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml
index 5b51dd68f..5036d7272 100644
--- a/src/dragonball/Cargo.toml
+++ b/src/dragonball/Cargo.toml
@@ -19,11 +19,11 @@ dbs-boot = "0.3.0"
 dbs-device = "0.2.0"
 dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
-dbs-upcall = { version = "0.1.0", optional = true }
+dbs-upcall = { version = "0.2.0", optional = true }
 dbs-utils = "0.2.0"
-dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] }
-kvm-bindings = "0.5.0"
-kvm-ioctls = "0.11.0"
+dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] }
+kvm-bindings = "0.6.0"
+kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
 libc = "0.2.39"
 linux-loader = "0.6.0"
@@ -37,7 +37,7 @@ slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1"
 vmm-sys-util = "0.11.0"
-virtio-queue = { version = "0.4.0", optional = true }
+virtio-queue = { version = "0.6.0", optional = true }
 vm-memory = { version = "0.9.0", features = ["backend-mmap"] }
 crossbeam-channel = "0.5.6"
 
diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs
index 56ee3617c..ed651f577 100644
--- a/src/dragonball/src/device_manager/mod.rs
+++ b/src/dragonball/src/device_manager/mod.rs
@@ -147,17 +147,13 @@ pub type Result<T> = ::std::result::Result<T, DeviceMgrError>;
 /// Type of the dragonball virtio devices.
 #[cfg(feature = "dbs-virtio-devices")]
 pub type DbsVirtioDevice = Box<
-    dyn VirtioDevice<
-        GuestAddressSpaceImpl,
-        virtio_queue::QueueStateSync,
-        vm_memory::GuestRegionMmap,
-    >,
+    dyn VirtioDevice<GuestAddressSpaceImpl, virtio_queue::QueueSync, vm_memory::GuestRegionMmap>,
 >;
 
 /// Type of the dragonball virtio mmio devices.
 #[cfg(feature = "dbs-virtio-devices")]
 pub type DbsMmioV2Device =
-    MmioV2Device<GuestAddressSpaceImpl, virtio_queue::QueueStateSync, vm_memory::GuestRegionMmap>;
+    MmioV2Device<GuestAddressSpaceImpl, virtio_queue::QueueSync, vm_memory::GuestRegionMmap>;
 
 /// Struct to support transactional operations for device management.
 pub struct DeviceManagerTx {
diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs
index 1c21ea38b..26f48c6d1 100644
--- a/src/dragonball/src/vcpu/vcpu_impl.rs
+++ b/src/dragonball/src/vcpu/vcpu_impl.rs
@@ -441,75 +441,77 @@ impl Vcpu {
     /// Returns error or enum specifying whether emulation was handled or interrupted.
     fn run_emulation(&mut self) -> Result<VcpuEmulation> {
         match Vcpu::emulate(&self.fd) {
-            Ok(run) => match run {
-                #[cfg(target_arch = "x86_64")]
-                VcpuExit::IoIn(addr, data) => {
-                    let _ = self.io_mgr.pio_read(addr, data);
-                    METRICS.vcpu.exit_io_in.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                #[cfg(target_arch = "x86_64")]
-                VcpuExit::IoOut(addr, data) => {
-                    if !self.check_io_port_info(addr, data)? {
-                        let _ = self.io_mgr.pio_write(addr, data);
+            Ok(run) => {
+                match run {
+                    #[cfg(target_arch = "x86_64")]
+                    VcpuExit::IoIn(addr, data) => {
+                        let _ = self.io_mgr.pio_read(addr, data);
+                        METRICS.vcpu.exit_io_in.inc();
+                        Ok(VcpuEmulation::Handled)
                     }
-                    METRICS.vcpu.exit_io_out.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                VcpuExit::MmioRead(addr, data) => {
-                    let _ = self.io_mgr.mmio_read(addr, data);
-                    METRICS.vcpu.exit_mmio_read.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                VcpuExit::MmioWrite(addr, data) => {
-                    let _ = self.io_mgr.mmio_write(addr, data);
-                    METRICS.vcpu.exit_mmio_write.inc();
-                    Ok(VcpuEmulation::Handled)
-                }
-                VcpuExit::Hlt => {
-                    info!("Received KVM_EXIT_HLT signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                VcpuExit::Shutdown => {
-                    info!("Received KVM_EXIT_SHUTDOWN signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                // Documentation specifies that below kvm exits are considered errors.
-                VcpuExit::FailEntry => {
-                    METRICS.vcpu.failures.inc();
-                    error!("Received KVM_EXIT_FAIL_ENTRY signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                VcpuExit::InternalError => {
-                    METRICS.vcpu.failures.inc();
-                    error!("Received KVM_EXIT_INTERNAL_ERROR signal");
-                    Err(VcpuError::VcpuUnhandledKvmExit)
-                }
-                VcpuExit::SystemEvent(event_type, event_flags) => match event_type {
-                    KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => {
-                        info!(
-                            "Received KVM_SYSTEM_EVENT: type: {}, event: {}",
-                            event_type, event_flags
-                        );
-                        Ok(VcpuEmulation::Stopped)
+                    #[cfg(target_arch = "x86_64")]
+                    VcpuExit::IoOut(addr, data) => {
+                        if !self.check_io_port_info(addr, data)? {
+                            let _ = self.io_mgr.pio_write(addr, data);
+                        }
+                        METRICS.vcpu.exit_io_out.inc();
+                        Ok(VcpuEmulation::Handled)
                     }
-                    _ => {
-                        METRICS.vcpu.failures.inc();
-                        error!(
-                            "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
-                            event_type, event_flags
-                        );
+                    VcpuExit::MmioRead(addr, data) => {
+                        let _ = self.io_mgr.mmio_read(addr, data);
+                        METRICS.vcpu.exit_mmio_read.inc();
+                        Ok(VcpuEmulation::Handled)
+                    }
+                    VcpuExit::MmioWrite(addr, data) => {
+                        let _ = self.io_mgr.mmio_write(addr, data);
+                        METRICS.vcpu.exit_mmio_write.inc();
+                        Ok(VcpuEmulation::Handled)
+                    }
+                    VcpuExit::Hlt => {
+                        info!("Received KVM_EXIT_HLT signal");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    VcpuExit::Shutdown => {
+                        info!("Received KVM_EXIT_SHUTDOWN signal");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    // Documentation specifies that below kvm exits are considered errors.
+                    VcpuExit::FailEntry(reason, cpu) => {
+                        METRICS.vcpu.failures.inc();
+                        error!("Received KVM_EXIT_FAIL_ENTRY signal, reason {reason}, cpu number {cpu}");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    VcpuExit::InternalError => {
+                        METRICS.vcpu.failures.inc();
+                        error!("Received KVM_EXIT_INTERNAL_ERROR signal");
+                        Err(VcpuError::VcpuUnhandledKvmExit)
+                    }
+                    VcpuExit::SystemEvent(event_type, event_flags) => match event_type {
+                        KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => {
+                            info!(
+                                "Received KVM_SYSTEM_EVENT: type: {}, event: {}",
+                                event_type, event_flags
+                            );
+                            Ok(VcpuEmulation::Stopped)
+                        }
+                        _ => {
+                            METRICS.vcpu.failures.inc();
+                            error!(
+                                "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
+                                event_type, event_flags
+                            );
+                            Err(VcpuError::VcpuUnhandledKvmExit)
+                        }
+                    },
+                    r => {
+                        METRICS.vcpu.failures.inc();
+                        // TODO: Are we sure we want to finish running a vcpu upon
+                        // receiving a vm exit that is not necessarily an error?
+                        error!("Unexpected exit reason on vcpu run: {:?}", r);
                         Err(VcpuError::VcpuUnhandledKvmExit)
                     }
-                },
-                r => {
-                    METRICS.vcpu.failures.inc();
-                    // TODO: Are we sure we want to finish running a vcpu upon
-                    // receiving a vm exit that is not necessarily an error?
-                    error!("Unexpected exit reason on vcpu run: {:?}", r);
-                    Err(VcpuError::VcpuUnhandledKvmExit)
                 }
-            },
+            }
             // The unwrap on raw_os_error can only fail if we have a logic
             // error in our code in which case it is better to panic.
             Err(ref e) => {
@@ -786,7 +788,7 @@ pub mod tests {
         MmioWrite,
         Hlt,
         Shutdown,
-        FailEntry,
+        FailEntry(u64, u32),
         InternalError,
         Unknown,
         SystemEvent(u32, u64),
@@ -807,7 +809,9 @@ pub mod tests {
                 EmulationCase::MmioWrite => Ok(VcpuExit::MmioWrite(0, &[])),
                 EmulationCase::Hlt => Ok(VcpuExit::Hlt),
                 EmulationCase::Shutdown => Ok(VcpuExit::Shutdown),
-                EmulationCase::FailEntry => Ok(VcpuExit::FailEntry),
+                EmulationCase::FailEntry(error_type, cpu_num) => {
+                    Ok(VcpuExit::FailEntry(*error_type, *cpu_num))
+                }
                 EmulationCase::InternalError => Ok(VcpuExit::InternalError),
                 EmulationCase::Unknown => Ok(VcpuExit::Unknown),
                 EmulationCase::SystemEvent(event_type, event_flags) => {
@@ -850,6 +854,8 @@ pub mod tests {
 
     #[cfg(target_arch = "aarch64")]
     fn create_vcpu() -> (Vcpu, Receiver<VcpuStateEvent>) {
+        use kvm_ioctls::Kvm;
+        use std::os::fd::AsRawFd;
         // Call for kvm too frequently would cause error in some host kernel.
         std::thread::sleep(std::time::Duration::from_millis(5));
 
@@ -918,7 +924,7 @@ pub mod tests {
         assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit)));
 
         // KVM_EXIT_FAIL_ENTRY signal
-        *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry;
+        *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry(0, 0);
         let res = vcpu.run_emulation();
         assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit)));
 
diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs
index 383f1f0a7..7b93c1a82 100644
--- a/src/dragonball/src/vcpu/vcpu_manager.rs
+++ b/src/dragonball/src/vcpu/vcpu_manager.rs
@@ -887,7 +887,9 @@ mod hotplug {
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
             let req = DevMgrRequest::AddVcpu(CpuDevRequest {
                 count: cpu_ids.len() as u8,
+                #[cfg(target_arch = "x86_64")]
                 apic_ids: cpu_ids_array,
+                #[cfg(target_arch = "x86_64")]
                 apic_ver: APIC_VERSION,
             });
             self.send_upcall_action(upcall_client, req)?;
@@ -924,7 +926,9 @@ mod hotplug {
             cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
             let req = DevMgrRequest::DelVcpu(CpuDevRequest {
                 count: cpu_num_to_be_del as u8,
+                #[cfg(target_arch = "x86_64")]
                 apic_ids: cpu_ids_array,
+                #[cfg(target_arch = "x86_64")]
                 apic_ver: APIC_VERSION,
             });
             self.send_upcall_action(upcall_client, req)?;
@@ -969,7 +973,10 @@ mod hotplug {
                                 vcpu_state_sender
                                     .send(VcpuStateEvent::Hotplug((
                                         result,
+                                        #[cfg(target_arch = "x86_64")]
                                         resp.info.apic_id_index,
+                                        #[cfg(target_arch = "aarch64")]
+                                        resp.info.cpu_id,
                                     )))
                                     .unwrap();
                                 vcpu_state_event.write(1).unwrap();
diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs
index fddbf9516..0ec8251f4 100644
--- a/src/dragonball/src/vm/aarch64.rs
+++ b/src/dragonball/src/vm/aarch64.rs
@@ -11,7 +11,7 @@ use std::fmt::Debug;
 use std::ops::Deref;
 
 use dbs_arch::gic::GICDevice;
-use dbs_arch::{DeviceInfoForFDT, DeviceType};
+use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel};
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
@@ -51,6 +51,8 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
         device_info,
         gic_device,
         initrd,
+        // We will add vpmu feature support in the future PRs. issue: #6168
+        &VpmuFeatureLevel::Disabled,
     )
     .map_err(Error::BootSystem)?;
     Ok(())
diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs
index 3e53e03a3..13e6584b8 100644
--- a/src/dragonball/src/vm/mod.rs
+++ b/src/dragonball/src/vm/mod.rs
@@ -1005,6 +1005,7 @@ pub mod tests {
         assert!(vm.remove_devices().is_ok());
     }
 
+    #[cfg(target_arch = "x86_64")]
     #[test]
     fn test_run_code() {
         skip_if_not_root!();
diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock
index fbff64e87..703082b87 100644
--- a/src/runtime-rs/Cargo.lock
+++ b/src/runtime-rs/Cargo.lock
@@ -14,9 +14,9 @@ dependencies = [
 
 [[package]]
 name = "actix-rt"
-version = "2.7.0"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000"
+checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e"
 dependencies = [
  "actix-macros",
  "futures-core",
@@ -25,9 +25,9 @@ dependencies = [
 
 [[package]]
 name = "addr2line"
-version = "0.17.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
 dependencies = [
  "gimli",
 ]
@@ -48,7 +48,7 @@ dependencies = [
  "kata-types",
  "log",
  "logging",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "protobuf",
  "protocols",
@@ -63,9 +63,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.18"
+version = "0.7.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
+checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
 dependencies = [
  "memchr",
 ]
@@ -81,9 +81,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
 
 [[package]]
 name = "api_client"
@@ -95,9 +95,9 @@ dependencies = [
 
 [[package]]
 name = "arc-swap"
-version = "1.5.0"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f"
+checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
 
 [[package]]
 name = "arrayref"
@@ -113,9 +113,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
 [[package]]
 name = "async-channel"
-version = "1.7.1"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28"
+checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
 dependencies = [
  "concurrent-queue",
  "event-listener",
@@ -124,23 +124,23 @@ dependencies = [
 
 [[package]]
 name = "async-executor"
-version = "1.4.1"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965"
+checksum = "17adb73da160dfb475c183343c8cccd80721ea5a605d3eb57125f0a7b7a92d0b"
 dependencies = [
+ "async-lock",
  "async-task",
  "concurrent-queue",
  "fastrand",
  "futures-lite",
- "once_cell",
  "slab",
 ]
 
 [[package]]
 name = "async-global-executor"
-version = "2.2.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940"
+checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776"
 dependencies = [
  "async-channel",
  "async-executor",
@@ -148,37 +148,37 @@ dependencies = [
  "async-lock",
  "blocking",
  "futures-lite",
- "num_cpus",
  "once_cell",
 ]
 
 [[package]]
 name = "async-io"
-version = "1.8.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ab006897723d9352f63e2b13047177c3982d8d79709d713ce7747a8f19fd1b0"
+checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794"
 dependencies = [
+ "async-lock",
  "autocfg",
  "concurrent-queue",
  "futures-lite",
  "libc",
  "log",
- "once_cell",
  "parking",
  "polling",
  "slab",
  "socket2",
  "waker-fn",
- "winapi",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
 name = "async-lock"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6"
+checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685"
 dependencies = [
  "event-listener",
+ "futures-lite",
 ]
 
 [[package]]
@@ -215,9 +215,9 @@ checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524"
 
 [[package]]
 name = "async-trait"
-version = "0.1.56"
+version = "0.1.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716"
+checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -226,9 +226,9 @@ dependencies = [
 
 [[package]]
 name = "atomic-waker"
-version = "1.0.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a"
+checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599"
 
 [[package]]
 name = "autocfg"
@@ -244,9 +244,9 @@ checksum = "cc17ab023b4091c10ff099f9deebaeeb59b5189df07e554c4fef042b70745d68"
 
 [[package]]
 name = "backtrace"
-version = "0.3.65"
+version = "0.3.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
+checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
 dependencies = [
  "addr2line",
  "cc",
@@ -281,16 +281,16 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.3.1"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
+checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef"
 dependencies = [
  "arrayref",
  "arrayvec",
  "cc",
  "cfg-if 1.0.0",
  "constant_time_eq",
- "digest 0.10.3",
+ "digest 0.10.6",
 ]
 
 [[package]]
@@ -304,32 +304,32 @@ dependencies = [
 
 [[package]]
 name = "block-buffer"
-version = "0.10.2"
+version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
 dependencies = [
  "generic-array",
 ]
 
 [[package]]
 name = "blocking"
-version = "1.2.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc"
+checksum = "3c67b173a56acffd6d2326fb7ab938ba0b00a71480e14902b2591c87bc5741e8"
 dependencies = [
  "async-channel",
+ "async-lock",
  "async-task",
  "atomic-waker",
  "fastrand",
  "futures-lite",
- "once_cell",
 ]
 
 [[package]]
 name = "bumpalo"
-version = "3.10.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
 [[package]]
 name = "byte-unit"
@@ -339,9 +339,9 @@ checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8"
 
 [[package]]
 name = "byte-unit"
-version = "4.0.17"
+version = "4.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "581ad4b3d627b0c09a0ccb2912148f839acaca0b93cf54cbe42b6c674e86079c"
+checksum = "3348673602e04848647fffaa8e9a861e7b5d5cae6570727b41bde0f722514484"
 dependencies = [
  "serde",
  "utf8-width",
@@ -365,32 +365,25 @@ dependencies = [
 
 [[package]]
 name = "bytes"
-version = "1.1.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
-
-[[package]]
-name = "cache-padded"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
 [[package]]
 name = "caps"
-version = "0.5.3"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61bf7211aad104ce2769ec05efcdfabf85ee84ac92461d142f22cf8badd0e54c"
+checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b"
 dependencies = [
- "errno",
  "libc",
  "thiserror",
 ]
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 dependencies = [
  "jobserver",
 ]
@@ -435,19 +428,38 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
+checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
 dependencies = [
  "iana-time-zone",
  "js-sys",
  "num-integer",
  "num-traits",
- "time 0.1.43",
+ "time 0.1.45",
  "wasm-bindgen",
  "winapi",
 ]
 
+[[package]]
+name = "cmake"
+version = "0.1.49"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
+
 [[package]]
 name = "common"
 version = "0.1.0"
@@ -459,7 +471,7 @@ dependencies = [
  "kata-sys-util",
  "kata-types",
  "lazy_static",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "persist",
  "protobuf",
@@ -480,18 +492,18 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101"
 
 [[package]]
 name = "concurrent-queue"
-version = "1.2.4"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c"
+checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e"
 dependencies = [
- "cache-padded",
+ "crossbeam-utils",
 ]
 
 [[package]]
 name = "constant_time_eq"
-version = "0.1.5"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279"
 
 [[package]]
 name = "containerd-shim-protos"
@@ -536,9 +548,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c"
 dependencies = [
  "cfg-if 1.0.0",
  "crossbeam-utils",
@@ -546,19 +558,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.8"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
+checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
 dependencies = [
  "cfg-if 1.0.0",
- "lazy_static",
 ]
 
 [[package]]
 name = "crypto-common"
-version = "0.1.3"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array",
  "typenum",
@@ -566,22 +577,56 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.1.22"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
+checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
 dependencies = [
  "quote",
  "syn",
 ]
 
 [[package]]
-name = "dashmap"
-version = "4.0.2"
+name = "cxx"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
+checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62"
 dependencies = [
- "cfg-if 1.0.0",
- "num_cpus",
+ "cc",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
@@ -592,40 +637,41 @@ checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc"
 dependencies = [
  "arc-swap",
  "libc",
- "nix 0.23.1",
+ "nix 0.23.2",
  "thiserror",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-allocator"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92a384ac9bd9c22c486c7a66e68cdc6cd504da7793b69bd891f3d85070c999b6"
+checksum = "543711b94b4bc1437d2ebb45f856452e96a45a67ab39f8dcf8c887c2a3701004"
 dependencies = [
  "thiserror",
 ]
 
 [[package]]
 name = "dbs-arch"
-version = "0.2.1"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f89357fc97fb3608473073be037ea0b22787b1fa4c68b8eb3dd51f3c5fd6b41"
+checksum = "194c844946cd1d13f7a9eb29b84afbc5354578eee2b06fea96226bc3872e7424"
 dependencies = [
  "kvm-bindings",
  "kvm-ioctls",
  "libc",
  "memoffset 0.6.5",
+ "thiserror",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-boot"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e6930547e688d8527705d1b7c4163c090c8535b8dd526d8251aa4dfdcbf2f82"
+checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108"
 dependencies = [
  "dbs-arch",
  "kvm-bindings",
@@ -648,15 +694,16 @@ dependencies = [
 
 [[package]]
 name = "dbs-interrupt"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f217820329cea9d8d2870f9cdda426c5ca4379e33283c39338841a86bdc36c"
+checksum = "1eb2c5bb9f8f123ace33b1b2e8d53dd2d87331ee770ad1f82e56c3382c6bed6d"
 dependencies = [
+ "dbs-arch",
  "dbs-device",
  "kvm-bindings",
  "kvm-ioctls",
  "libc",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -670,24 +717,14 @@ dependencies = [
  "log",
  "serde",
  "vm-superio",
- "vmm-sys-util 0.11.0",
-]
-
-[[package]]
-name = "dbs-uhttp"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fd0544fe7ba81fa8deb8800843836d279a81b051e2e8ab046fe1b0cb096c1cc"
-dependencies = [
- "libc",
- "mio",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-upcall"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2fa8b67657cd71779eaceea1b5fa989b62a1be629a07be8498417772e5a8d35"
+checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f"
 dependencies = [
  "anyhow",
  "dbs-utils",
@@ -710,14 +747,14 @@ dependencies = [
  "serde",
  "thiserror",
  "timerfd",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "dbs-virtio-devices"
-version = "0.1.1"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f70cc3a62fa1c169beca6921ef0d3cf38fdfe7cd732ac76c8517bc8a3df9338"
+checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668"
 dependencies = [
  "byteorder",
  "caps",
@@ -731,7 +768,8 @@ dependencies = [
  "kvm-ioctls",
  "libc",
  "log",
- "nix 0.23.1",
+ "nix 0.23.2",
+ "nydus-api",
  "nydus-blobfs",
  "nydus-rafs",
  "rlimit",
@@ -742,7 +780,7 @@ dependencies = [
  "virtio-bindings",
  "virtio-queue",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -767,11 +805,11 @@ dependencies = [
 
 [[package]]
 name = "digest"
-version = "0.10.3"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
 dependencies = [
- "block-buffer 0.10.2",
+ "block-buffer 0.10.3",
  "crypto-common",
  "subtle",
 ]
@@ -781,7 +819,7 @@ name = "dragonball"
 version = "0.1.0"
 dependencies = [
  "arc-swap",
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "crossbeam-channel",
  "dbs-address-space",
  "dbs-allocator",
@@ -799,7 +837,7 @@ dependencies = [
  "libc",
  "linux-loader",
  "log",
- "nix 0.24.2",
+ "nix 0.24.3",
  "seccompiler",
  "serde",
  "serde_derive",
@@ -809,14 +847,14 @@ dependencies = [
  "thiserror",
  "virtio-queue",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "either"
-version = "1.6.1"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 
 [[package]]
 name = "epoll"
@@ -862,29 +900,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "377fa591135fbe23396a18e2655a6d5481bf7c5823cdfa3cc81b01a229cbe640"
 dependencies = [
  "libc",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "fail"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011"
+checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
 dependencies = [
- "lazy_static",
  "log",
+ "once_cell",
  "rand 0.8.5",
 ]
 
 [[package]]
 name = "fastrand"
-version = "1.7.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf"
+checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
 dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "filetime"
+version = "0.2.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "redox_syscall",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "fixedbitset"
 version = "0.2.0"
@@ -893,11 +943,12 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
 [[package]]
 name = "flate2"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
 dependencies = [
  "crc32fast",
+ "libz-sys",
  "miniz_oxide",
 ]
 
@@ -909,11 +960,10 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
+checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
 dependencies = [
- "matches",
  "percent-encoding",
 ]
 
@@ -925,23 +975,19 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 
 [[package]]
 name = "fuse-backend-rs"
-version = "0.9.6"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "994a3bfb694ee52bf8f3bca80d784b723f150810998219337e429cc5dbe92717"
+checksum = "08af89cb80a7c5693bd63a2b1ee7ac31a307670977c18fda036b3aa94be8c47f"
 dependencies = [
  "arc-swap",
  "bitflags",
  "caps",
  "core-foundation-sys",
- "io-uring",
  "lazy_static",
  "libc",
  "log",
  "mio",
- "nix 0.24.2",
- "scoped-tls",
- "slab",
- "socket2",
+ "nix 0.24.3",
  "tokio-uring",
  "virtio-queue",
  "vm-memory",
@@ -1040,12 +1086,6 @@ version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
 
-[[package]]
-name = "futures-timer"
-version = "3.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
-
 [[package]]
 name = "futures-util"
 version = "0.3.26"
@@ -1066,9 +1106,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.5"
+version = "0.14.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
 dependencies = [
  "typenum",
  "version_check",
@@ -1087,32 +1127,32 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.6"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
+checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "wasi 0.10.2+wasi-snapshot-preview1",
+ "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
 [[package]]
 name = "gimli"
-version = "0.26.1"
+version = "0.27.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
+checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
 
 [[package]]
 name = "glob"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "gloo-timers"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9"
+checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1129,28 +1169,11 @@ dependencies = [
  "cfg-if 0.1.10",
 ]
 
-[[package]]
-name = "governor"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7df0ee4b237afb71e99f7e2fbd840ffec2d6c4bb569f69b2af18aa1f63077d38"
-dependencies = [
- "dashmap",
- "futures 0.3.26",
- "futures-timer",
- "no-std-compat",
- "nonzero_ext",
- "parking_lot 0.11.2",
- "quanta",
- "rand 0.8.5",
- "smallvec",
-]
-
 [[package]]
 name = "hashbrown"
-version = "0.11.2"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "heck"
@@ -1163,15 +1186,15 @@ dependencies = [
 
 [[package]]
 name = "heck"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
 dependencies = [
  "libc",
 ]
@@ -1184,11 +1207,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
 [[package]]
 name = "http"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "fnv",
  "itoa",
 ]
@@ -1199,7 +1222,7 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "http",
  "pin-project-lite",
 ]
@@ -1218,11 +1241,11 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
 [[package]]
 name = "hyper"
-version = "0.14.20"
+version = "0.14.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
+checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -1268,7 +1291,7 @@ dependencies = [
  "kata-types",
  "libc",
  "logging",
- "nix 0.24.2",
+ "nix 0.24.3",
  "persist",
  "rand 0.8.5",
  "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1280,38 +1303,48 @@ dependencies = [
  "slog-scope",
  "thiserror",
  "tokio",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.46"
+version = "0.1.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501"
+checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
+ "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
  "winapi",
 ]
 
 [[package]]
-name = "idna"
-version = "0.2.3"
+name = "iana-time-zone-haiku"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
+checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+dependencies = [
+ "cxx",
+ "cxx-build",
+]
+
+[[package]]
+name = "idna"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
 dependencies = [
- "matches",
  "unicode-bidi",
  "unicode-normalization",
 ]
 
 [[package]]
 name = "indexmap"
-version = "1.8.2"
+version = "1.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a"
+checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -1328,15 +1361,19 @@ dependencies = [
 
 [[package]]
 name = "io-lifetimes"
-version = "0.6.1"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504"
+checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3"
+dependencies = [
+ "libc",
+ "windows-sys 0.45.0",
+]
 
 [[package]]
 name = "io-uring"
-version = "0.5.2"
+version = "0.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d75829ed9377bab6c90039fe47b9d84caceb4b5063266142e21bcce6550cda8"
+checksum = "dd1e1a01cfb924fd8c5c43b6827965db394f5a3a16c599ce03452266e1cf984c"
 dependencies = [
  "bitflags",
  "libc",
@@ -1353,33 +1390,33 @@ dependencies = [
 
 [[package]]
 name = "itertools"
-version = "0.10.3"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
 dependencies = [
  "either",
 ]
 
 [[package]]
 name = "itoa"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
 [[package]]
 name = "jobserver"
-version = "0.1.24"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.59"
+version = "0.3.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1396,7 +1433,7 @@ dependencies = [
  "kata-types",
  "lazy_static",
  "libc",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "once_cell",
  "rand 0.7.3",
@@ -1425,7 +1462,7 @@ dependencies = [
  "slog",
  "slog-scope",
  "thiserror",
- "toml 0.5.9",
+ "toml 0.5.11",
 ]
 
 [[package]]
@@ -1439,22 +1476,22 @@ dependencies = [
 
 [[package]]
 name = "kvm-bindings"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a78c049190826fff959994b7c1d8a2930d0a348f1b8f3aa4f9bb34cd5d7f2952"
+checksum = "efe70e65a5b092161d17f5005b66e5eefe7a94a70c332e755036fc4af78c4e79"
 dependencies = [
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
 name = "kvm-ioctls"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97422ba48d7ffb66fd4d18130f72ab66f9bbbf791fb7a87b9291cdcfec437593"
+checksum = "c3a321cabd827642499c77e27314f388dd83a717a5ca716b86476fb947f73ae4"
 dependencies = [
  "kvm-bindings",
  "libc",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.11.1",
 ]
 
 [[package]]
@@ -1463,12 +1500,45 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "leaky-bucket"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79e0d52231827974ba682f6257fd42a2f79749689f7ca0d763e198a0f7051c91"
+dependencies = [
+ "parking_lot 0.12.1",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
+[[package]]
+name = "libz-sys"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
+dependencies = [
+ "cc",
+ "cmake",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "link-cplusplus"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "linux-loader"
 version = "0.6.0"
@@ -1480,9 +1550,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.0.46"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
+checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
 [[package]]
 name = "linux_container"
@@ -1497,9 +1567,9 @@ dependencies = [
 
 [[package]]
 name = "lock_api"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
+checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -1526,6 +1596,16 @@ dependencies = [
  "slog-scope",
 ]
 
+[[package]]
+name = "lz4"
+version = "1.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1"
+dependencies = [
+ "libc",
+ "lz4-sys",
+]
+
 [[package]]
 name = "lz4-sys"
 version = "1.9.4"
@@ -1536,21 +1616,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "mach"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "matches"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
-
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -1577,23 +1642,23 @@ dependencies = [
 
 [[package]]
 name = "miniz_oxide"
-version = "0.5.3"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
+checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
 dependencies = [
  "adler",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
+checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
 dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.42.0",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1630,9 +1695,9 @@ dependencies = [
 
 [[package]]
 name = "netlink-packet-utils"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e"
+checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34"
 dependencies = [
  "anyhow",
  "byteorder",
@@ -1646,7 +1711,7 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "futures 0.3.26",
  "log",
  "netlink-packet-core",
@@ -1657,11 +1722,11 @@ dependencies = [
 
 [[package]]
 name = "netlink-sys"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027"
+checksum = "260e21fbb6f3d253a14df90eb0000a6066780a15dd901a7519ce02d77a94985b"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "futures 0.3.26",
  "libc",
  "log",
@@ -1670,9 +1735,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.23.1"
+version = "0.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6"
+checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
 dependencies = [
  "bitflags",
  "cc",
@@ -1683,9 +1748,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.24.2"
+version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
+checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
 dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
@@ -1719,18 +1784,6 @@ dependencies = [
  "static_assertions",
 ]
 
-[[package]]
-name = "no-std-compat"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
-
-[[package]]
-name = "nonzero_ext"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
-
 [[package]]
 name = "num-integer"
 version = "0.1.45"
@@ -1752,53 +1805,38 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.13.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
  "hermit-abi",
  "libc",
 ]
 
-[[package]]
-name = "num_threads"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "nydus-api"
-version = "0.1.3"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61fbfbdb58ff07bed50b412d4315b3c5808979bb5decb56706ac66d53daf2cf3"
+checksum = "1899def1a22ed32b1d60de4e444f525c4023a208ee0d1136a65399cff82837ce"
 dependencies = [
- "dbs-uhttp",
- "http",
- "lazy_static",
  "libc",
  "log",
- "mio",
  "nydus-error",
- "nydus-utils",
  "serde",
- "serde_derive",
  "serde_json",
- "url",
- "vmm-sys-util 0.10.0",
+ "toml 0.5.11",
 ]
 
 [[package]]
 name = "nydus-blobfs"
-version = "0.1.1"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ef818ecadc217f49ce8d48506b885d8d26f877d26b0108d90d8b82547663d95"
+checksum = "784cf6e1319da7a94734987dcc71d2940f74231256922431a505c832fc778dd3"
 dependencies = [
  "fuse-backend-rs",
  "libc",
  "log",
+ "nydus-api",
  "nydus-error",
  "nydus-rafs",
  "nydus-storage",
@@ -1809,9 +1847,9 @@ dependencies = [
 
 [[package]]
 name = "nydus-error"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90960fb7268286328d11f18e747bed58d8e3bbea6f401bd316e91fe39f4f7213"
+checksum = "ae2ec1efd1589377dbefca6b1047294c71b2fbab164d93319f97b20faae92001"
 dependencies = [
  "backtrace",
  "httpdate",
@@ -1823,9 +1861,9 @@ dependencies = [
 
 [[package]]
 name = "nydus-rafs"
-version = "0.1.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a06e8b0b4a90acc2d128d2f3b1ab6ae5d325116f1f69754bd3628dbd4499f4"
+checksum = "e0ace6945daa16842e72e9fe7647e2b8715856f50f07350cce82bd68db1ed02c"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -1837,72 +1875,70 @@ dependencies = [
  "libc",
  "log",
  "lz4-sys",
- "nix 0.24.2",
+ "nix 0.24.3",
  "nydus-api",
  "nydus-error",
  "nydus-storage",
  "nydus-utils",
  "serde",
  "serde_json",
- "sha2 0.10.5",
  "spmc",
  "vm-memory",
 ]
 
 [[package]]
 name = "nydus-storage"
-version = "0.5.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5dd10c443f47a0ac7d71021f7658a605c2be5b46576a91f3238babbaf3f459e"
+checksum = "e08bc5ea9054fca2ec8b19dcce25ea600679b7fbf035aad86cfe4a659002c88b"
 dependencies = [
- "anyhow",
  "arc-swap",
  "bitflags",
- "dbs-uhttp",
  "fuse-backend-rs",
- "futures 0.3.26",
- "governor",
+ "hex",
  "lazy_static",
+ "leaky-bucket",
  "libc",
  "log",
- "nix 0.24.2",
+ "nix 0.24.3",
  "nydus-api",
  "nydus-error",
  "nydus-utils",
  "serde",
  "serde_json",
- "sha2 0.10.5",
+ "tar",
  "tokio",
  "vm-memory",
- "vmm-sys-util 0.10.0",
 ]
 
 [[package]]
 name = "nydus-utils"
-version = "0.3.2"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7e976c67052c3ff63372e2a07701923796d25a77eac605824b26d406ab0918"
+checksum = "d1e681d7207a1ec500323d5ca39ebb7e381fc4f14db5ff0c532c18ff1226a81f"
 dependencies = [
  "blake3",
  "flate2",
  "lazy_static",
  "libc",
+ "libz-sys",
  "log",
+ "lz4",
  "lz4-sys",
- "nix 0.24.2",
+ "nix 0.24.3",
  "nydus-error",
  "serde",
  "serde_json",
- "sha2 0.10.5",
+ "sha2 0.10.6",
  "tokio",
  "zstd",
 ]
 
 [[package]]
 name = "object"
-version = "0.28.4"
+version = "0.30.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424"
+checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
 dependencies = [
  "memchr",
 ]
@@ -1919,9 +1955,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.12.0"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
 [[package]]
 name = "opaque-debug"
@@ -1943,7 +1979,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
 dependencies = [
  "instant",
  "lock_api",
- "parking_lot_core 0.8.5",
+ "parking_lot_core 0.8.6",
 ]
 
 [[package]]
@@ -1953,14 +1989,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.3",
+ "parking_lot_core 0.9.7",
 ]
 
 [[package]]
 name = "parking_lot_core"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
+checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
 dependencies = [
  "cfg-if 1.0.0",
  "instant",
@@ -1972,28 +2008,28 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.3"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys 0.36.1",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
 name = "paste"
-version = "1.0.7"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
+checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
 
 [[package]]
 name = "percent-encoding"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
 [[package]]
 name = "persist"
@@ -2053,30 +2089,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
-name = "polling"
-version = "2.3.0"
+name = "pkg-config"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+
+[[package]]
+name = "polling"
+version = "2.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6"
 dependencies = [
  "autocfg",
  "cfg-if 1.0.0",
  "libc",
  "log",
  "wepoll-ffi",
- "winapi",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.16"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.50"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
 dependencies = [
  "unicode-ident",
 ]
@@ -2087,7 +2129,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "prost-derive",
 ]
 
@@ -2097,7 +2139,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "heck 0.3.3",
  "itertools",
  "log",
@@ -2128,15 +2170,15 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
 dependencies = [
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "prost",
 ]
 
 [[package]]
 name = "protobuf"
-version = "2.27.1"
+version = "2.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
+checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
 dependencies = [
  "serde",
  "serde_derive",
@@ -2144,23 +2186,42 @@ dependencies = [
 
 [[package]]
 name = "protobuf-codegen"
-version = "2.27.1"
+version = "2.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707"
+checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6"
 dependencies = [
  "protobuf",
 ]
 
 [[package]]
 name = "protobuf-codegen-pure"
-version = "2.27.1"
+version = "2.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6"
+checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865"
 dependencies = [
  "protobuf",
  "protobuf-codegen",
 ]
 
+[[package]]
+name = "protobuf-codegen-pure3"
+version = "2.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b351add14db0721ad0842f4858aec11a5088684112ef163fc50f113c63e69b2e"
+dependencies = [
+ "protobuf",
+ "protobuf-codegen3",
+]
+
+[[package]]
+name = "protobuf-codegen3"
+version = "2.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73c5878d0fa872bd7d61782c6aa2d2d56761ba4ed4514eb6992f5f83162f1d2f"
+dependencies = [
+ "protobuf",
+]
+
 [[package]]
 name = "protocols"
 version = "0.1.0"
@@ -2172,27 +2233,11 @@ dependencies = [
  "ttrpc-codegen",
 ]
 
-[[package]]
-name = "quanta"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
-dependencies = [
- "crossbeam-utils",
- "libc",
- "mach",
- "once_cell",
- "raw-cpuid",
- "wasi 0.10.2+wasi-snapshot-preview1",
- "web-sys",
- "winapi",
-]
-
 [[package]]
 name = "quote"
-version = "1.0.18"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
@@ -2241,7 +2286,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
  "rand_chacha 0.3.1",
- "rand_core 0.6.3",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -2261,7 +2306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core 0.6.3",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -2290,11 +2335,11 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.6.3"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.6",
+ "getrandom 0.2.8",
 ]
 
 [[package]]
@@ -2306,15 +2351,6 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
-[[package]]
-name = "raw-cpuid"
-version = "10.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "738bc47119e3eeccc7e94c4a506901aea5e7b4944ecd0829cbebf4af04ceda12"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "rdrand"
 version = "0.4.0"
@@ -2326,18 +2362,18 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.13"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
  "bitflags",
 ]
 
 [[package]]
 name = "regex"
-version = "1.5.6"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2346,18 +2382,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.26"
+version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
-
-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
+checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
 [[package]]
 name = "resource"
@@ -2368,7 +2395,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "bitflags",
- "byte-unit 4.0.17",
+ "byte-unit 4.0.18",
  "cgroups-rs",
  "futures 0.3.26",
  "hypervisor",
@@ -2379,7 +2406,7 @@ dependencies = [
  "logging",
  "netlink-packet-route",
  "netlink-sys",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "persist",
  "rand 0.7.3",
@@ -2414,7 +2441,7 @@ dependencies = [
  "log",
  "netlink-packet-route",
  "netlink-proto",
- "nix 0.24.2",
+ "nix 0.24.3",
  "thiserror",
  "tokio",
 ]
@@ -2450,29 +2477,29 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
 
 [[package]]
 name = "rustix"
-version = "0.34.8"
+version = "0.36.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2079c267b8394eb529872c3cf92e181c378b41fea36e68130357b52493701d2e"
+checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
 dependencies = [
  "bitflags",
  "errno",
  "io-lifetimes",
  "libc",
  "linux-raw-sys",
- "winapi",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.6"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f"
+checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70"
 
 [[package]]
 name = "ryu"
-version = "1.0.10"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
 
 [[package]]
 name = "safe-path"
@@ -2492,9 +2519,9 @@ dependencies = [
 
 [[package]]
 name = "scoped-tls"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
+checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
 
 [[package]]
 name = "scopeguard"
@@ -2502,6 +2529,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scratch"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
+
 [[package]]
 name = "seccompiler"
 version = "0.2.0"
@@ -2533,9 +2566,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.91"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
 dependencies = [
  "itoa",
  "ryu",
@@ -2597,13 +2630,13 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5"
+checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
 dependencies = [
  "cfg-if 1.0.0",
  "cpufeatures",
- "digest 0.10.3",
+ "digest 0.10.6",
 ]
 
 [[package]]
@@ -2619,7 +2652,7 @@ dependencies = [
  "libc",
  "log",
  "logging",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "protobuf",
  "rand 0.8.5",
@@ -2660,18 +2693,21 @@ dependencies = [
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
+checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "slab"
-version = "0.4.6"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
+checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
+dependencies = [
+ "autocfg",
+]
 
 [[package]]
 name = "slog"
@@ -2700,7 +2736,7 @@ dependencies = [
  "serde",
  "serde_json",
  "slog",
- "time 0.3.9",
+ "time 0.3.20",
 ]
 
 [[package]]
@@ -2727,15 +2763,15 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "socket2"
-version = "0.4.4"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
+checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
 dependencies = [
  "libc",
  "winapi",
@@ -2755,20 +2791,20 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
 [[package]]
 name = "strum"
-version = "0.24.0"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8"
+checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
 dependencies = [
  "strum_macros",
 ]
 
 [[package]]
 name = "strum_macros"
-version = "0.24.0"
+version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef"
+checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
 dependencies = [
- "heck 0.4.0",
+ "heck 0.4.1",
  "proc-macro2",
  "quote",
  "rustversion",
@@ -2793,9 +2829,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.107"
+version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2809,24 +2845,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
 
 [[package]]
-name = "tempfile"
-version = "3.3.0"
+name = "tar"
+version = "0.4.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
  "cfg-if 1.0.0",
  "fastrand",
- "libc",
  "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "rustix",
+ "windows-sys 0.42.0",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+dependencies = [
+ "winapi-util",
 ]
 
 [[package]]
 name = "test-utils"
 version = "0.1.0"
 dependencies = [
- "nix 0.24.2",
+ "nix 0.24.3",
 ]
 
 [[package]]
@@ -2838,18 +2893,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.31"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.31"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2858,10 +2913,11 @@ dependencies = [
 
 [[package]]
 name = "thread_local"
-version = "1.1.4"
+version = "1.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
 dependencies = [
+ "cfg-if 1.0.0",
  "once_cell",
 ]
 
@@ -2876,30 +2932,47 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.1.43"
+version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
+checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
 dependencies = [
  "libc",
+ "wasi 0.10.0+wasi-snapshot-preview1",
  "winapi",
 ]
 
 [[package]]
 name = "time"
-version = "0.3.9"
+version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd"
+checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
 dependencies = [
  "itoa",
- "libc",
- "num_threads",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+
+[[package]]
+name = "time-macros"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
+dependencies = [
+ "time-core",
 ]
 
 [[package]]
 name = "timerfd"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29f85a7c965b8e7136952f59f2a359694c78f105b2d2ff99cf6c2c404bf7e33f"
+checksum = "0664936efa25f2bbe03ca25b62c50f5f492abec07e59d6dcf45131014b33483f"
 dependencies = [
  "rustix",
 ]
@@ -2915,18 +2988,18 @@ dependencies = [
 
 [[package]]
 name = "tinyvec_macros"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.25.0"
+version = "1.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
+checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64"
 dependencies = [
  "autocfg",
- "bytes 1.1.0",
+ "bytes 1.4.0",
  "libc",
  "memchr",
  "mio",
@@ -2936,14 +3009,14 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.42.0",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.0"
+version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2952,9 +3025,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-uring"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3ad494f39874984d990ade7f6319dafbcd3301ff0b1841f8a55a1ebb3e742c8"
+checksum = "0d5e02bb137e030b3a547c65a3bd2f1836d66a97369fdcc69034002b10e155ef"
 dependencies = [
  "io-uring",
  "libc",
@@ -2966,9 +3039,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-vsock"
-version = "0.3.1"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6"
+checksum = "9b33556828911d16e24d8b5d336446b0bf6b4b9bfda52cbdc2fa35b7a2862ebc"
 dependencies = [
  "bytes 0.4.12",
  "futures 0.3.26",
@@ -2988,9 +3061,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.5.9"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
 dependencies = [
  "serde",
 ]
@@ -3003,29 +3076,41 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.35"
+version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
 dependencies = [
  "cfg-if 1.0.0",
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
 ]
 
 [[package]]
-name = "tracing-core"
-version = "0.1.28"
+name = "tracing-attributes"
+version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7"
+checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
 dependencies = [
  "once_cell",
 ]
 
 [[package]]
 name = "try-lock"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
 [[package]]
 name = "ttrpc"
@@ -3038,7 +3123,7 @@ dependencies = [
  "futures 0.3.26",
  "libc",
  "log",
- "nix 0.23.1",
+ "nix 0.23.2",
  "protobuf",
  "protobuf-codegen-pure",
  "thiserror",
@@ -3048,63 +3133,69 @@ dependencies = [
 
 [[package]]
 name = "ttrpc-codegen"
-version = "0.2.0"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408"
+checksum = "2365c9179ad82b29bda1b0162c7542ab5861a7844abfedd8cfdf8bd7e12358f9"
 dependencies = [
  "protobuf",
- "protobuf-codegen",
- "protobuf-codegen-pure",
+ "protobuf-codegen-pure3",
+ "protobuf-codegen3",
  "ttrpc-compiler",
 ]
 
 [[package]]
 name = "ttrpc-compiler"
-version = "0.4.1"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15"
+checksum = "ed57c2d6669099791507b8b491b2940f2e8975b52a73fe82efad24257d10e9bc"
 dependencies = [
  "derive-new",
  "prost",
  "prost-build",
  "prost-types",
  "protobuf",
- "protobuf-codegen",
+ "protobuf-codegen3",
  "tempfile",
 ]
 
 [[package]]
 name = "typenum"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.8"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.0"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.19"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
+checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
 dependencies = [
  "tinyvec",
 ]
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.9.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "unix_socket2"
@@ -3117,13 +3208,12 @@ dependencies = [
 
 [[package]]
 name = "url"
-version = "2.2.2"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
+checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
 dependencies = [
  "form_urlencoded",
  "idna",
- "matches",
  "percent-encoding",
 ]
 
@@ -3152,6 +3242,12 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -3176,7 +3272,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "logging",
- "nix 0.24.2",
+ "nix 0.24.3",
  "oci",
  "persist",
  "protobuf",
@@ -3199,13 +3295,14 @@ checksum = "3ff512178285488516ed85f15b5d0113a7cdb89e9e8a760b269ae4f02b84bd6b"
 
 [[package]]
 name = "virtio-queue"
-version = "0.4.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "519c0a333c871650269cba303bc108075d52a0c0d64f9b91fae61829b53725af"
+checksum = "435dd49c7b38419729afd43675850c7b5dc4728f2fabd70c7a9079a331e4f8c6"
 dependencies = [
  "log",
+ "virtio-bindings",
  "vm-memory",
- "vmm-sys-util 0.11.0",
+ "vmm-sys-util 0.10.0",
 ]
 
 [[package]]
@@ -3243,9 +3340,9 @@ dependencies = [
 
 [[package]]
 name = "vmm-sys-util"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc06a16ee8ebf0d9269aed304030b0d20a866b8b3dd3d4ce532596ac567a0d24"
+checksum = "dd64fe09d8e880e600c324e7d664760a17f56e9672b7495a86381b49e4f72f46"
 dependencies = [
  "bitflags",
  "libc",
@@ -3258,7 +3355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133"
 dependencies = [
  "libc",
- "nix 0.23.1",
+ "nix 0.23.2",
 ]
 
 [[package]]
@@ -3285,9 +3382,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 
 [[package]]
 name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
+version = "0.10.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
 
 [[package]]
 name = "wasi"
@@ -3297,9 +3394,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.82"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
 dependencies = [
  "cfg-if 1.0.0",
  "wasm-bindgen-macro",
@@ -3307,9 +3404,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.82"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
 dependencies = [
  "bumpalo",
  "log",
@@ -3322,9 +3419,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.32"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad"
+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
 dependencies = [
  "cfg-if 1.0.0",
  "js-sys",
@@ -3334,9 +3431,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.82"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -3344,9 +3441,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.82"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3357,9 +3454,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.82"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
 
 [[package]]
 name = "wasm_container"
@@ -3374,9 +3471,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.57"
+version = "0.3.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283"
+checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3393,13 +3490,13 @@ dependencies = [
 
 [[package]]
 name = "which"
-version = "4.2.5"
+version = "4.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
+checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
 dependencies = [
  "either",
- "lazy_static",
  "libc",
+ "once_cell",
 ]
 
 [[package]]
@@ -3418,25 +3515,21 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows-sys"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
-dependencies = [
- "windows_aarch64_msvc 0.36.1",
- "windows_i686_gnu 0.36.1",
- "windows_i686_msvc 0.36.1",
- "windows_x86_64_gnu 0.36.1",
- "windows_x86_64_msvc 0.36.1",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.42.0"
@@ -3444,12 +3537,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
  "windows_aarch64_gnullvm",
- "windows_aarch64_msvc 0.42.1",
- "windows_i686_gnu 0.42.1",
- "windows_i686_msvc 0.42.1",
- "windows_x86_64_gnu 0.42.1",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
  "windows_x86_64_gnullvm",
- "windows_x86_64_msvc 0.42.1",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
 ]
 
 [[package]]
@@ -3458,48 +3575,24 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.1"
@@ -3512,18 +3605,21 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
+[[package]]
+name = "xattr"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "zstd"
 version = "0.11.2+zstd.1.5.2"
@@ -3545,10 +3641,11 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.1+zstd.1.5.2"
+version = "2.0.7+zstd.1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
+checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5"
 dependencies = [
  "cc",
  "libc",
+ "pkg-config",
 ]

From e07162e79d15c47e224fcd97a1fe96a8ba2d346d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 2 Mar 2023 10:17:18 +0100
Subject: [PATCH 60/76] workflows: Fix action name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We have a few actions in the `payload-after-push.*.yaml` that are
referring to Confidential Containers, but they should be referring to
Kata Containers instead.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/payload-after-push-amd64.yaml | 2 +-
 .github/workflows/payload-after-push-arm64.yaml | 2 +-
 .github/workflows/payload-after-push-s390x.yaml | 2 +-
 .github/workflows/payload-after-push.yaml       | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/payload-after-push-amd64.yaml b/.github/workflows/payload-after-push-amd64.yaml
index 30b4303c9..88497ca45 100644
--- a/.github/workflows/payload-after-push-amd64.yaml
+++ b/.github/workflows/payload-after-push-amd64.yaml
@@ -75,7 +75,7 @@ jobs:
     needs: create-kata-tarball
     runs-on: ubuntu-latest
     steps:
-      - name: Login to Confidential Containers quay.io
+      - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2
         with:
           registry: quay.io
diff --git a/.github/workflows/payload-after-push-arm64.yaml b/.github/workflows/payload-after-push-arm64.yaml
index 0dc11a665..56faeb10e 100644
--- a/.github/workflows/payload-after-push-arm64.yaml
+++ b/.github/workflows/payload-after-push-arm64.yaml
@@ -83,7 +83,7 @@ jobs:
     needs: create-kata-tarball
     runs-on: arm64
     steps:
-      - name: Login to Confidential Containers quay.io
+      - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2
         with:
           registry: quay.io
diff --git a/.github/workflows/payload-after-push-s390x.yaml b/.github/workflows/payload-after-push-s390x.yaml
index 0919da2a0..4fa147205 100644
--- a/.github/workflows/payload-after-push-s390x.yaml
+++ b/.github/workflows/payload-after-push-s390x.yaml
@@ -82,7 +82,7 @@ jobs:
     needs: create-kata-tarball
     runs-on: s390x
     steps:
-      - name: Login to Confidential Containers quay.io
+      - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2
         with:
           registry: quay.io
diff --git a/.github/workflows/payload-after-push.yaml b/.github/workflows/payload-after-push.yaml
index c121f8d9f..0007bad15 100644
--- a/.github/workflows/payload-after-push.yaml
+++ b/.github/workflows/payload-after-push.yaml
@@ -31,7 +31,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Login to Confidential Containers quay.io
+      - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2
         with:
           registry: quay.io

From 7d292d7fc3e818f3206a7ef29ca5b0249a80ac7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 2 Mar 2023 10:18:10 +0100
Subject: [PATCH 61/76] workflows: Fix the path of imported workflows
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In `payload-after-push.yaml` we ended up mentioning cc-*.yaml workflows,
which are non existent in the main branch.

Let's adapt the name to the correct ones.

Fixes: #6343

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/payload-after-push.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/payload-after-push.yaml b/.github/workflows/payload-after-push.yaml
index 0007bad15..f07c8859c 100644
--- a/.github/workflows/payload-after-push.yaml
+++ b/.github/workflows/payload-after-push.yaml
@@ -7,19 +7,19 @@ on:
 
 jobs:
   build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
+    uses: ./.github/workflows/payload-after-push-amd64.yaml
     with:
       target-arch: amd64
     secrets: inherit
 
   build-assets-arm64:
-    uses: ./.github/workflows/cc-payload-after-push-arm64.yaml
+    uses: ./.github/workflows/payload-after-push-arm64.yaml
     with:
       target-arch: arm64
     secrets: inherit
 
   build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
+    uses: ./.github/workflows/payload-after-push-s390x.yaml
     with:
       target-arch: s390x
     secrets: inherit

From 8030e469b22083e7ecf9615555e57a509807655a Mon Sep 17 00:00:00 2001
From: Yipeng Yin <yinyipeng@bytedance.com>
Date: Thu, 2 Mar 2023 15:46:05 +0800
Subject: [PATCH 62/76] fix(runtime-rs): add exited state to ensure cleanup

Set process status to exited at end of io wait, which indicate process
exited only, but stop process has not been finished. Otherwise, the
cleanup_container will be skipped.

Fixes: #6393

Signed-off-by: Yipeng Yin <yinyipeng@bytedance.com>
---
 .../crates/runtimes/common/src/types/mod.rs   |  1 +
 .../common/src/types/trans_into_shim.rs       |  1 +
 .../src/container_manager/container_inner.rs  | 24 ++++++++++---------
 .../src/container_manager/process.rs          |  2 +-
 4 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/runtime-rs/crates/runtimes/common/src/types/mod.rs b/src/runtime-rs/crates/runtimes/common/src/types/mod.rs
index 0e6f80a4f..3772a8a7c 100644
--- a/src/runtime-rs/crates/runtimes/common/src/types/mod.rs
+++ b/src/runtime-rs/crates/runtimes/common/src/types/mod.rs
@@ -184,6 +184,7 @@ pub enum ProcessStatus {
     Stopped = 3,
     Paused = 4,
     Pausing = 5,
+    Exited = 6,
 }
 
 #[derive(Debug, Clone)]
diff --git a/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs b/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs
index 3c3134e8f..345e02d93 100644
--- a/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs
+++ b/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs
@@ -56,6 +56,7 @@ impl From<ProcessStatus> for api::Status {
             ProcessStatus::Stopped => api::Status::STOPPED,
             ProcessStatus::Paused => api::Status::PAUSED,
             ProcessStatus::Pausing => api::Status::PAUSING,
+            ProcessStatus::Exited => api::Status::STOPPED,
         }
     }
 }
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs
index 5e1cbb398..b041af076 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs
@@ -200,20 +200,22 @@ impl ContainerInner {
             return Ok(());
         }
 
-        self.check_state(vec![ProcessStatus::Running])
+        self.check_state(vec![ProcessStatus::Running, ProcessStatus::Exited])
             .await
             .context("check state")?;
 
-        // if use force mode to stop container, stop always successful
-        // send kill signal to container
-        // ignore the error of sending signal, since the process would
-        // have been killed and exited yet.
-        self.signal_process(process, Signal::SIGKILL as u32, false)
-            .await
-            .map_err(|e| {
-                warn!(logger, "failed to signal kill. {:?}", e);
-            })
-            .ok();
+        if state == ProcessStatus::Running {
+            // if use force mode to stop container, stop always successful
+            // send kill signal to container
+            // ignore the error of sending signal, since the process would
+            // have been killed and exited yet.
+            self.signal_process(process, Signal::SIGKILL as u32, false)
+                .await
+                .map_err(|e| {
+                    warn!(logger, "failed to signal kill. {:?}", e);
+                })
+                .ok();
+        }
 
         match process.process_type {
             ProcessType::Container => self
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs
index 5953f2f69..438a817e2 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs
@@ -182,7 +182,7 @@ impl Process {
             drop(exit_status);
 
             let mut status = status.write().await;
-            *status = ProcessStatus::Stopped;
+            *status = ProcessStatus::Exited;
             drop(status);
 
             drop(exit_notifier);

From f8e44172f6d132de5e772d92e9b672afc71c2568 Mon Sep 17 00:00:00 2001
From: tg5788re <jfokugas@gmail.com>
Date: Fri, 3 Mar 2023 09:56:12 -0600
Subject: [PATCH 63/76] utils: Make kata-manager.sh runs checks

Updated the `kata-manager.sh` script to make it run all the checks on
the host system before attempting to create a container. If any checks
fail, they will indicate to the user what the problem is in a clearer
manner than those reported by the container manager.

Fixes: #6281.

Signed-off-by: tg5788re <jfokugas@gmail.com>
---
 utils/kata-manager.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh
index 8d78aa447..fbc911ab4 100755
--- a/utils/kata-manager.sh
+++ b/utils/kata-manager.sh
@@ -638,6 +638,8 @@ test_installation()
 {
 	info "Testing $kata_project\n"
 
+	sudo kata-runtime check -v
+
 	local image="docker.io/library/busybox:latest"
 	sudo ctr image pull "$image"
 

From 6267909501a1dc221e94ec2c6e80c7c0e1c083e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 2 Mar 2023 15:44:33 +0100
Subject: [PATCH 64/76] kata-deploy: Allow passing BASE_IMAGE_{NAME,TAG}
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's break the IMAGE build parameter into BASE_IMAGE_NAME and
BASE_IMAGE_TAG, as it makes it easier to replace the default CentOS
image by something else.

Spoiler alert, the default CentOS image is **not** multi-arch, and we do
want to support at least aarch64 and s390x in the near term future.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/Dockerfile                    | 5 +++--
 .../local-build/kata-deploy-build-and-upload-payload.sh   | 8 ++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile
index 94533a906..1302a3e6d 100644
--- a/tools/packaging/kata-deploy/Dockerfile
+++ b/tools/packaging/kata-deploy/Dockerfile
@@ -3,8 +3,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Specify alternative base image, e.g. clefos for s390x
-ARG IMAGE
-FROM ${IMAGE:-registry.centos.org/centos}:7
+ARG BASE_IMAGE_NAME=registry.centos.org/centos
+ARG BASE_IMAGE_TAG=7
+FROM $BASE_IMAGE_NAME:$BASE_IMAGE_TAG
 ARG KATA_ARTIFACTS=./kata-static.tar.xz
 ARG DESTINATION=/opt/kata-artifacts
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index 29ea59b17..e1403fd76 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -20,8 +20,8 @@ IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-$(uname -m)"
 echo "Building the image"
 if [ "$(uname -m)" = "s390x" ]; then
 	docker build \
-		--build-arg IMG_NAME=clefos \
-		--build-arg IMG_TAG=7 \
+		--build-arg BASE_IMAGE_NAME=clefos \
+		--build-arg BASE_IMAGE_TAG=7 \
 		--tag ${IMAGE_TAG} .
 else
 	docker build --tag ${IMAGE_TAG} .
@@ -36,8 +36,8 @@ if [ -n "${TAG}" ]; then
 	echo "Building the ${ADDITIONAL_TAG} image"
 	if [ "$(uname -m)" = "s390x" ]; then
 		docker build \
-			--build-arg IMG_NAME=clefos \
-			--build-arg IMG_TAG=7 \
+			--build-arg BASE_IMAGE_NAME=clefos \
+			--build-arg BASE_IMAGE_TAG=7 \
 			--tag ${ADDITIONAL_TAG} .
 	else
 		docker build --tag ${ADDITIONAL_TAG} .

From 78ba363f8e81ff5d6c26fe534a2f2eaa832cc1cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 2 Mar 2023 15:49:01 +0100
Subject: [PATCH 65/76] kata-deploy: Use different images for s390x and aarch64
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As the image provided as part of registry.centos.org is not a multi-arch
one, at least not for CentOS 7, we need to expand the script used to
build the image to pass images that are known to work for s390x (ClefOS)
and aarch64 (CentOS, but coming from dockerhub).

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../kata-deploy-build-and-upload-payload.sh   | 51 +++++++++++++------
 1 file changed, 35 insertions(+), 16 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index e1403fd76..8e0290afe 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -18,14 +18,23 @@ pushd ${KATA_DEPLOY_DIR}
 IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-$(uname -m)"
 
 echo "Building the image"
-if [ "$(uname -m)" = "s390x" ]; then
-	docker build \
-		--build-arg BASE_IMAGE_NAME=clefos \
-		--build-arg BASE_IMAGE_TAG=7 \
-		--tag ${IMAGE_TAG} .
-else
-	docker build --tag ${IMAGE_TAG} .
-fi
+case $(uname -m) in
+	aarch64)
+		docker build \
+			--build-arg BASE_IMAGE_NAME=cdocker.io/library/centos \
+			--build-arg BASE_IMAGE_TAG=7 \
+			--tag ${IMAGE_TAG} .
+		;;
+	s390x)
+		docker build \
+			--build-arg BASE_IMAGE_NAME=docker.io/library/clefos \
+			--build-arg BASE_IMAGE_TAG=7 \
+			--tag ${IMAGE_TAG} .
+		;;
+	*)
+		docker build --tag ${IMAGE_TAG} .
+		;;
+esac
 
 echo "Pushing the image to quay.io"
 docker push ${IMAGE_TAG}
@@ -34,14 +43,24 @@ if [ -n "${TAG}" ]; then
 	ADDITIONAL_TAG="${REGISTRY}:${TAG}"
 
 	echo "Building the ${ADDITIONAL_TAG} image"
-	if [ "$(uname -m)" = "s390x" ]; then
-		docker build \
-			--build-arg BASE_IMAGE_NAME=clefos \
-			--build-arg BASE_IMAGE_TAG=7 \
-			--tag ${ADDITIONAL_TAG} .
-	else
-		docker build --tag ${ADDITIONAL_TAG} .
-	fi
+
+	case $(uname -m) in
+		aarch64)
+			docker build \
+				--build-arg BASE_IMAGE_NAME=docker.io/library/centos  \
+				--build-arg BASE_IMAGE_TAG=7 \
+				--tag ${ADDITIONAL_TAG} .
+			;;
+		s390x)
+			docker build \
+				--build-arg BASE_IMAGE_NAME=docker.io/library/clefos \
+				--build-arg BASE_IMAGE_TAG=7 \
+				--tag ${ADDITIONAL_TAG} .
+			;;
+		*)
+			docker build --tag ${ADDITIONAL_TAG} .
+			;;
+	esac
 
 	echo "Pushing the image ${ADDITIONAL_TAG} to quay.io"
 	docker push ${ADDITIONAL_TAG}

From 9bc7bef3d62213169bd7bed1c110a25a6a5062a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 2 Mar 2023 16:13:07 +0100
Subject: [PATCH 66/76] kata-deploy: Fix path to the Dockerfile
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As part of bd1ed26c8d0ef8da99bc2ffffb099f93f73a3880, we've pointed to
the Dockerfile that's used in the CC branch, which is wrong.

For what we're doing on main, we should be pointing to the one under the
`kata-deploy` folder, and not the one under the non-existent
`kata-deploy-cc` one.

Fixes: #6343

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../local-build/kata-deploy-build-and-upload-payload.sh         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index 8e0290afe..ffb835eb9 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-KATA_DEPLOY_DIR="`dirname ${0}`/../../kata-deploy-cc"
+KATA_DEPLOY_DIR="`dirname ${0}`/../../kata-deploy"
 KATA_DEPLOY_ARTIFACT="${1:-"kata-static.tar.xz"}"
 REGISTRY="${2:-"quay.io/kata-containers/kata-deploy"}"
 TAG="${3:-}"

From 760f78137db0c3b2806a53a10d425f4c6bc863fe Mon Sep 17 00:00:00 2001
From: xuejun-xj <jiyunxue@linux.alibaba.com>
Date: Mon, 6 Mar 2023 10:36:15 +0800
Subject: [PATCH 67/76] dragonball: support pmu on aarch64

This commit adds support for pmu virtualization on aarch64. The
initialization of pmu is in the following order:
1. Receive pmu parameter(vpmu_feature) from runtime-rs to determine the
VpmuFeatureLevel.
2. Judge whether to initialize pmu devices and add pmu device node into
fdt on aarch64, according to VpmuFeatureLevel.

Fixes: #6168

Signed-off-by: xuejun-xj <jiyunxue@linux.alibaba.com>
---
 src/dragonball/src/error.rs             |  7 +++++++
 src/dragonball/src/vcpu/aarch64.rs      |  7 +++++--
 src/dragonball/src/vcpu/mod.rs          |  6 ++----
 src/dragonball/src/vcpu/vcpu_impl.rs    |  5 +++++
 src/dragonball/src/vcpu/vcpu_manager.rs | 24 +++++++++++++++++++-----
 src/dragonball/src/vm/aarch64.rs        | 25 +++++++++++++++++++++++--
 src/dragonball/src/vm/mod.rs            |  7 +++++++
 7 files changed, 68 insertions(+), 13 deletions(-)

diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs
index 35f092a50..66a24f562 100644
--- a/src/dragonball/src/error.rs
+++ b/src/dragonball/src/error.rs
@@ -9,6 +9,8 @@
 
 //! Error codes for the virtual machine monitor subsystem.
 
+#[cfg(target_arch = "aarch64")]
+use dbs_arch::pmu::PmuError;
 #[cfg(feature = "dbs-virtio-devices")]
 use dbs_virtio_devices::Error as VirtIoError;
 
@@ -61,6 +63,11 @@ pub enum Error {
     #[error("failed to write MP table to guest memory: {0}")]
     MpTableSetup(#[source] dbs_boot::mptable::Error),
 
+    /// Create pmu device error
+    #[cfg(target_arch = "aarch64")]
+    #[error("Create pmu device error: {0}")]
+    PmuDeviceError(#[source] PmuError),
+
     /// Fail to boot system
     #[error("failed to boot system: {0}")]
     BootSystem(#[source] dbs_boot::Error),
diff --git a/src/dragonball/src/vcpu/aarch64.rs b/src/dragonball/src/vcpu/aarch64.rs
index dc4b9c61a..ae45cd99d 100644
--- a/src/dragonball/src/vcpu/aarch64.rs
+++ b/src/dragonball/src/vcpu/aarch64.rs
@@ -11,7 +11,7 @@ use std::sync::mpsc::{channel, Sender};
 use std::sync::Arc;
 
 use crate::IoManagerCached;
-use dbs_arch::regs;
+use dbs_arch::{regs, VpmuFeatureLevel};
 use dbs_boot::get_fdt_addr;
 use dbs_utils::time::TimestampUs;
 use kvm_ioctls::{VcpuFd, VmFd};
@@ -81,7 +81,7 @@ impl Vcpu {
     /// * `_pgtable_addr` - pgtable address for ap vcpu (not used in aarch64)
     pub fn configure(
         &mut self,
-        _vcpu_config: &VcpuConfig,
+        vcpu_config: &VcpuConfig,
         vm_fd: &VmFd,
         vm_as: &GuestAddressSpaceImpl,
         kernel_load_addr: Option<GuestAddress>,
@@ -99,6 +99,9 @@ impl Vcpu {
         if self.id > 0 {
             kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
         }
+        if vcpu_config.vpmu_feature == VpmuFeatureLevel::FullyEnabled {
+            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
+        }
 
         self.fd.vcpu_init(&kvi).map_err(VcpuError::VcpuArmInit)?;
 
diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs
index b04baf29f..0011d81bd 100644
--- a/src/dragonball/src/vcpu/mod.rs
+++ b/src/dragonball/src/vcpu/mod.rs
@@ -7,9 +7,7 @@ mod sm;
 mod vcpu_impl;
 mod vcpu_manager;
 
-#[cfg(target_arch = "x86_64")]
-use dbs_arch::cpuid::VpmuFeatureLevel;
-
+use dbs_arch::VpmuFeatureLevel;
 pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo};
 
 #[cfg(feature = "hotplug")]
@@ -32,6 +30,6 @@ pub struct VcpuConfig {
     /// if vpmu feature is Disabled, it means vpmu feature is off (by default)
     /// if vpmu feature is LimitedlyEnabled, it means minimal vpmu counters are supported (cycles and instructions)
     /// if vpmu feature is FullyEnabled, it means all vpmu counters are supported
-    #[cfg(target_arch = "x86_64")]
+    /// For aarch64, VpmuFeatureLevel only supports Disabled and FullyEnabled.
     pub vpmu_feature: VpmuFeatureLevel,
 }
diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs
index 26f48c6d1..3dffd579f 100644
--- a/src/dragonball/src/vcpu/vcpu_impl.rs
+++ b/src/dragonball/src/vcpu/vcpu_impl.rs
@@ -760,6 +760,11 @@ impl Vcpu {
         // State machine reached its end.
         StateMachine::finish(Self::exited)
     }
+
+    /// Get vcpu file descriptor.
+    pub fn vcpu_fd(&self) -> &VcpuFd {
+        self.fd.as_ref()
+    }
 }
 
 impl Drop for Vcpu {
diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs
index 7b93c1a82..8f39af519 100644
--- a/src/dragonball/src/vcpu/vcpu_manager.rs
+++ b/src/dragonball/src/vcpu/vcpu_manager.rs
@@ -15,6 +15,7 @@ use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
 use std::sync::{Arc, Barrier, Mutex, RwLock};
 use std::time::Duration;
 
+use dbs_arch::VpmuFeatureLevel;
 #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
 use dbs_upcall::{DevMgrService, UpcallClient};
 use dbs_utils::epoll_manager::{EpollManager, EventOps, EventSet, Events, MutEventSubscriber};
@@ -281,11 +282,20 @@ impl VcpuManager {
         let supported_cpuid = kvm_context
             .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES)
             .map_err(VcpuManagerError::Kvm)?;
-        #[cfg(target_arch = "x86_64")]
+        #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
         let vpmu_feature_level = match vm_config_info.vpmu_feature {
-            1 => dbs_arch::cpuid::VpmuFeatureLevel::LimitedlyEnabled,
-            2 => dbs_arch::cpuid::VpmuFeatureLevel::FullyEnabled,
-            _ => dbs_arch::cpuid::VpmuFeatureLevel::Disabled,
+            #[cfg(target_arch = "x86_64")]
+            1 => VpmuFeatureLevel::LimitedlyEnabled,
+            #[cfg(target_arch = "aarch64")]
+            1 => {
+                log::warn!(
+                    "Limitedly enabled vpmu feature isn't supported on aarch64 for now.\
+                       This will be supported in the future. The vpmu_feature will be set disabled!"
+                );
+                VpmuFeatureLevel::Disabled
+            }
+            2 => VpmuFeatureLevel::FullyEnabled,
+            _ => VpmuFeatureLevel::Disabled,
         };
 
         let vcpu_manager = Arc::new(Mutex::new(VcpuManager {
@@ -297,7 +307,6 @@ impl VcpuManager {
                 cores_per_die: vm_config_info.cpu_topology.cores_per_die,
                 dies_per_socket: vm_config_info.cpu_topology.dies_per_socket,
                 sockets: vm_config_info.cpu_topology.sockets,
-                #[cfg(target_arch = "x86_64")]
                 vpmu_feature: vpmu_feature_level,
             },
             vcpu_seccomp_filter,
@@ -799,6 +808,11 @@ impl VcpuManager {
         )
         .map_err(VcpuManagerError::Vcpu)
     }
+
+    /// get vpmu_feature config
+    pub fn vpmu_feature(&self) -> VpmuFeatureLevel {
+        self.vcpu_config.vpmu_feature
+    }
 }
 
 #[cfg(feature = "hotplug")]
diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs
index 0ec8251f4..edc753245 100644
--- a/src/dragonball/src/vm/aarch64.rs
+++ b/src/dragonball/src/vm/aarch64.rs
@@ -11,6 +11,7 @@ use std::fmt::Debug;
 use std::ops::Deref;
 
 use dbs_arch::gic::GICDevice;
+use dbs_arch::pmu::initialize_pmu;
 use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel};
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
@@ -43,6 +44,7 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
     device_info: Option<&HashMap<(DeviceType, String), T>>,
     gic_device: &Box<dyn GICDevice>,
     initrd: &Option<super::InitrdConfig>,
+    vpmu_feature: &VpmuFeatureLevel,
 ) -> super::Result<()> {
     dbs_boot::fdt::create_fdt(
         guest_mem,
@@ -51,8 +53,7 @@ fn configure_system<T: DeviceInfoForFDT + Clone + Debug, M: GuestMemory>(
         device_info,
         gic_device,
         initrd,
-        // We will add vpmu feature support in the future PRs. issue: #6168
-        &VpmuFeatureLevel::Disabled,
+        vpmu_feature,
     )
     .map_err(Error::BootSystem)?;
     Ok(())
@@ -78,6 +79,23 @@ impl Vm {
         Ok(())
     }
 
+    /// Setup pmu devices for guest vm.
+    pub fn setup_pmu_devices(&mut self) -> std::result::Result<(), StartMicroVmError> {
+        let vm = self.vm_fd();
+        let mut vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
+        let vpmu_feature = vcpu_manager.vpmu_feature();
+        if vpmu_feature == VpmuFeatureLevel::Disabled {
+            return Ok(());
+        }
+
+        for vcpu in vcpu_manager.vcpus_mut() {
+            initialize_pmu(vm, vcpu.vcpu_fd())
+                .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupPmu(e)))?;
+        }
+
+        Ok(())
+    }
+
     /// Initialize the virtual machine instance.
     ///
     /// It initialize the virtual machine instance by:
@@ -115,6 +133,7 @@ impl Vm {
             .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load)
             .map_err(StartMicroVmError::Vcpu)?;
         self.setup_interrupt_controller()?;
+        self.setup_pmu_devices()?;
         self.init_devices(epoll_mgr)?;
 
         Ok(())
@@ -131,6 +150,7 @@ impl Vm {
         initrd: Option<InitrdConfig>,
     ) -> std::result::Result<(), StartMicroVmError> {
         let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
+        let vpmu_feature = vcpu_manager.vpmu_feature();
         let vcpu_mpidr = vcpu_manager
             .vcpus()
             .into_iter()
@@ -145,6 +165,7 @@ impl Vm {
             self.device_manager.get_mmio_device_info(),
             self.get_irqchip(),
             &initrd,
+            &vpmu_feature,
         )
         .map_err(StartMicroVmError::ConfigureSystem)
     }
diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs
index 13e6584b8..a9ccb02ea 100644
--- a/src/dragonball/src/vm/mod.rs
+++ b/src/dragonball/src/vm/mod.rs
@@ -10,6 +10,8 @@ use std::sync::{Arc, Mutex, RwLock};
 use dbs_address_space::AddressSpace;
 #[cfg(target_arch = "aarch64")]
 use dbs_arch::gic::GICDevice;
+#[cfg(target_arch = "aarch64")]
+use dbs_arch::pmu::PmuError;
 use dbs_boot::InitrdConfig;
 use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
@@ -69,6 +71,11 @@ pub enum VmError {
     #[cfg(target_arch = "aarch64")]
     #[error("failed to configure GIC")]
     SetupGIC(GICError),
+
+    /// Cannot setup pmu device
+    #[cfg(target_arch = "aarch64")]
+    #[error("failed to setup pmu device")]
+    SetupPmu(#[source] PmuError),
 }
 
 /// Configuration information for user defined NUMA nodes.

From 2d43e131022c398fa3ec9e7927a99a5daf5db346 Mon Sep 17 00:00:00 2001
From: Szymon Fugas <sz.fugas@gmail.com>
Date: Fri, 3 Mar 2023 15:58:11 +0100
Subject: [PATCH 68/76] docs: fix typo in AWS installation guide

Fixes referring to previously created key file with .pen extension instead of .pem.

Fixes: #6412
Signed-off-by: Sig00rd <sfugas@virtuslab.com>
---
 docs/install/aws-installation-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/install/aws-installation-guide.md b/docs/install/aws-installation-guide.md
index ee8494d60..d1bac643d 100644
--- a/docs/install/aws-installation-guide.md
+++ b/docs/install/aws-installation-guide.md
@@ -123,7 +123,7 @@ Refer to [this guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-l
 SSH into the machine
 
 ```bash
-$ ssh -i MyKeyPair.pen ubuntu@${IP}
+$ ssh -i MyKeyPair.pem ubuntu@${IP}
 ```
 
 Go onto the next step.

From cd2aaeda2a07d2605634cc381b1bb3d68fe562a6 Mon Sep 17 00:00:00 2001
From: SinghWang <wangxin_0611@126.com>
Date: Thu, 23 Feb 2023 17:07:40 +0800
Subject: [PATCH 69/76] kata-deploy: Switch to using an ubuntu image

Let's make sure we use a multi-arch image for building kata-deploy.
A few changes were also added in order to get systemd working inside the
kata-deploy image, due to the switch from CentOS to Ubuntu.

Fixes: #6358
Signed-off-by: SinghWang <wangxin_0611@126.com>
---
 tools/packaging/kata-deploy/Dockerfile        | 27 +++++++++----------
 .../kata-cleanup/base/kata-cleanup.yaml       | 10 +++----
 .../kata-deploy/base/kata-deploy.yaml         | 10 +++----
 3 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile
index 1302a3e6d..ee5519cd8 100644
--- a/tools/packaging/kata-deploy/Dockerfile
+++ b/tools/packaging/kata-deploy/Dockerfile
@@ -3,28 +3,27 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Specify alternative base image, e.g. clefos for s390x
-ARG BASE_IMAGE_NAME=registry.centos.org/centos
-ARG BASE_IMAGE_TAG=7
+ARG BASE_IMAGE_NAME=ubuntu
+ARG BASE_IMAGE_TAG=20.04
 FROM $BASE_IMAGE_NAME:$BASE_IMAGE_TAG
+ENV DEBIAN_FRONTEND=noninteractive
 ARG KATA_ARTIFACTS=./kata-static.tar.xz
 ARG DESTINATION=/opt/kata-artifacts
 
 COPY ${KATA_ARTIFACTS} ${WORKDIR}
 
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 RUN \
-yum -y update && \
-yum -y install xz && \
-yum clean all && \
+apt-get update && \
+apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \
+mkdir -p /etc/apt/keyrings/ && \
+curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \
+echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \
+apt-get update && \
+apt-get install -y --no-install-recommends kubectl && \
+apt-get clean && rm -rf /var/lib/apt/lists/ && \
 mkdir -p ${DESTINATION} && \
 tar xvf ${KATA_ARTIFACTS} -C ${DESTINATION}
 
-# hadolint will deny echo -e, heredocs don't work in Dockerfiles, shell substitution doesn't work with $'...'
-RUN \
-echo "[kubernetes]" >> /etc/yum.repos.d/kubernetes.repo && \
-echo "name=Kubernetes" >> /etc/yum.repos.d/kubernetes.repo && \
-echo "baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$(uname -m)" >> /etc/yum.repos.d/kubernetes.repo && \
-echo "gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg" >> /etc/yum.repos.d/kubernetes.repo && \
-yum -y install kubectl && \
-yum clean all
-
 COPY scripts ${DESTINATION}/scripts
diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
index 851e958a4..095876b73 100644
--- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
@@ -27,19 +27,19 @@ spec:
             fieldRef:
               fieldPath: spec.nodeName
         securityContext:
-          privileged: false
+          privileged: true
         volumeMounts:
         - name: dbus
-          mountPath: /var/run/dbus
+          mountPath: /var/run/dbus/system_bus_socket
         - name: systemd
-          mountPath: /run/systemd
+          mountPath: /run/systemd/system
       volumes:
         - name: dbus
           hostPath:
-            path: /var/run/dbus
+            path: /var/run/dbus/system_bus_socket
         - name: systemd
           hostPath:
-            path: /run/systemd
+            path: /run/systemd/system
   updateStrategy:
     rollingUpdate:
       maxUnavailable: 1
diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
index a03a56b84..97e98ee74 100644
--- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -29,7 +29,7 @@ spec:
             fieldRef:
               fieldPath: spec.nodeName
         securityContext:
-          privileged: false
+          privileged: true
         volumeMounts:
         - name: crio-conf
           mountPath: /etc/crio/
@@ -38,9 +38,9 @@ spec:
         - name: kata-artifacts
           mountPath: /opt/kata/
         - name: dbus
-          mountPath: /var/run/dbus
+          mountPath: /var/run/dbus/system_bus_socket
         - name: systemd
-          mountPath: /run/systemd
+          mountPath: /run/systemd/system
         - name: local-bin
           mountPath: /usr/local/bin/
       volumes:
@@ -56,10 +56,10 @@ spec:
             type: DirectoryOrCreate
         - name: dbus
           hostPath:
-            path: /var/run/dbus
+            path: /var/run/dbus/system_bus_socket
         - name: systemd
           hostPath:
-            path: /run/systemd
+            path: /run/systemd/system
         - name: local-bin
           hostPath:
             path: /usr/local/bin/

From 76b4591e2b0929024144f9afbcd29bd248f80f75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 6 Mar 2023 11:07:33 +0100
Subject: [PATCH 70/76] tools: Adjust the build-and-upload-payload.sh script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Now that we've switched the base container image to using Ubuntu instead
of CentOS, we don't need any kind of extra logic to correctly build the
image for different architectures, as Ubuntu is a multi-arch image that
supports all the architectures we're targetting.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../kata-deploy-build-and-upload-payload.sh   | 36 ++-----------------
 1 file changed, 2 insertions(+), 34 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index ffb835eb9..c4ff1abf5 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -18,23 +18,7 @@ pushd ${KATA_DEPLOY_DIR}
 IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-$(uname -m)"
 
 echo "Building the image"
-case $(uname -m) in
-	aarch64)
-		docker build \
-			--build-arg BASE_IMAGE_NAME=cdocker.io/library/centos \
-			--build-arg BASE_IMAGE_TAG=7 \
-			--tag ${IMAGE_TAG} .
-		;;
-	s390x)
-		docker build \
-			--build-arg BASE_IMAGE_NAME=docker.io/library/clefos \
-			--build-arg BASE_IMAGE_TAG=7 \
-			--tag ${IMAGE_TAG} .
-		;;
-	*)
-		docker build --tag ${IMAGE_TAG} .
-		;;
-esac
+docker build --tag ${IMAGE_TAG} .
 
 echo "Pushing the image to quay.io"
 docker push ${IMAGE_TAG}
@@ -44,23 +28,7 @@ if [ -n "${TAG}" ]; then
 
 	echo "Building the ${ADDITIONAL_TAG} image"
 
-	case $(uname -m) in
-		aarch64)
-			docker build \
-				--build-arg BASE_IMAGE_NAME=docker.io/library/centos  \
-				--build-arg BASE_IMAGE_TAG=7 \
-				--tag ${ADDITIONAL_TAG} .
-			;;
-		s390x)
-			docker build \
-				--build-arg BASE_IMAGE_NAME=docker.io/library/clefos \
-				--build-arg BASE_IMAGE_TAG=7 \
-				--tag ${ADDITIONAL_TAG} .
-			;;
-		*)
-			docker build --tag ${ADDITIONAL_TAG} .
-			;;
-	esac
+	docker build --tag ${ADDITIONAL_TAG} .
 
 	echo "Pushing the image ${ADDITIONAL_TAG} to quay.io"
 	docker push ${ADDITIONAL_TAG}

From dbae281924b3327755af1ecd0023b792097212ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 6 Mar 2023 11:30:27 +0100
Subject: [PATCH 71/76] workflows: Properly set the kata-tarball architecture
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's make sure the kata-tarball architecture upload / downloaded / used
is exactly the same one that we need as part of the architecture we're
using to generate the image.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/payload-after-push-amd64.yaml | 8 ++++----
 .github/workflows/payload-after-push-arm64.yaml | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/payload-after-push-amd64.yaml b/.github/workflows/payload-after-push-amd64.yaml
index 88497ca45..2b4814b84 100644
--- a/.github/workflows/payload-after-push-amd64.yaml
+++ b/.github/workflows/payload-after-push-amd64.yaml
@@ -45,7 +45,7 @@ jobs:
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
         with:
-          name: kata-artifacts
+          name: kata-artifacts-amd64
           path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -58,7 +58,7 @@ jobs:
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
-          name: kata-artifacts
+          name: kata-artifacts-amd64
           path: kata-artifacts
       - name: merge-artifacts
         run: |
@@ -66,7 +66,7 @@ jobs:
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: kata-static-tarball
+          name: kata-static-tarball-amd64
           path: kata-static.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -86,7 +86,7 @@ jobs:
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:
-          name: kata-static-tarball
+          name: kata-static-tarball-amd64
 
       - name: build-and-push-kata-payload
         id: build-and-push-kata-payload
diff --git a/.github/workflows/payload-after-push-arm64.yaml b/.github/workflows/payload-after-push-arm64.yaml
index 56faeb10e..c7315bab0 100644
--- a/.github/workflows/payload-after-push-arm64.yaml
+++ b/.github/workflows/payload-after-push-arm64.yaml
@@ -49,7 +49,7 @@ jobs:
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
         with:
-          name: kata-artifacts
+          name: kata-artifacts-arm64
           path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -66,7 +66,7 @@ jobs:
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:
-          name: kata-artifacts
+          name: kata-artifacts-arm64
           path: kata-artifacts
       - name: merge-artifacts
         run: |
@@ -74,7 +74,7 @@ jobs:
       - name: store-artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: kata-static-tarball
+          name: kata-static-tarball-arm64
           path: kata-static.tar.xz
           retention-days: 1
           if-no-files-found: error
@@ -98,7 +98,7 @@ jobs:
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:
-          name: kata-static-tarball
+          name: kata-static-tarball-arm64
 
       - name: build-and-push-kata-payload
         id: build-and-push-kata-payload

From 736aae47a4d2df9eed086fadb5eb5d872b66dd1e Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 10 Feb 2023 14:34:53 +0100
Subject: [PATCH 72/76] rustjail: print type of cgroup manager

Since the cgroup manager is wrapped in a dyn now, the print in
LinuxContainer::new has been useless and just says "CgroupManager". Extend the
Debug trait for 'dyn Manager' to print the type of the cgroup manager so that
it's easier to debug issues.

Fixes: #5779
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/rustjail/src/cgroups/fs/mod.rs          | 4 ++++
 src/agent/rustjail/src/cgroups/mock.rs            | 4 ++++
 src/agent/rustjail/src/cgroups/mod.rs             | 4 +++-
 src/agent/rustjail/src/cgroups/systemd/manager.rs | 4 ++++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs
index becc56036..6eaa9870d 100644
--- a/src/agent/rustjail/src/cgroups/fs/mod.rs
+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs
@@ -267,6 +267,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "cgroupfs"
+    }
 }
 
 fn set_network_resources(
diff --git a/src/agent/rustjail/src/cgroups/mock.rs b/src/agent/rustjail/src/cgroups/mock.rs
index fbad6d664..3bcc99955 100644
--- a/src/agent/rustjail/src/cgroups/mock.rs
+++ b/src/agent/rustjail/src/cgroups/mock.rs
@@ -66,6 +66,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "mock"
+    }
 }
 
 impl Manager {
diff --git a/src/agent/rustjail/src/cgroups/mod.rs b/src/agent/rustjail/src/cgroups/mod.rs
index 24275c314..c4e3b178b 100644
--- a/src/agent/rustjail/src/cgroups/mod.rs
+++ b/src/agent/rustjail/src/cgroups/mod.rs
@@ -52,10 +52,12 @@ pub trait Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Err(anyhow!("not supported!"))
     }
+
+    fn name(&self) -> &str;
 }
 
 impl Debug for dyn Manager + Send + Sync {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-        write!(f, "CgroupManager")
+        write!(f, "{}", self.name())
     }
 }
diff --git a/src/agent/rustjail/src/cgroups/systemd/manager.rs b/src/agent/rustjail/src/cgroups/systemd/manager.rs
index 481ef1e60..c52e727e4 100644
--- a/src/agent/rustjail/src/cgroups/systemd/manager.rs
+++ b/src/agent/rustjail/src/cgroups/systemd/manager.rs
@@ -101,6 +101,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "systemd"
+    }
 }
 
 impl Manager {

From 8546387348877682df0437a2a35c5b74a428cad8 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 24 Feb 2023 13:43:26 +0100
Subject: [PATCH 73/76] agent: determine value of use_systemd_cgroup before
 LinuxContainer::new()

Right now LinuxContainer::new() gets passed a CreateOpts struct, but then
modifies the use_systemd_cgroup field inside that struct. Pull the cgroups path
parsing logic into do_create_container, so that CreateOpts can be immutable in
LinuxContainer::new. This is just moving things around, there should be no
functional changes.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/rustjail/src/container.rs | 11 ++---------
 src/agent/src/rpc.rs                | 10 ++++++++--
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
index f75b6e6d5..ed69ef60d 100644
--- a/src/agent/rustjail/src/container.rs
+++ b/src/agent/rustjail/src/container.rs
@@ -1449,7 +1449,7 @@ impl LinuxContainer {
     pub fn new<T: Into<String> + Display + Clone>(
         id: T,
         base: T,
-        mut config: Config,
+        config: Config,
         logger: &Logger,
     ) -> Result<Self> {
         let base = base.into();
@@ -1475,21 +1475,14 @@ impl LinuxContainer {
         .context(format!("Cannot change owner of container {} root", id))?;
 
         let spec = config.spec.as_ref().unwrap();
-
         let linux = spec.linux.as_ref().unwrap();
-
-        // determine which cgroup driver to take and then assign to config.use_systemd_cgroup
-        // systemd: "[slice]:[prefix]:[name]"
-        // fs: "/path_a/path_b"
-        let cpath = if SYSTEMD_CGROUP_PATH_FORMAT.is_match(linux.cgroups_path.as_str()) {
-            config.use_systemd_cgroup = true;
+        let cpath = if config.use_systemd_cgroup {
             if linux.cgroups_path.len() == 2 {
                 format!("system.slice:kata_agent:{}", id.as_str())
             } else {
                 linux.cgroups_path.clone()
             }
         } else {
-            config.use_systemd_cgroup = false;
             if linux.cgroups_path.is_empty() {
                 format!("/{}", id.as_str())
             } else {
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
index c52d866d6..008176541 100644
--- a/src/agent/src/rpc.rs
+++ b/src/agent/src/rpc.rs
@@ -36,7 +36,7 @@ use protocols::health::{
 use protocols::types::Interface;
 use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
 use rustjail::cgroups::notifier;
-use rustjail::container::{BaseContainer, Container, LinuxContainer};
+use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
 use rustjail::process::Process;
 use rustjail::specconv::CreateOpts;
 
@@ -210,9 +210,15 @@ impl AgentService {
         // restore the cwd for kata-agent process.
         defer!(unistd::chdir(&olddir).unwrap());
 
+        // determine which cgroup driver to take and then assign to use_systemd_cgroup
+        // systemd: "[slice]:[prefix]:[name]"
+        // fs: "/path_a/path_b"
+        let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path);
+        let use_systemd_cgroup = SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path);
+
         let opts = CreateOpts {
             cgroup_name: "".to_string(),
-            use_systemd_cgroup: false,
+            use_systemd_cgroup,
             no_pivot_root: s.no_pivot_root,
             no_new_keyring: false,
             spec: Some(oci.clone()),

From 23488312f5459a151e93d70b972d43197db50645 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 10 Feb 2023 14:50:54 +0100
Subject: [PATCH 74/76] agent: always use cgroupfs when running as init

The logic to decide which cgroup driver is used is currently based on the
cgroup path that the host provides. This requires host and guest to use the
same cgroup driver. If the guest uses kata-agent as init, then systemd can't be
used as the cgroup driver. If the host requests a systemd cgroup, this
currently results in a rustjail panic:

  thread 'tokio-runtime-worker' panicked at 'called `Result::unwrap()` on an `Err` value: I/O error: No such file or directory (os error 2)

  Caused by:
      No such file or directory (os error 2)', rustjail/src/cgroups/systemd/manager.rs:44:51
  stack backtrace:
     0:     0x7ff0fe77a793 - std::backtrace_rs::backtrace::libunwind::trace::h8c197fa9a679d134
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
     1:     0x7ff0fe77a793 - std::backtrace_rs::backtrace::trace_unsynchronized::h9ee19d58b6d5934a
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
     2:     0x7ff0fe77a793 - std::sys_common::backtrace::_print_fmt::h4badc450600fc417
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:65:5
     3:     0x7ff0fe77a793 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::had334ddb529a2169
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:44:22
     4:     0x7ff0fdce815e - core::fmt::write::h1aa7694f03e44db2
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/fmt/mod.rs:1209:17
     5:     0x7ff0fe74e0c4 - std::io::Write::write_fmt::h61b2bdc565be41b5
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/io/mod.rs:1682:15
     6:     0x7ff0fe77cd3f - std::sys_common::backtrace::_print::h4ec69798b72ff254
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:47:5
     7:     0x7ff0fe77cd3f - std::sys_common::backtrace::print::h0e6c02048dec3c77
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:34:9
     8:     0x7ff0fe77c93f - std::panicking::default_hook::{{closure}}::hcdb7e705dc37ea6e
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:267:22
     9:     0x7ff0fe77d9b8 - std::panicking::default_hook::he03a933a0f01790f
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:286:9
    10:     0x7ff0fe77d9b8 - std::panicking::rust_panic_with_hook::he26b680bfd953008
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:688:13
    11:     0x7ff0fe77d482 - std::panicking::begin_panic_handler::{{closure}}::h559120d2dd1c6180
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:579:13
    12:     0x7ff0fe77d3ec - std::sys_common::backtrace::__rust_end_short_backtrace::h36db621fc93b005a
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:137:18
    13:     0x7ff0fe77d3c1 - rust_begin_unwind
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:575:5
    14:     0x7ff0fda52ee2 - core::panicking::panic_fmt::he7679b415d25c5f4
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/panicking.rs:65:14
    15:     0x7ff0fda53182 - core::result::unwrap_failed::hb71caff146724b6b
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/result.rs:1791:5
    16:     0x7ff0fe5bd738 - <rustjail::cgroups::systemd::manager::Manager as rustjail::cgroups::Manager>::apply::hd46958d9d807d2ca
    17:     0x7ff0fe606d80 - <rustjail::container::LinuxContainer as rustjail::container::BaseContainer>::start::{{closure}}::h1de806d91fcb878f
    18:     0x7ff0fe604a76 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h1749c148adcc235f
    19:     0x7ff0fdc0c992 - kata_agent::rpc::AgentService::do_create_container::{{closure}}::{{closure}}::hc1b87a15dfdf2f64
    20:     0x7ff0fdb80ae4 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h846a8c9e4fb67707
    21:     0x7ff0fe3bb816 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h53de16ff66ed3972
    22:     0x7ff0fdb519cb - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h1cbece980286c0f4
    23:     0x7ff0fdf4019c - <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::hc8e72d155feb8d1f
    24:     0x7ff0fdfa5fd8 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h0a407ffe2559449a
    25:     0x7ff0fdf033a1 - tokio::runtime::task::raw::poll::h1045d9f1db9742de
    26:     0x7ff0fe7a8ce2 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h4924ae3464af7fbd
    27:     0x7ff0fe7afb85 - tokio::runtime::task::raw::poll::h5c843be39646b833
    28:     0x7ff0fe7a05ee - std::sys_common::backtrace::__rust_begin_short_backtrace::ha7777c55b98a9bd1
    29:     0x7ff0fe7a9bdb - core::ops::function::FnOnce::call_once{{vtable.shim}}::h27ec83c953360cdd
    30:     0x7ff0fe7801d5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hed812350c5aef7a8
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/alloc/src/boxed.rs:1987:9
    31:     0x7ff0fe7801d5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hc7df8e435a658960
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/alloc/src/boxed.rs:1987:9
    32:     0x7ff0fe7801d5 - std::sys::unix::thread::Thread::new::thread_start::h575491a8a17dbb33
                                 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys/unix/thread.rs:108:17

Forward the value of "init_mode" to AgentService, so that we can force cgroupfs
when systemd is unavailable.

Fixes: #5779
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/rustjail/src/container.rs |  9 ++++-----
 src/agent/src/main.rs               |  2 +-
 src/agent/src/rpc.rs                | 21 +++++++++++++++++----
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
index ed69ef60d..15b7d39ce 100644
--- a/src/agent/rustjail/src/container.rs
+++ b/src/agent/rustjail/src/container.rs
@@ -1482,12 +1482,11 @@ impl LinuxContainer {
             } else {
                 linux.cgroups_path.clone()
             }
+        } else if linux.cgroups_path.is_empty() {
+            format!("/{}", id.as_str())
         } else {
-            if linux.cgroups_path.is_empty() {
-                format!("/{}", id.as_str())
-            } else {
-                linux.cgroups_path.clone()
-            }
+            // if we have a systemd cgroup path we need to convert it to a fs cgroup path
+            linux.cgroups_path.replace(':', "/")
         };
 
         let cgroup_manager: Box<dyn Manager + Send + Sync> = if config.use_systemd_cgroup {
diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs
index 2c1f41def..d8e9fc828 100644
--- a/src/agent/src/main.rs
+++ b/src/agent/src/main.rs
@@ -339,7 +339,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
     server.start().await?;
 
     rx.await?;
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
index 008176541..3be8fed5c 100644
--- a/src/agent/src/rpc.rs
+++ b/src/agent/src/rpc.rs
@@ -137,6 +137,7 @@ macro_rules! is_allowed {
 #[derive(Clone, Debug)]
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
+    init_mode: bool,
 }
 
 impl AgentService {
@@ -213,8 +214,13 @@ impl AgentService {
         // determine which cgroup driver to take and then assign to use_systemd_cgroup
         // systemd: "[slice]:[prefix]:[name]"
         // fs: "/path_a/path_b"
+        // If agent is init we can't use systemd cgroup mode, no matter what the host tells us
         let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path);
-        let use_systemd_cgroup = SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path);
+        let use_systemd_cgroup = if self.init_mode {
+            false
+        } else {
+            SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path)
+        };
 
         let opts = CreateOpts {
             cgroup_name: "".to_string(),
@@ -1679,9 +1685,11 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
     Ok(content)
 }
 
-pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> Result<TtrpcServer> {
-    let agent_service =
-        Box::new(AgentService { sandbox: s }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
+pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
+    let agent_service = Box::new(AgentService {
+        sandbox: s,
+        init_mode,
+    }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
 
     let agent_worker = Arc::new(agent_service);
 
@@ -2157,6 +2165,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::UpdateInterfaceRequest::default();
@@ -2174,6 +2183,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::UpdateRoutesRequest::default();
@@ -2191,6 +2201,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::AddARPNeighborsRequest::default();
@@ -2324,6 +2335,7 @@ mod tests {
 
             let agent_service = Box::new(AgentService {
                 sandbox: Arc::new(Mutex::new(sandbox)),
+                init_mode: true,
             });
 
             let result = agent_service
@@ -2804,6 +2816,7 @@ OtherField:other
         let sandbox = Sandbox::new(&logger).unwrap();
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let ctx = mk_ttrpc_context();

From 33c5c49719ce8816ccb4bba4fb1cda2f0a51cdc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 28 Nov 2022 21:10:52 +0100
Subject: [PATCH 75/76] packaging: Move repo_root_dir to lib.sh
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is used in several parts of the code, and can have a single
declaration as part of the `lib.sh` file, which is already imported by
all the places where it's used.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/guest-image/build_image.sh                   | 5 +++--
 .../kata-deploy/local-build/kata-deploy-binaries.sh          | 3 ++-
 tools/packaging/scripts/lib.sh                               | 2 ++
 tools/packaging/static-build/kernel/build.sh                 | 4 ++--
 tools/packaging/static-build/ovmf/build.sh                   | 1 -
 tools/packaging/static-build/qemu/build-base-qemu.sh         | 1 -
 tools/packaging/static-build/shim-v2/build.sh                | 1 -
 tools/packaging/static-build/td-shim/build.sh                | 1 -
 tools/packaging/static-build/virtiofsd/build.sh              | 1 -
 9 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh
index 3913df51d..914b72a79 100755
--- a/tools/packaging/guest-image/build_image.sh
+++ b/tools/packaging/guest-image/build_image.sh
@@ -14,11 +14,12 @@ set -o pipefail
 readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly packaging_root_dir="$(cd "${script_dir}/../" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)"
+
+source "${packaging_root_dir}/scripts/lib.sh"
+
 readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
 
 export GOPATH=${GOPATH:-${HOME}/go}
-source "${packaging_root_dir}/scripts/lib.sh"
 
 arch_target="$(uname -m)"
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index a7143096c..ded375cb6 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -15,8 +15,9 @@ readonly project="kata-containers"
 readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+source "${script_dir}/../../scripts/lib.sh"
+
 readonly prefix="/opt/kata"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build"
 readonly version_file="${repo_root_dir}/VERSION"
 readonly versions_yaml="${repo_root_dir}/versions.yaml"
diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh
index 5ac31d0e3..d83976b35 100644
--- a/tools/packaging/scripts/lib.sh
+++ b/tools/packaging/scripts/lib.sh
@@ -13,6 +13,8 @@ export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}"
 
 this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+export repo_root_dir="$(cd "${this_script_dir}/../../../" && pwd)"
+
 short_commit_length=10
 
 hub_bin="hub-bin"
diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh
index 8235ba22f..fef38b98a 100755
--- a/tools/packaging/static-build/kernel/build.sh
+++ b/tools/packaging/static-build/kernel/build.sh
@@ -9,11 +9,11 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
-readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
 
+readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
+
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
 container_image="${KERNEL_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh
index 256a89249..e18162861 100755
--- a/tools/packaging/static-build/ovmf/build.sh
+++ b/tools/packaging/static-build/ovmf/build.sh
@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly ovmf_builder="${script_dir}/build-ovmf.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh
index d5ac3e259..c27ded323 100755
--- a/tools/packaging/static-build/qemu/build-base-qemu.sh
+++ b/tools/packaging/static-build/qemu/build-base-qemu.sh
@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly qemu_builder="${script_dir}/build-qemu.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh
index 2164cf9d1..2aac61274 100755
--- a/tools/packaging/static-build/shim-v2/build.sh
+++ b/tools/packaging/static-build/shim-v2/build.sh
@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 
 source "${script_dir}/../../scripts/lib.sh"
 
diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh
index f822fd827..50642fd30 100755
--- a/tools/packaging/static-build/td-shim/build.sh
+++ b/tools/packaging/static-build/td-shim/build.sh
@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly tdshim_builder="${script_dir}/build-td-shim.sh"
 
 source "${script_dir}/../../scripts/lib.sh"
diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh
index 3a41c1211..313786eb0 100755
--- a/tools/packaging/static-build/virtiofsd/build.sh
+++ b/tools/packaging/static-build/virtiofsd/build.sh
@@ -9,7 +9,6 @@ set -o nounset
 set -o pipefail
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)"
 readonly virtiofsd_builder="${script_dir}/build-static-virtiofsd.sh"
 
 source "${script_dir}/../../scripts/lib.sh"

From 43ce3f7588c6b1e411b36f23e73a196fb4a68861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 8 Mar 2023 21:13:34 +0100
Subject: [PATCH 76/76] packaging: Simplify get_last_modification()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There's no need to pass repo_root_dir to get_last_modification() as the
variable used everywhere is exported from that very same file.

Fixes: #6431

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/scripts/lib.sh                       | 6 ++----
 tools/packaging/static-build/kernel/build.sh         | 2 +-
 tools/packaging/static-build/ovmf/build.sh           | 2 +-
 tools/packaging/static-build/qemu/build-base-qemu.sh | 2 +-
 tools/packaging/static-build/shim-v2/build.sh        | 2 +-
 tools/packaging/static-build/td-shim/build.sh        | 2 +-
 tools/packaging/static-build/virtiofsd/build.sh      | 2 +-
 7 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh
index d83976b35..10c535172 100644
--- a/tools/packaging/scripts/lib.sh
+++ b/tools/packaging/scripts/lib.sh
@@ -102,11 +102,9 @@ get_kata_hash() {
 	git ls-remote --heads --tags "https://github.com/${project}/${repo}.git" | grep "${ref}" | awk '{print $1}'
 }
 
-# $1 - Repo's root dir
-# $2 - The file we're looking for the last modification
+# $1 - The file we're looking for the last modification
 get_last_modification() {
-	local repo_root_dir="${1}"
-	local file="${2}"
+	local file="${1}"
 
 	# This is a workaround needed for when running this code on Jenkins
 	git config --global --add safe.directory ${repo_root_dir} &> /dev/null
diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh
index fef38b98a..f85ba4ec1 100755
--- a/tools/packaging/static-build/kernel/build.sh
+++ b/tools/packaging/static-build/kernel/build.sh
@@ -16,7 +16,7 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${KERNEL_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:kernel-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${KERNEL_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:kernel-$(get_last_modification ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || \
 	(sudo docker build -t "${container_image}" "${script_dir}" && \
diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh
index e18162861..2dfbe5a20 100755
--- a/tools/packaging/static-build/ovmf/build.sh
+++ b/tools/packaging/static-build/ovmf/build.sh
@@ -15,7 +15,7 @@ source "${script_dir}/../../scripts/lib.sh"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${OVMF_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:ovmf-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${OVMF_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:ovmf-$(get_last_modification ${script_dir})-$(uname -m)}"
 ovmf_build="${ovmf_build:-x86_64}"
 kata_version="${kata_version:-}"
 ovmf_repo="${ovmf_repo:-}"
diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh
index c27ded323..55ab71d35 100755
--- a/tools/packaging/static-build/qemu/build-base-qemu.sh
+++ b/tools/packaging/static-build/qemu/build-base-qemu.sh
@@ -38,7 +38,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d")
 [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
 [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
 
-container_image="${QEMU_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:qemu-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${QEMU_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:qemu-$(get_last_modification ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || (sudo "${container_engine}" build \
 	--build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \
diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh
index 2aac61274..35ebabcab 100755
--- a/tools/packaging/static-build/shim-v2/build.sh
+++ b/tools/packaging/static-build/shim-v2/build.sh
@@ -19,7 +19,7 @@ RUST_VERSION=${RUST_VERSION}
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
-container_image="${SHIM_V2_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${SHIM_V2_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || \
        	(sudo docker build  \
diff --git a/tools/packaging/static-build/td-shim/build.sh b/tools/packaging/static-build/td-shim/build.sh
index 50642fd30..58611b96c 100755
--- a/tools/packaging/static-build/td-shim/build.sh
+++ b/tools/packaging/static-build/td-shim/build.sh
@@ -29,7 +29,7 @@ package_output_dir="${package_output_dir:-}"
 [ -n "${tdshim_version}" ] || die "Failed to get TD-shim version or commit"
 [ -n "${tdshim_toolchain}" ] || die "Failed to get TD-shim toolchain to be used to build the project"
 
-container_image="${TDSHIM_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${TDSHIM_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:td-shim-${tdshim_toolchain}-$(get_last_modification ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || (sudo docker build \
 	--build-arg RUST_TOOLCHAIN="${tdshim_toolchain}" \
diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh
index 313786eb0..68b335821 100755
--- a/tools/packaging/static-build/virtiofsd/build.sh
+++ b/tools/packaging/static-build/virtiofsd/build.sh
@@ -48,7 +48,7 @@ case ${ARCH} in
 		;;
 esac
 
-container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${repo_root_dir} ${script_dir})-$(uname -m)}"
+container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${script_dir})-$(uname -m)}"
 
 sudo docker pull ${container_image} || \
 	(sudo docker build \