From 2bda92face263448402e4336d660e32e46a431f9 Mon Sep 17 00:00:00 2001 From: fupan Date: Wed, 17 May 2023 16:45:49 +0800 Subject: [PATCH 01/76] netlink: Fix the issue of update_interface When updating an interface, there's maybe an existed interface whose name would be the same with the updated required name, thus it would update failed with interface name existed error. Thus we should rename the existed interface with an temporary name and swap it with the previouse interface name last. Fixes: #6842 Signed-off-by: fupan --- src/agent/src/netlink.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index f5e9d271b..280b0d95a 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -89,6 +89,27 @@ impl Handle { .await?; } + // we need to update the link's interface name, thus we should rename the existed link whose name + // is the same with the link's request name, otherwise, it would update the link failed with the + // name conflicted. + let mut new_link = None; + if link.name() != iface.name { + if let Ok(link) = self.find_link(LinkFilter::Name(iface.name.as_str())).await { + // update the existing interface name with a temporary name, otherwise + // it would failed to udpate this interface with an existing name. + let mut request = self.handle.link().set(link.index()); + request.message_mut().header = link.header.clone(); + + request + .name(format!("{}_temp", link.name())) + .up() + .execute() + .await?; + + new_link = Some(link); + } + } + // Update link let mut request = self.handle.link().set(link.index()); request.message_mut().header = link.header.clone(); @@ -101,6 +122,14 @@ impl Handle { .execute() .await?; + // swap the updated iface's name. + if let Some(nlink) = new_link { + let mut request = self.handle.link().set(nlink.index()); + request.message_mut().header = nlink.header.clone(); + + request.name(link.name()).up().execute().await?; + } + Ok(()) } From 433b5add4ab04f78f506d6753bd6a16569a175c2 Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Tue, 23 May 2023 18:11:45 +0800 Subject: [PATCH 02/76] kubernetes: add agnhost command in pod yaml Fixes: #6928 Signed-off-by: Linda Yu --- .../runtimeclass_workloads/pod-empty-dir-fsgroup.yaml | 4 ++++ .../kubernetes/runtimeclass_workloads/pod-http-liveness.yaml | 2 ++ .../kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml | 2 ++ 3 files changed, 8 insertions(+) diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml index e887cc92c..83b0ee0f7 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml @@ -16,6 +16,8 @@ spec: containers: - name: mounttest-container image: ${agnhost_image} + command: + - /agnhost args: - mounttest - --fs_type=/test-volume @@ -27,6 +29,8 @@ spec: mountPath: /test-volume - name: mounttest-container-2 image: ${agnhost_image} + command: + - /agnhost args: - mounttest - --fs_type=/test-volume-2 diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml index 3d336761f..081b7941f 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml @@ -15,6 +15,8 @@ spec: containers: - name: liveness image: ${agnhost_image} + command: + - /agnhost args: - liveness livenessProbe: diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml index 6d5343cfe..4fbe3b6d6 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml @@ -15,6 +15,8 @@ spec: containers: - name: tcp-liveness image: ${agnhost_image} + command: + - /agnhost args: - liveness ports: From 0e47cfc4c779273384f7737de6f744b2bbf75a47 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Wed, 24 May 2023 11:27:04 -0400 Subject: [PATCH 03/76] runtime: sending SIGKILL to qemu There is a race condition when virtiofsd is killed without finishing all the clients. Because of that, when a pod is stopped, QEMU detects virtiofsd is gone, which is legitimate. Sending a SIGTERM first before killing could introduce some latency during the shutdown. Fixes #6757. Signed-off-by: Beraldo Leal --- src/runtime/virtcontainers/qemu.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 3a8c55cd3..d856490e5 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1122,22 +1122,21 @@ func (q *qemu) StopVM(ctx context.Context, waitOnly bool) (err error) { return err } + pids := q.GetPids() + if len(pids) == 0 { + return errors.New("cannot determine QEMU PID") + } + pid := pids[0] + if waitOnly { - pids := q.GetPids() - if len(pids) == 0 { - return errors.New("cannot determine QEMU PID") - } - - pid := pids[0] - err := utils.WaitLocalProcess(pid, qemuStopSandboxTimeoutSecs, syscall.Signal(0), q.Logger()) if err != nil { return err } } else { - err := q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) + err = syscall.Kill(pid, syscall.SIGKILL) if err != nil { - q.Logger().WithError(err).Error("Fail to execute qmp QUIT") + q.Logger().WithError(err).Error("Fail to send SIGKILL to qemu") return err } } From c04c872c4239eac282327fd052324120f1d8dd86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 25 May 2023 09:29:26 +0200 Subject: [PATCH 04/76] gha: aks: Increase the timeout time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We've seen tests being aborted close to the end of the run due to the timeout. Let's increase it, avoiding to hit such cases again.. Fixes: #6964 Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-aks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 4ed5a2d30..db09a4605 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -62,7 +62,7 @@ jobs: az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64 - name: Run tests - timeout-minutes: 35 + timeout-minutes: 60 run: | sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml From 557b840814fba6ee90f0131f740c6e70ac50c871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 25 May 2023 09:31:39 +0200 Subject: [PATCH 05/76] gha: aks: Wait longer to start running the tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're still facing issues related to the time taken to deploy the kata-deplot daemonset and starting to run the tests. Ideally, we should solve this with a readiness probe, and that's the approach we want to take in the future. However, for now, let's just make sure those tests are not on the way of the community. Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-aks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index db09a4605..aab1ebd78 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -76,7 +76,7 @@ jobs: # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, # which may cause issues like not having the node properly labeled or the artefacts # properly deployed when the tests actually start running. - sleep 150s + sleep 240s pushd tests/integration/kubernetes sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml From eee7aae71decca8212b4ac0e4875146dea79fc50 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Thu, 25 May 2023 20:00:25 +0800 Subject: [PATCH 06/76] runtime-rs/sandbox_bindmounts: add support for sandbox bindmounts sandbox_bind_mounts supports kinds of mount patterns, for example: (1) "/path/to", default readonly mode. (2) "/path/to:ro", same as (1). (3) "/path/to:rw", readwrite mode. Both support configuration and annotation: (1)[runtime] sandbox_bind_mounts=["/path/to", "/path/to:rw", "/mnt/to:ro"] (2) annotation will alse be supported, restricted as below: io.katacontainers.config.runtime.sandbox_bind_mounts = "/path/to /path/to:rw /mnt/to:ro" Fixes: #6597 Signed-off-by: alex.lyn --- src/libs/kata-types/src/annotations/mod.rs | 14 ++ src/libs/kata-types/src/config/runtime.rs | 19 ++- src/libs/kata-types/src/mount.rs | 43 +++++ .../config/configuration-dragonball.toml.in | 9 + .../crates/resource/src/manager_inner.rs | 30 +++- .../crates/resource/src/share_fs/mod.rs | 1 + .../src/share_fs/sandbox_bind_mounts.rs | 155 ++++++++++++++++++ .../crates/resource/src/share_fs/utils.rs | 16 +- .../src/share_fs/virtio_fs_share_mount.rs | 14 +- 9 files changed, 289 insertions(+), 12 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 3af0563c1..f094ddd70 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -316,6 +316,10 @@ pub const KATA_ANNO_CFG_VFIO_MODE: &str = "io.katacontainers.config.runtime.vfio pub const KATA_ANNO_CFG_HYPERVISOR_PREFETCH_FILES_LIST: &str = "io.katacontainers.config.hypervisor.prefetch_files.list"; +/// A sandbox annotation for sandbox level volume sharing with host. +pub const KATA_ANNO_CFG_SANDBOX_BIND_MOUNTS: &str = + "io.katacontainers.config.runtime.sandbox_bind_mounts"; + /// A helper structure to query configuration information by check annotations. #[derive(Debug, Default, Deserialize)] pub struct Annotation { @@ -950,6 +954,16 @@ impl Annotation { KATA_ANNO_CFG_VFIO_MODE => { config.runtime.vfio_mode = value.to_string(); } + KATA_ANNO_CFG_SANDBOX_BIND_MOUNTS => { + let args: Vec = value + .to_string() + .split_ascii_whitespace() + .map(str::to_string) + .collect(); + for arg in args { + config.runtime.sandbox_bind_mounts.push(arg.to_string()); + } + } _ => { warn!(sl!(), "Annotation {} not enabled", key); } diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 067ff6776..dddd3adc5 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -8,7 +8,8 @@ use std::path::Path; use super::default; use crate::config::{ConfigOps, TomlConfig}; -use crate::{eother, resolve_path, validate_path}; +use crate::mount::split_bind_mounts; +use crate::{eother, validate_path}; /// Type of runtime VirtContainer. pub const RUNTIME_NAME_VIRTCONTAINER: &str = "virt_container"; @@ -146,7 +147,14 @@ impl ConfigOps for Runtime { } for bind in conf.runtime.sandbox_bind_mounts.iter_mut() { - resolve_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?; + // Split the bind mount, canonicalize the path and then append rw mode to it. + let (real_path, mode) = split_bind_mounts(bind); + match Path::new(real_path).canonicalize() { + Err(e) => return Err(eother!("sandbox bind mount `{}` is invalid: {}", bind, e)), + Ok(path) => { + *bind = format!("{}{}", path.display(), mode); + } + } } Ok(()) @@ -176,7 +184,12 @@ impl ConfigOps for Runtime { } for bind in conf.runtime.sandbox_bind_mounts.iter() { - validate_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?; + // Just validate the real_path. + let (real_path, _mode) = split_bind_mounts(bind); + validate_path!( + real_path.to_owned(), + "sandbox bind mount `{}` is invalid: {}" + )?; } Ok(()) diff --git a/src/libs/kata-types/src/mount.rs b/src/libs/kata-types/src/mount.rs index f66e828bd..d77943848 100644 --- a/src/libs/kata-types/src/mount.rs +++ b/src/libs/kata-types/src/mount.rs @@ -25,6 +25,15 @@ pub const KATA_MOUNT_INFO_FILE_NAME: &str = "mountInfo.json"; /// KATA_DIRECT_VOLUME_ROOT_PATH is the root path used for concatenating with the direct-volume mount info file path pub const KATA_DIRECT_VOLUME_ROOT_PATH: &str = "/run/kata-containers/shared/direct-volumes"; +/// SANDBOX_BIND_MOUNTS_DIR is for sandbox bindmounts +pub const SANDBOX_BIND_MOUNTS_DIR: &str = "sandbox-mounts"; + +/// SANDBOX_BIND_MOUNTS_RO is for sandbox bindmounts with readonly +pub const SANDBOX_BIND_MOUNTS_RO: &str = ":ro"; + +/// SANDBOX_BIND_MOUNTS_RO is for sandbox bindmounts with readwrite +pub const SANDBOX_BIND_MOUNTS_RW: &str = ":rw"; + /// Information about a mount. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] pub struct Mount { @@ -128,6 +137,28 @@ impl NydusExtraOptions { serde_json::from_slice(&extra_options_buf).context("deserialize nydus's extraoption") } } + +/// sandbox bindmount format: /path/to/dir, or /path/to/dir:ro[:rw] +/// the real path is without suffix ":ro" or ":rw". +pub fn split_bind_mounts(bindmount: &str) -> (&str, &str) { + let (real_path, mode) = if bindmount.ends_with(SANDBOX_BIND_MOUNTS_RO) { + ( + bindmount.trim_end_matches(SANDBOX_BIND_MOUNTS_RO), + SANDBOX_BIND_MOUNTS_RO, + ) + } else if bindmount.ends_with(SANDBOX_BIND_MOUNTS_RW) { + ( + bindmount.trim_end_matches(SANDBOX_BIND_MOUNTS_RW), + SANDBOX_BIND_MOUNTS_RW, + ) + } else { + // default bindmount format + (bindmount, "") + }; + + (real_path, mode) +} + #[cfg(test)] mod tests { use super::*; @@ -137,6 +168,18 @@ mod tests { assert!(!is_kata_special_volume("kata:")); } + #[test] + fn test_split_bind_mounts() { + let test01 = "xxx0:ro"; + let test02 = "xxx2:rw"; + let test03 = "xxx3:is"; + let test04 = "xxx4"; + assert_eq!(split_bind_mounts(test01), ("xxx0", ":ro")); + assert_eq!(split_bind_mounts(test02), ("xxx2", ":rw")); + assert_eq!(split_bind_mounts(test03), ("xxx3:is", "")); + assert_eq!(split_bind_mounts(test04), ("xxx4", "")); + } + #[test] fn test_is_kata_guest_mount_volume() { assert!(is_kata_guest_mount_volume("kata:guest-mount:nfs")); diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index 4c7d3db05..e9c2b8c19 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -309,3 +309,12 @@ experimental=@DEFAULTEXPFEATURES@ # - When running single containers using a tool like ctr, container sizing information will be available. static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@ +# If specified, sandbox_bind_mounts identifieds host paths to be mounted(ro, rw) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +# Now it supports three kinds of bind mount format: +# - "/path/to", default readonly mode. +# - "/path/to:ro", readonly mode. +# - "/path/to:rw", readwrite mode. +sandbox_bind_mounts=@DEFBINDMOUNTS@ diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 7e50485ae..16f55a36a 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -25,7 +25,7 @@ use crate::{ manager::ManagerArgs, network::{self, Network}, rootfs::{RootFsResource, Rootfs}, - share_fs::{self, ShareFs}, + share_fs::{self, sandbox_bind_mounts::SandboxBindMounts, ShareFs}, volume::{Volume, VolumeResource}, ResourceConfig, }; @@ -97,6 +97,12 @@ impl ResourceManagerInner { .setup_device_before_start_vm(self.hypervisor.as_ref()) .await .context("setup share fs device before start vm")?; + + // setup sandbox bind mounts: setup = true + self.handle_sandbox_bindmounts(true) + .await + .context("failed setup sandbox bindmounts")?; + Some(share_fs) } else { None @@ -308,6 +314,22 @@ impl ResourceManagerInner { Ok(devices) } + async fn handle_sandbox_bindmounts(&self, setup: bool) -> Result<()> { + let bindmounts = self.toml_config.runtime.sandbox_bind_mounts.clone(); + if bindmounts.is_empty() { + info!(sl!(), "sandbox bindmounts empty, just skip it."); + return Ok(()); + } + + let sb_bindmnt = SandboxBindMounts::new(self.sid.clone(), bindmounts)?; + + if setup { + sb_bindmnt.setup_sandbox_bind_mounts() + } else { + sb_bindmnt.cleanup_sandbox_bind_mounts() + } + } + pub async fn update_cgroups( &self, cid: &str, @@ -324,6 +346,12 @@ impl ResourceManagerInner { .delete() .await .context("delete cgroup")?; + + // cleanup sandbox bind mounts: setup = false + self.handle_sandbox_bindmounts(false) + .await + .context("failed to cleanup sandbox bindmounts")?; + // clean up share fs mount if let Some(share_fs) = &self.share_fs { share_fs diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs index 12bb64420..4d70a6c7b 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs @@ -18,6 +18,7 @@ pub use utils::{ mod virtio_fs_share_mount; use virtio_fs_share_mount::VirtiofsShareMount; pub use virtio_fs_share_mount::EPHEMERAL_PATH; +pub mod sandbox_bind_mounts; use std::{collections::HashMap, fmt::Debug, path::PathBuf, sync::Arc}; diff --git a/src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs b/src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs new file mode 100644 index 000000000..13bd28103 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs @@ -0,0 +1,155 @@ +// Copyright (c) 2023 Alibaba Cloud +// Copyright (c) 2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +// Note: +// sandbox_bind_mounts supports kinds of mount patterns, for example: +// (1) "/path/to", with default readonly mode. +// (2) "/path/to:ro", same as (1). +// (3) "/path/to:rw", with readwrite mode. +// +// sandbox_bind_mounts: ["/path/to", "/path/to:rw", "/mnt/to:ro"] +// + +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context, Result}; + +use super::utils::{do_get_host_path, mkdir_with_permissions}; +use kata_sys_util::{fs::get_base_name, mount}; +use kata_types::mount::{SANDBOX_BIND_MOUNTS_DIR, SANDBOX_BIND_MOUNTS_RO, SANDBOX_BIND_MOUNTS_RW}; + +#[derive(Clone, Default, Debug)] +pub struct SandboxBindMounts { + sid: String, + host_mounts_path: PathBuf, + sandbox_bindmounts: Vec, +} + +impl SandboxBindMounts { + pub fn new(sid: String, sandbox_bindmounts: Vec) -> Result { + // /run/kata-containers/shared/sandboxes//rw/passthrough/sandbox-mounts + let bindmounts_path = + do_get_host_path(SANDBOX_BIND_MOUNTS_DIR, sid.as_str(), "", true, false); + let host_mounts_path = PathBuf::from(bindmounts_path); + + Ok(SandboxBindMounts { + sid, + host_mounts_path, + sandbox_bindmounts, + }) + } + + fn parse_sandbox_bind_mounts<'a>(&self, bindmnt_src: &'a str) -> Result<(&'a str, &'a str)> { + // get the bindmount's r/w mode + let bindmount_mode = if bindmnt_src.ends_with(SANDBOX_BIND_MOUNTS_RW) { + SANDBOX_BIND_MOUNTS_RW + } else { + SANDBOX_BIND_MOUNTS_RO + }; + + // get the true bindmount from the string + let bindmount = bindmnt_src.trim_end_matches(bindmount_mode); + + Ok((bindmount_mode, bindmount)) + } + + pub fn setup_sandbox_bind_mounts(&self) -> Result<()> { + let mut mounted_list: Vec = Vec::new(); + let mut mounted_map: HashMap = HashMap::new(); + for src in &self.sandbox_bindmounts { + let (bindmount_mode, bindmount) = self + .parse_sandbox_bind_mounts(src) + .context("parse sandbox bind mounts failed")?; + + // get the basename of the canonicalized mount path mnt_name: dirX + let mnt_name = get_base_name(bindmount)? + .into_string() + .map_err(|e| anyhow!("failed to get base name {:?}", e))?; + + // if repeated mounted, do umount it and return error + if mounted_map.insert(mnt_name.clone(), true).is_some() { + for p in &mounted_list { + nix::mount::umount(p) + .context("mounted_map insert one repeated mounted, do umount it")?; + } + + return Err(anyhow!( + "sandbox-bindmounts: path {} is already specified.", + bindmount + )); + } + + // mount_dest: /run/kata-containers/shared/sandboxes//rw/passthrough/sandbox-mounts/dirX + let mount_dest = self.host_mounts_path.clone().join(mnt_name.as_str()); + mkdir_with_permissions(self.host_mounts_path.clone().to_path_buf(), 0o750).context( + format!( + "create host mounts path {:?}", + self.host_mounts_path.clone() + ), + )?; + + info!( + sl!(), + "sandbox-bindmounts mount_src: {:?} => mount_dest: {:?}", bindmount, &mount_dest + ); + + // mount -o bind,ro host_shared mount_dest + // host_shared: ${bindmount} + mount::bind_mount_unchecked(Path::new(bindmount), &mount_dest, true).map_err(|e| { + for p in &mounted_list { + nix::mount::umount(p).unwrap_or_else(|x| { + format!("do umount failed: {:?}", x); + }); + } + e + })?; + + // default sandbox bind mounts mode is ro. + if bindmount_mode == SANDBOX_BIND_MOUNTS_RO { + info!(sl!(), "sandbox readonly bind mount."); + // dest_ro: /run/kata-containers/shared/sandboxes//ro/passthrough/sandbox-mounts + let mount_dest_ro = + do_get_host_path(SANDBOX_BIND_MOUNTS_DIR, &self.sid, "", true, true); + let sandbox_bindmounts_ro = [mount_dest_ro, mnt_name.clone()].join("/"); + + mount::bind_remount(sandbox_bindmounts_ro, true) + .context("remount ro directory with ro permission")?; + } + + mounted_list.push(mount_dest); + } + + Ok(()) + } + + pub fn cleanup_sandbox_bind_mounts(&self) -> Result<()> { + for src in &self.sandbox_bindmounts { + let parsed_mnts = self + .parse_sandbox_bind_mounts(src) + .context("parse sandbox bind mounts")?; + + let mnt_name = get_base_name(parsed_mnts.1)? + .into_string() + .map_err(|e| anyhow!("failed to convert to string{:?}", e))?; + + // /run/kata-containers/shared/sandboxes//passthrough/rw/sandbox-mounts/dir + let mnt_dest = self.host_mounts_path.join(mnt_name.as_str()); + mount::umount_timeout(mnt_dest, 0).context("umount bindmount failed")?; + } + + if fs::metadata(self.host_mounts_path.clone())?.is_dir() { + fs::remove_dir_all(self.host_mounts_path.clone()).context(format!( + "remove sandbox bindmount point {:?}.", + self.host_mounts_path.clone() + ))?; + } + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs index c93cbec54..3300c74ef 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -4,13 +4,27 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::path::{Path, PathBuf}; +use std::{ + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; use anyhow::Result; use kata_sys_util::mount; use super::*; +pub(crate) fn mkdir_with_permissions(path_target: PathBuf, mode: u32) -> Result<()> { + let new_path = &path_target; + std::fs::create_dir_all(new_path) + .context(format!("unable to create new path: {:?}", new_path))?; + + // mode format: 0o750, ... + std::fs::set_permissions(new_path, std::fs::Permissions::from_mode(mode))?; + + Ok(()) +} + pub(crate) fn ensure_dir_exist(path: &Path) -> Result<()> { if !path.exists() { std::fs::create_dir_all(path).context(format!("failed to create directory {:?}", path))?; diff --git a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs index 2cea9904d..6f875d29e 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs @@ -12,7 +12,6 @@ use kata_types::k8s::is_watchable_mount; use kata_types::mount; use nix::sys::stat::stat; use std::fs; -use std::os::unix::fs::PermissionsExt; use std::path::Path; const WATCHABLE_PATH_NAME: &str = "watchable"; @@ -21,7 +20,10 @@ pub const EPHEMERAL_PATH: &str = "/run/kata-containers/sandbox/ephemeral"; use super::{ get_host_rw_shared_path, - utils::{self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path}, + utils::{ + self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path, + mkdir_with_permissions, + }, ShareFsMount, ShareFsMountResult, ShareFsRootfsConfig, ShareFsVolumeConfig, KATA_GUEST_SHARE_DIR, PASSTHROUGH_FS_DIR, }; @@ -79,13 +81,11 @@ impl ShareFsMount for VirtiofsShareMount { .join(PASSTHROUGH_FS_DIR) .join(WATCHABLE_PATH_NAME); - fs::create_dir_all(&watchable_host_path).context(format!( - "unable to create watchable path: {:?}", - &watchable_host_path, + mkdir_with_permissions(watchable_host_path.clone(), 0o750).context(format!( + "unable to create watchable path {:?}", + watchable_host_path ))?; - fs::set_permissions(watchable_host_path, fs::Permissions::from_mode(0o750))?; - // path: /run/kata-containers/shared/containers/passthrough/watchable/config-map-name let file_name = Path::new(&guest_path) .file_name() From 4af4ced1aaa3c84951bbde717e256235596ca795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Thu, 13 Apr 2023 15:05:10 -0700 Subject: [PATCH 07/76] gha: Create Mariner host as part of k8s tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current testing setup only supports running Kata on top of an Ubuntu host. This adds Mariner to the matrix of testable hosts for k8s tests, with Cloud Hypervisor as a VMM. As preparation for the upcoming PR that will change only the actual test code (rather than workflow YAMLs), this also introduces a new file `setup.sh` that will be used to set host-specific parameters at test run-time. Fixes: #6961 Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 11 ++++++++-- .../kubernetes/run_kubernetes_tests.sh | 4 ++++ tests/integration/kubernetes/setup.sh | 20 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100755 tests/integration/kubernetes/setup.sh diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index aab1ebd78..2f22c70c3 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -17,10 +17,15 @@ jobs: strategy: fail-fast: false matrix: + host_os: + - ubuntu vmm: - clh - dragonball - qemu + include: + - host_os: cbl-mariner + vmm: clh runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -46,7 +51,8 @@ jobs: -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \ -s "Standard_D4s_v5" \ --node-count 1 \ - --generate-ssh-keys + --generate-ssh-keys \ + ${{ matrix.host_os == 'cbl-mariner' && '--os-sku mariner --workload-runtime KataMshvVmIsolation' }} - name: Install `bats` run: | @@ -79,10 +85,11 @@ jobs: sleep 240s pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml + bash setup.sh bash run_kubernetes_tests.sh popd env: + KATA_HOST_OS: ${{ matrix.host_os }} KATA_HYPERVISOR: ${{ matrix.vmm }} - name: Delete AKS cluster diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index db1e16633..0975ec0d5 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -54,6 +54,10 @@ else ) fi +if [ ${KATA_HOST_OS} == "cbl-mariner" ]; then + exit 0 +fi + # we may need to skip a few test cases when running on non-x86_64 arch arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml" if [ -f "${arch_config_file}" ]; then diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh new file mode 100755 index 000000000..0c3baf2dc --- /dev/null +++ b/tests/integration/kubernetes/setup.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +kubernetes_dir=$(dirname "$(readlink -f "$0")") + +set_runtime_class() { + sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml +} + +main() { + set_runtime_class +} + +main "$@" From 25d2fb0fdecf5d76fcb8a4ecde5c54c0d4c3e240 Mon Sep 17 00:00:00 2001 From: Fupan Li Date: Tue, 26 Jul 2022 15:50:35 +0800 Subject: [PATCH 08/76] agent: fix the issue of exec hang with a backgroud process When run a exec process in backgroud without tty, the exec will hang and didn't terminated. For example: crictl -i sh -c 'nohup tail -f /dev/null &' Fixes: #4747 Signed-off-by: Fupan Li --- src/agent/rustjail/src/process.rs | 2 +- src/agent/src/rpc.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/agent/rustjail/src/process.rs b/src/agent/rustjail/src/process.rs index 0e7fe73ef..cdecae130 100644 --- a/src/agent/rustjail/src/process.rs +++ b/src/agent/rustjail/src/process.rs @@ -161,7 +161,7 @@ impl Process { pub fn notify_term_close(&mut self) { let notify = self.term_exit_notifier.clone(); - notify.notify_one(); + notify.notify_waiters(); } pub fn close_stdin(&mut self) { diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 447768b6b..dede3204c 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -603,15 +603,16 @@ impl AgentService { let cid = req.container_id; let eid = req.exec_id; - let mut term_exit_notifier = Arc::new(tokio::sync::Notify::new()); + let term_exit_notifier; let reader = { let s = self.sandbox.clone(); let mut sandbox = s.lock().await; let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?; + term_exit_notifier = p.term_exit_notifier.clone(); + if p.term_master.is_some() { - term_exit_notifier = p.term_exit_notifier.clone(); p.get_reader(StreamType::TermMaster) } else if stdout { if p.parent_stdout.is_some() { From 5ddc4f94c55b3ec5678ac5d85b0c25460c9c53be Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Fri, 26 May 2023 11:18:29 +0800 Subject: [PATCH 09/76] runtime-rs/kata-ctl: Enhancement of DirectVolumeMount. Move the get_volume_mount_info to kata-types/src/mount.rs. If so, it becomes a common method of DirectVolumeMountInfo and reduces duplicated code. Fixes: #6701 Signed-off-by: alex.lyn --- src/agent/Cargo.lock | 8 +++++++ src/libs/Cargo.lock | 1 + src/libs/kata-types/Cargo.toml | 1 + src/libs/kata-types/src/mount.rs | 23 ++++++++++++++++++- src/runtime-rs/Cargo.lock | 1 + src/tools/kata-ctl/Cargo.lock | 20 +++++++++++++++++ src/tools/kata-ctl/src/ops/volume_ops.rs | 28 ++++-------------------- 7 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 3b0c3efc8..d5a0dcbe7 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -882,6 +882,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -1767,6 +1768,13 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] + [[package]] name = "scan_fmt" version = "0.2.6" diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 2f03109f8..6415f8d1e 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -519,6 +519,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 598cde620..f0a0f4471 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -27,6 +27,7 @@ thiserror = "1.0" toml = "0.5.8" oci = { path = "../oci" } +safe-path = { path = "../safe-path" } [dev-dependencies] tempfile = "3" diff --git a/src/libs/kata-types/src/mount.rs b/src/libs/kata-types/src/mount.rs index f66e828bd..00a2409b1 100644 --- a/src/libs/kata-types/src/mount.rs +++ b/src/libs/kata-types/src/mount.rs @@ -5,7 +5,7 @@ // use anyhow::{anyhow, Context, Result}; -use std::{collections::HashMap, path::PathBuf}; +use std::{collections::HashMap, fs, path::PathBuf}; /// Prefix to mark a volume as Kata special. pub const KATA_VOLUME_TYPE_PREFIX: &str = "kata:"; @@ -71,6 +71,27 @@ pub struct DirectVolumeMountInfo { pub options: Vec, } +/// join_path joins user provided volumepath with kata direct-volume root path +/// the volume_path is base64-encoded and then safely joined to the end of path prefix +pub fn join_path(prefix: &str, volume_path: &str) -> Result { + if volume_path.is_empty() { + return Err(anyhow!("volume path must not be empty")); + } + let b64_encoded_path = base64::encode(volume_path.as_bytes()); + + Ok(safe_path::scoped_join(prefix, b64_encoded_path)?) +} + +/// get DirectVolume mountInfo from mountinfo.json. +pub fn get_volume_mount_info(volume_path: &str) -> Result { + let mount_info_file_path = + join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?.join(KATA_MOUNT_INFO_FILE_NAME); + let mount_info_file = fs::read_to_string(mount_info_file_path)?; + let mount_info: DirectVolumeMountInfo = serde_json::from_str(&mount_info_file)?; + + Ok(mount_info) +} + /// Check whether a mount type is a marker for Kata specific volume. pub fn is_kata_special_volume(ty: &str) -> bool { ty.len() > KATA_VOLUME_TYPE_PREFIX.len() && ty.starts_with(KATA_VOLUME_TYPE_PREFIX) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 276facdbb..df550eccb 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -1481,6 +1481,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path 0.1.0", "serde", "serde_json", "slog", diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index ed90e1117..3056071be 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -764,7 +764,9 @@ dependencies = [ "lazy_static", "num_cpus", "oci", + "proc-mounts", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -1045,6 +1047,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "partition-identity" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa925f9becb532d758b0014b472c576869910929cf4c3f8054b386f19ab9e21" +dependencies = [ + "thiserror", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -1118,6 +1129,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-mounts" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d652f8435d0ab70bf4f3590a6a851d59604831a458086541b95238cc51ffcf2" +dependencies = [ + "partition-identity", +] + [[package]] name = "prost" version = "0.8.0" diff --git a/src/tools/kata-ctl/src/ops/volume_ops.rs b/src/tools/kata-ctl/src/ops/volume_ops.rs index 1027dcb72..0b037bf81 100644 --- a/src/tools/kata-ctl/src/ops/volume_ops.rs +++ b/src/tools/kata-ctl/src/ops/volume_ops.rs @@ -8,12 +8,12 @@ use crate::args::{DirectVolSubcommand, DirectVolumeCommand}; use anyhow::{anyhow, Ok, Result}; use futures::executor; use kata_types::mount::{ - DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, KATA_MOUNT_INFO_FILE_NAME, + get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, + KATA_MOUNT_INFO_FILE_NAME, }; use nix; use reqwest::StatusCode; -use safe_path; -use std::{fs, path::PathBuf, time::Duration}; +use std::{fs, time::Duration}; use url; use agent::ResizeVolumeRequest; @@ -90,17 +90,6 @@ async fn stats(volume_path: &str) -> Result> { Ok(Some(body)) } -// join_path joins user provided volumepath with kata direct-volume root path -// the volume_path is base64-encoded and then safely joined to the end of path prefix -fn join_path(prefix: &str, volume_path: &str) -> Result { - if volume_path.is_empty() { - return Err(anyhow!("volume path must not be empty")); - } - let b64_encoded_path = base64::encode(volume_path.as_bytes()); - - Ok(safe_path::scoped_join(prefix, b64_encoded_path)?) -} - // add writes the mount info (json string) of a direct volume into a filesystem path known to Kata Containers. pub fn add(volume_path: &str, mount_info: &str) -> Result> { let mount_info_dir_path = join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?; @@ -129,15 +118,6 @@ pub fn remove(volume_path: &str) -> Result> { Ok(None) } -pub fn get_volume_mount_info(volume_path: &str) -> Result { - let mount_info_file_path = - join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?.join(KATA_MOUNT_INFO_FILE_NAME); - let mount_info_file = fs::read_to_string(mount_info_file_path)?; - let mount_info: DirectVolumeMountInfo = serde_json::from_str(&mount_info_file)?; - - Ok(mount_info) -} - // get_sandbox_id_for_volume finds the id of the first sandbox found in the dir. // We expect a direct-assigned volume is associated with only a sandbox at a time. pub fn get_sandbox_id_for_volume(volume_path: &str) -> Result { @@ -170,7 +150,7 @@ mod tests { use super::*; use kata_types::mount::DirectVolumeMountInfo; use serial_test::serial; - use std::{collections::HashMap, fs}; + use std::{collections::HashMap, fs, path::PathBuf}; use tempfile::tempdir; use test_utils::skip_if_not_root; From af16d3fca4d770b5b81183d9dfdc43f1f467447d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Mon, 29 May 2023 12:12:46 -0700 Subject: [PATCH 10/76] gha: Unbreak CI and fix cluster creation step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the regression introduced by #6686 by properly injecting the `--os-sku mariner --workload-runtime KataMshvVmIsolation` flags. Error reference: https://github.com/kata-containers/kata-containers/actions/runs/5111460297/jobs/9188819103 Fixes: #6982 Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 2f22c70c3..71ec7eb39 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -52,7 +52,7 @@ jobs: -s "Standard_D4s_v5" \ --node-count 1 \ --generate-ssh-keys \ - ${{ matrix.host_os == 'cbl-mariner' && '--os-sku mariner --workload-runtime KataMshvVmIsolation' }} + $([ "${{ matrix.host_os == 'cbl-mariner' }}" = "true" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") - name: Install `bats` run: | From fd9b41464680aa677d5ef42711cf454c09687fb0 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 17 Apr 2023 16:17:04 +0800 Subject: [PATCH 11/76] dragonball: update comment for init_microvm Rewrite the comment of Vm::init_microvm method for aarch64. Fixes cargo test warnings on aarch64. Fixes: #6969 Signed-off-by: xuejun-xj --- src/dragonball/src/device_manager/mod.rs | 1 + src/dragonball/src/vm/aarch64.rs | 11 +++++------ src/dragonball/src/vm/mod.rs | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 766c5eef9..49e9666ac 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -1019,6 +1019,7 @@ mod tests { use vm_memory::{GuestAddress, MmapRegion}; use super::*; + #[cfg(target_arch = "x86_64")] use crate::vm::CpuTopology; impl DeviceManager { diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index edc753245..0ac4553ed 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -99,12 +99,11 @@ impl Vm { /// Initialize the virtual machine instance. /// /// It initialize the virtual machine instance by: - /// 1) initialize virtual machine global state and configuration. - /// 2) create system devices, such as interrupt controller. - /// 3) create and start IO devices, such as serial, console, block, net, vsock etc. - /// 4) create and initialize vCPUs. - /// 5) configure CPU power management features. - /// 6) load guest kernel image. + /// 1) Initialize virtual machine reset event fd. + /// 2) Create and initialize vCPUs. + /// 3) Create and initialize interrupt controller. + /// 4) Create and initialize vPMU device. + /// 5) Create and initialize devices, such as virtio, block, net, vsock, vfio etc. pub fn init_microvm( &mut self, epoll_mgr: EpollManager, diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index a9ccb02ea..852d78c2b 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -860,6 +860,7 @@ impl Vm { #[cfg(test)] pub mod tests { + #[cfg(target_arch = "x86_64")] use kvm_ioctls::VcpuExit; use linux_loader::cmdline::Cmdline; use test_utils::skip_if_not_root; From 64c764c1475fc9230b4836f5e02c81fc6e1410a7 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Fri, 26 May 2023 14:34:16 +0800 Subject: [PATCH 12/76] dragonball: update dbs-boot to v0.4.0 dbs-boot-v0.4.0 refectors the create_fdt interface. It simplifies the parameters needed to be passed and abstracts them into three structs. By the way, it also reserves some interfaces for future feature: numa passthrough and cache passthrough. Fixes: #6969 Signed-off-by: xuejun-xj --- src/dragonball/Cargo.lock | 4 +- src/dragonball/Cargo.toml | 2 +- src/dragonball/src/vm/aarch64.rs | 128 ++++++++++++++++--------------- 3 files changed, 71 insertions(+), 63 deletions(-) diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock index c5be35bfb..f2e087213 100644 --- a/src/dragonball/Cargo.lock +++ b/src/dragonball/Cargo.lock @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "dbs-boot" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108" +checksum = "5466a92f75aa928a9103dcb2088f6d1638ef9da8945fad7389a73864dfa0182c" dependencies = [ "dbs-arch", "kvm-bindings", diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 5036d7272..f70463266 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -15,7 +15,7 @@ bytes = "1.1.0" dbs-address-space = "0.2.0" dbs-allocator = "0.1.0" dbs-arch = "0.2.0" -dbs-boot = "0.3.0" +dbs-boot = "0.4.0" dbs-device = "0.2.0" dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] } dbs-legacy-devices = "0.1.0" diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 0ac4553ed..d6ff4c79d 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -6,65 +6,30 @@ // Use of this source code is governed by a BSD-style license that can be // found in the THIRD-PARTY file. -use std::collections::HashMap; -use std::fmt::Debug; use std::ops::Deref; +use std::sync::MutexGuard; use dbs_arch::gic::GICDevice; use dbs_arch::pmu::initialize_pmu; -use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel}; +use dbs_arch::{MMIODeviceInfo, VpmuFeatureLevel}; +use dbs_boot::fdt_utils::*; use dbs_boot::InitrdConfig; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; -use linux_loader::loader::Cmdline; -use vm_memory::{GuestAddressSpace, GuestMemory}; +use linux_loader::cmdline::{Cmdline, Error as CmdlineError}; +use vm_memory::GuestAddressSpace; use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, StartMicroVmError}; use crate::event_manager::EventManager; +use crate::vcpu::VcpuManager; -/// Configures the system and should be called once per vm before starting vcpu threads. -/// For aarch64, we only setup the FDT. -/// -/// # Arguments -/// -/// * `guest_mem` - The memory to be used by the guest. -/// * `cmdline` - The kernel commandline. -/// * `vcpu_mpidr` - Array of MPIDR register values per vcpu. -/// * `device_info` - A hashmap containing the attached devices for building FDT device nodes. -/// * `gic_device` - The GIC device. -/// * `initrd` - Information about an optional initrd. -#[allow(clippy::borrowed_box)] -fn configure_system( - guest_mem: &M, - cmdline: &str, - vcpu_mpidr: Vec, - device_info: Option<&HashMap<(DeviceType, String), T>>, - gic_device: &Box, - initrd: &Option, - vpmu_feature: &VpmuFeatureLevel, -) -> super::Result<()> { - dbs_boot::fdt::create_fdt( - guest_mem, - vcpu_mpidr, - cmdline, - device_info, - gic_device, - initrd, - vpmu_feature, - ) - .map_err(Error::BootSystem)?; - Ok(()) -} - -#[cfg(target_arch = "aarch64")] impl Vm { /// Gets a reference to the irqchip of the VM - #[allow(clippy::borrowed_box)] - pub fn get_irqchip(&self) -> &Box { - self.irqchip_handle.as_ref().unwrap() + pub fn get_irqchip(&self) -> &dyn GICDevice { + self.irqchip_handle.as_ref().unwrap().as_ref() } /// Creates the irq chip in-kernel device model. @@ -138,6 +103,50 @@ impl Vm { Ok(()) } + /// Generate fdt information about VM. + fn get_fdt_vm_info<'a>( + &'a self, + vm_memory: &'a GuestMemoryImpl, + cmdline: &'a str, + initrd_config: Option<&'a InitrdConfig>, + vcpu_manager: &'a MutexGuard, + ) -> FdtVmInfo { + let guest_memory = vm_memory.memory(); + let vcpu_mpidr = vcpu_manager + .vcpus() + .into_iter() + .map(|cpu| cpu.get_mpidr()) + .collect(); + let vcpu_boot_onlined = vec![]; + let vpmu_feature = vcpu_manager.vpmu_feature(); + // This configuration is used for passing cache information into guest. + // TODO: dragonball-sandbox #274; kata-containers #6969 + let cache_passthrough_enabled = false; + let fdt_vcpu_info = FdtVcpuInfo::new( + vcpu_mpidr, + vcpu_boot_onlined, + vpmu_feature, + cache_passthrough_enabled, + ); + + FdtVmInfo::new(guest_memory, cmdline, initrd_config, fdt_vcpu_info) + } + + // This method is used for passing cache/numa information into guest + // TODO: dragonball-sandbox #274,#275; kata-containers #6969 + /// Generate fdt information about cache/numa + fn get_fdt_numa_info(&self) -> FdtNumaInfo { + FdtNumaInfo::default() + } + + /// Generate fdt information about devices + fn get_fdt_device_info(&self) -> FdtDeviceInfo { + FdtDeviceInfo::new( + self.device_manager().get_mmio_device_info(), + self.get_irqchip(), + ) + } + /// Execute system architecture specific configurations. /// /// 1) set guest kernel boot parameters @@ -149,24 +158,23 @@ impl Vm { initrd: Option, ) -> std::result::Result<(), StartMicroVmError> { let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; - let vpmu_feature = vcpu_manager.vpmu_feature(); - let vcpu_mpidr = vcpu_manager - .vcpus() - .into_iter() - .map(|cpu| cpu.get_mpidr()) - .collect(); - let guest_memory = vm_memory.memory(); + let cmdline_cstring = cmdline + .as_cstring() + .map_err(StartMicroVmError::ProcessCommandlne)?; + let fdt_vm_info = self.get_fdt_vm_info( + vm_memory, + cmdline_cstring + .to_str() + .map_err(|_| StartMicroVmError::ProcessCommandlne(CmdlineError::InvalidAscii))?, + initrd.as_ref(), + &vcpu_manager, + ); + let fdt_numa_info = self.get_fdt_numa_info(); + let fdt_device_info = self.get_fdt_device_info(); - configure_system( - guest_memory, - cmdline.as_cstring().unwrap().to_str().unwrap(), - vcpu_mpidr, - self.device_manager.get_mmio_device_info(), - self.get_irqchip(), - &initrd, - &vpmu_feature, - ) - .map_err(StartMicroVmError::ConfigureSystem) + dbs_boot::fdt::create_fdt(fdt_vm_info, fdt_numa_info, fdt_device_info) + .map(|_| ()) + .map_err(|e| StartMicroVmError::ConfigureSystem(Error::BootSystem(e))) } pub(crate) fn register_events( From e31772cfea21efca6749ed46943d7be18b880d99 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Wed, 10 May 2023 09:45:13 +0800 Subject: [PATCH 13/76] dragonball: add support resize_vcpu on aarch64 This commit add support of resize_vcpu on aarch64. As kvm will check whether vgic is initialized when calling KVM_CREATE_VCPU ioctl, all the vcpu fds should be created before vm is booted. To support resizing vcpu scenario, we use max_vcpu_count for create_vcpus and setup_interrupt_controller interfaces. The SetVmConfiguration API will ensure max_vcpu_count >= boot_vcpu_count. Fixes: #6010 Signed-off-by: xuejun-xj --- src/dragonball/src/api/v1/vmm_action.rs | 6 --- src/dragonball/src/vcpu/vcpu_manager.rs | 50 ++++++++++++++++++++++--- src/dragonball/src/vm/aarch64.rs | 2 +- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index a271d04cb..6ad7bfcb7 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -626,12 +626,6 @@ impl VmmService { #[cfg(feature = "hotplug")] fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult { - if !cfg!(target_arch = "x86_64") { - // TODO: Arm need to support vcpu hotplug. issue: #6010 - warn!("This arch do not support vm resize!"); - return Ok(VmmData::Empty); - } - if !cfg!(feature = "dbs-upcall") { warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature."); return Ok(VmmData::Empty); diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 8f39af519..41200d010 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -374,11 +374,17 @@ impl VcpuManager { entry_addr: GuestAddress, ) -> Result<()> { info!("create boot vcpus"); - self.create_vcpus( - self.vcpu_config.boot_vcpu_count, - Some(request_ts), - Some(entry_addr), - )?; + let boot_vcpu_count = if cfg!(target_arch = "aarch64") { + // On aarch64, kvm doesn't allow to call KVM_CREATE_VCPU ioctl after vm has been booted + // because of vgic check. To support vcpu hotplug/hotunplug feature, we should create + // all the vcpufd at booting procedure. + // SetVmConfiguration API will ensure max_vcpu_count >= boot_vcpu_count, so it is safe + // to directly use max_vcpu_count here. + self.vcpu_config.max_vcpu_count + } else { + self.vcpu_config.boot_vcpu_count + }; + self.create_vcpus(boot_vcpu_count, Some(request_ts), Some(entry_addr))?; Ok(()) } @@ -1213,7 +1219,10 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); // test start boot vcpus assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); @@ -1267,8 +1276,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); + assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // invalid cpuid for pause let cpu_indexes = vec![2]; @@ -1304,9 +1319,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // invalid cpuid for exit let cpu_indexes = vec![2]; @@ -1330,9 +1350,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // exit all success assert!(vcpu_manager.exit_all_vcpus().is_ok()); @@ -1351,9 +1376,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // invalid cpuid for exit let cpu_indexes = vec![2]; @@ -1377,9 +1407,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // revalidate all success assert!(vcpu_manager.revalidate_all_vcpus_cache().is_ok()); @@ -1395,9 +1430,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // set vcpus in hotplug action let cpu_ids = vec![0]; diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index d6ff4c79d..2bd7a8a85 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -34,7 +34,7 @@ impl Vm { /// Creates the irq chip in-kernel device model. pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> { - let vcpu_count = self.vm_config.vcpu_count; + let vcpu_count = self.vm_config.max_vcpu_count; self.irqchip_handle = Some( dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) From 560442e6ed2985167c2308a6536f64bd02ccc5db Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Wed, 10 May 2023 10:17:06 +0800 Subject: [PATCH 14/76] dragonball: add vcpu_boot_onlined vector This commit implements the vcpu_boot_onlined vector in get_fdt_vm_info. "boot_enabled" means whether this vcpu should be onlined at first boot. It will be used by fdt, which write an attribute called boot_enabled, and will be handled by guest kernel to pass the correct cpu number to function "bringup_nonboot_cpus". Fixes: #6010 Signed-off-by: xuejun-xj --- src/dragonball/src/vm/aarch64.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 2bd7a8a85..fe8f23207 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -117,7 +117,9 @@ impl Vm { .into_iter() .map(|cpu| cpu.get_mpidr()) .collect(); - let vcpu_boot_onlined = vec![]; + let vm_config = self.vm_config(); + let mut vcpu_boot_onlined = vec![1; vm_config.vcpu_count as usize]; + vcpu_boot_onlined.resize(vm_config.max_vcpu_count as usize, 0); let vpmu_feature = vcpu_manager.vpmu_feature(); // This configuration is used for passing cache information into guest. // TODO: dragonball-sandbox #274; kata-containers #6969 From ffe3157a46cb3bb939b6d5156bb325825906d1d2 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Wed, 10 May 2023 18:54:45 +0800 Subject: [PATCH 15/76] dragonball: add arm64 patches for upcall The vcpu hotplug/hotunplug feature is implemented with upcall. This commit add three patches to support the feature on aarch64. Patches: > 0005: add support of upcall on aarch64 > 0006: skip activate offline cpus' MSI interrupt > 0007: set the correct boot cpu number Fixes: #6010 Signed-off-by: xuejun-xj --- tools/packaging/kernel/kata_config_version | 2 +- ...l-devmgr-suppots-cpu-hotplug-on-arm6.patch | 163 ++++++++++++++++++ ...msi-control-msi-irq-number-activated.patch | 67 +++++++ ...date-bringup_nonboot_cpus-parameters.patch | 139 +++++++++++++++ 4 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch create mode 100644 tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch create mode 100644 tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index f96ac0672..fe4afb0df 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -105 +106 diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch new file mode 100644 index 000000000..74dcc732e --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch @@ -0,0 +1,163 @@ +From 16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72 Mon Sep 17 00:00:00 2001 +Message-Id: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Wed, 10 May 2023 13:55:43 +0800 +Subject: [PATCH 1/3] upcall: dragonball-devmgr suppots cpu hotplug on arm64 + +Enable vcpuhotplug feature on aarch64 in guest kernel. It communicates +with dragonball by using upcall. This commit does these changes: + +1. Wraps x86 related fields with CONFIG_X86_64. +2. Add "cpu_event_notification" for arm64. +3. Add "add_cpu_dev" and "del_cpu_dev" for arm64. + +Signed-off-by: xuejun-xj +Reviewed-by : Chao Wu +Reviewed-by: Zizheng Bian +Reviewed-by: Baolin Wang +--- + .../upcall_srv/dragonball_device_manager.c | 84 ++++++++++++++++++- + 1 file changed, 81 insertions(+), 3 deletions(-) + +diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c +index 5a95b2ba63e8..088d38623b8d 100644 +--- a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c ++++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c +@@ -85,15 +85,21 @@ struct devmgr_req { + #if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) + struct { + uint8_t count; ++#ifdef CONFIG_X86_64 + uint8_t apic_ver; + uint8_t apic_ids[256]; ++#endif + } cpu_dev_info; + #endif + } msg_load; + }; + + struct cpu_dev_reply_info { ++#if defined(CONFIG_X86_64) + uint32_t apic_index; ++#elif defined(CONFIG_ARM64) ++ uint32_t cpu_id; ++#endif + }; + + struct devmgr_reply { +@@ -190,7 +196,8 @@ static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size, + msg->msg_flags = msg_flags; + } + +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) ++#if defined(CONFIG_X86_64) + static int get_cpu_id(int apic_id) + { + int i; +@@ -219,6 +226,24 @@ static void cpu_event_notification( + _fill_msg_header(&rep->msg_header, + sizeof(struct cpu_dev_reply_info), action_type, 0); + } ++#elif defined(CONFIG_ARM64) ++/** ++ * Return the first failed hotplug index of the cpu_id to dragonball. ++ * If hotplug/hotunplug succeeds, it will equals to the expected cpu count. ++ */ ++static void cpu_event_notification( ++ uint8_t cpu_id, ++ int ret, ++ uint32_t action_type, ++ struct devmgr_reply *rep) ++{ ++ pr_info("cpu event notification: cpu_id %d\n", cpu_id); ++ rep->msg_load.cpu_dev_info.cpu_id = cpu_id; ++ rep->ret = ret; ++ _fill_msg_header(&rep->msg_header, ++ sizeof(struct cpu_dev_reply_info), action_type, 0); ++} ++#endif + #endif + + #if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO) +@@ -262,7 +287,8 @@ static int del_mmio_dev(struct devmgr_req *req, + #endif + + +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) ++#if defined(CONFIG_X86_64) + static int add_cpu_upcall(int apic_id, uint8_t apic_ver) + { + int cpu_id, node_id; +@@ -430,6 +456,58 @@ static int del_cpu_dev(struct devmgr_req *req, + cpu_event_notification(i, ret, DEL_CPU, rep); + return ret; + } ++#elif defined(CONFIG_ARM64) ++static int add_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep) ++{ ++ int i, ret = 0; ++ unsigned int cpu_id, nr_online_cpus; ++ uint8_t count = req->msg_load.cpu_dev_info.count; ++ ++ nr_online_cpus = num_online_cpus(); ++ ++ pr_info("Current vcpu number: %d, Add vcpu number: %d\n", ++ nr_online_cpus, count); ++ ++ for (i = 0; i < count; ++i) { ++ cpu_id = nr_online_cpus + i; ++ ret = add_cpu(cpu_id); ++ if (ret != 0) ++ break; ++ } ++ ++ cpu_event_notification(nr_online_cpus + i, ret, ADD_CPU, rep); ++ return ret; ++} ++ ++static int del_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep) ++{ ++ int i, ret = 0; ++ unsigned int cpu_id, nr_online_cpus; ++ uint8_t count = req->msg_load.cpu_dev_info.count; ++ ++ nr_online_cpus = num_online_cpus(); ++ ++ pr_info("Current vcpu number: %d, Delete vcpu number: %d\n", ++ nr_online_cpus, count); ++ ++ if (count >= nr_online_cpus) { ++ pr_err("cpu del parameter check error: cannot remove all vcpus\n"); ++ ret = -EINVAL; ++ cpu_event_notification(0, ret, DEL_CPU, rep); ++ return ret; ++ } ++ ++ for (i = 0; i < count; ++i) { ++ cpu_id = nr_online_cpus - i - 1; ++ ret = remove_cpu(cpu_id); ++ if (ret != 0) ++ break; ++ } ++ ++ cpu_event_notification(nr_online_cpus - i, ret, DEL_CPU, rep); ++ return ret; ++} ++#endif + #endif + + static struct { +@@ -440,7 +518,7 @@ static struct { + {ADD_MMIO, add_mmio_dev}, + {DEL_MMIO, del_mmio_dev}, + #endif +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) + {ADD_CPU, add_cpu_dev}, + {DEL_CPU, del_cpu_dev}, + #endif +-- +2.28.0 + diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch new file mode 100644 index 000000000..b40cf6666 --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch @@ -0,0 +1,67 @@ +From 6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a Mon Sep 17 00:00:00 2001 +Message-Id: <6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a.1685428663.git.jiyunxue@linux.alibaba.com> +In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Wed, 10 May 2023 14:51:40 +0800 +Subject: [PATCH 2/3] msi: control msi irq number activated + +When passthroughing pci device, kernel will initialize and activate +(max_cpu_count+1) msi irq. However, in vcpu hotplugging situation, +because of vgic, max_cpu_count may be greater than online_cpu_count. +Those offline cpus will also be activated by kernel, which cause failure +of passthroughing pci device. + +To solve this problem, this patch add a function +"check_affinity_mask_online" to check if msi_desc->affinity contains +online cpus. If current cpu is offline, it will continue the for loop to +skip activating related irq. + +Signed-off-by: xuejun-xj +Reviewed-by: Shuo Tan +Reviewed-by: Baolin Wang +--- + kernel/irq/msi.c | 20 ++++++++++++++++++++ + 1 file changed, 20 insertions(+) + +diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c +index d924676c8781..d60a3fc654e6 100644 +--- a/kernel/irq/msi.c ++++ b/kernel/irq/msi.c +@@ -395,6 +395,23 @@ static bool msi_check_reservation_mode(struct irq_domain *domain, + return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit; + } + ++/* This function is used for check whether the cpu affinity belongs to the ++ * online cpus. When we passthrough the nvme devices, the kernel will allocate ++ * maxcpus+1 MSI irqs and then activate them. In vcpu hotplug situations, it ++ * may happen that kernel activates the offline cpus when bootcpus < maxcpus. ++ * To avoid this conflict, this function check the affinities. ++ */ ++static inline bool check_affinity_mask_online(struct irq_affinity_desc *affinity) ++{ ++ int cpu; ++ ++ for_each_cpu(cpu, &affinity->mask) ++ if (cpu_online(cpu)) ++ return true; ++ ++ return false; ++} ++ + int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + int nvec) + { +@@ -445,6 +462,9 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + goto skip_activate; + + for_each_msi_vector(desc, i, dev) { ++ if (desc->affinity ++ && !check_affinity_mask_online(desc->affinity)) ++ continue; + if (desc->irq == i) { + virq = desc->irq; + dev_dbg(dev, "irq [%d-%d] for MSI\n", +-- +2.28.0 + diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch new file mode 100644 index 000000000..71d4fb977 --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch @@ -0,0 +1,139 @@ +From a05086142be13d43c7fc92500bcb870a2f37e485 Mon Sep 17 00:00:00 2001 +Message-Id: +In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Tue, 23 May 2023 09:43:02 +0800 +Subject: [PATCH 3/3] smp: update bringup_nonboot_cpus parameters + +On aarch64, kvm doesn't allow vmm to call KVM_CREATE_VCPU ioctls after +vm has already started, which is caused by vgic_initialized check in +kvm_arch_vcpu_precreate() function. Therefore, to support vcpu hotplug +feature on aarch64, all the vcpus should be created and configured ready +for start at booting procedure. + +To solve the problem, dragonball will add a property in each cpu node, +called "boot-onlined". This property indicates whether this cpu should +be onlined at first boot. It has two values: 0 and 1. 0 means offline, +while 1 means online. + +This commit also add a helper function called "of_get_cpu_boot_onlined", +which parse the cpu node and get the value of boot-onlined property. +Then update the global variable "boot_onlined_cpu". + +When kernel calling smp_init(), bringup_nonboot_cpus will start all the +other cpus except cpu0. The activated cpu number equals setup_max_cpus. +In vcpu hotplug scenario, vmm will create all the vcpufd before vm is +initialized, while activating only a few vcpus at first boot. The +setup_max_cpus variable will be initialized as all vcpu count. This +cause that the other cpus cannot find enough cpu threads, and they will +wait for 5 seconds each cpu. + +Therefore, we use boot_onlined_cpu instead of setup_max_cpus to give +"bringup_nonboot_cpus" correct cpu number it needs. + +Signed-off-by: xuejun-xj +--- + .../devicetree/bindings/arm/cpus.yaml | 11 +++++++++ + arch/arm64/kernel/smp.c | 24 +++++++++++++++++++ + kernel/smp.c | 10 +++++++- + 3 files changed, 44 insertions(+), 1 deletion(-) + +diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml +index 14cd727d3c4b..691bb352d842 100644 +--- a/Documentation/devicetree/bindings/arm/cpus.yaml ++++ b/Documentation/devicetree/bindings/arm/cpus.yaml +@@ -316,6 +316,17 @@ properties: + formed by encoding the target CPU id into the low bits of the + physical start address it should jump to. + ++ boot-onlined: ++ $ref: '/schemas/types.yaml#/definitions/uint32' ++ description: | ++ The boot-onlined property is an optional u32 value that indicates ++ whether the cpu device should be activated at first boot. This is ++ useful in vcpu hotplug scenario to pass correct value of activated ++ cpu number. ++ ++ This property has two values: 0 and 1. 1 means the cpu should be ++ activated while 0 means it shouldn't. ++ + if: + # If the enable-method property contains one of those values + properties: +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 18e9727d3f64..5db8041929a6 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -464,6 +464,27 @@ void __init smp_prepare_boot_cpu(void) + init_gic_priority_masking(); + } + ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++extern unsigned int boot_onlined_cpu; ++static void __init of_get_cpu_boot_onlined(struct device_node *dn) ++{ ++ unsigned int boot_onlined; ++ int r; ++ ++ r = of_property_read_u32(dn, "boot-onlined", &boot_onlined); ++ if (r) { ++ pr_err("%pOF: missing boot-onlined property\n", dn); ++ return; ++ } ++ /* ++ * Property boot-onlined has two values: 0 and 1. ++ * 0 means offline, and 1 means online. ++ * Here just count the number of boot_onlined_cpu. ++ */ ++ boot_onlined_cpu += boot_onlined; ++} ++#endif ++ + static u64 __init of_get_cpu_mpidr(struct device_node *dn) + { + const __be32 *cell; +@@ -654,6 +675,9 @@ static void __init of_parse_and_init_cpus(void) + struct device_node *dn; + + for_each_of_cpu_node(dn) { ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++ of_get_cpu_boot_onlined(dn); ++#endif + u64 hwid = of_get_cpu_mpidr(dn); + + if (hwid == INVALID_HWID) +diff --git a/kernel/smp.c b/kernel/smp.c +index 25240fb2df94..567615b9a008 100644 +--- a/kernel/smp.c ++++ b/kernel/smp.c +@@ -801,17 +801,25 @@ void __init setup_nr_cpu_ids(void) + nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; + } + ++/* Setup number of CPUs to activate */ ++unsigned int boot_onlined_cpu = 0; ++ + /* Called by boot processor to activate the rest. */ + void __init smp_init(void) + { + int num_nodes, num_cpus; ++ int num_onlined_cpu = setup_max_cpus; + + idle_threads_init(); + cpuhp_threads_init(); + + pr_info("Bringing up secondary CPUs ...\n"); + +- bringup_nonboot_cpus(setup_max_cpus); ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++ if (boot_onlined_cpu != 0) ++ num_onlined_cpu = boot_onlined_cpu; ++#endif ++ bringup_nonboot_cpus(num_onlined_cpu); + + num_nodes = num_online_nodes(); + num_cpus = num_online_cpus(); +-- +2.28.0 + From 43e73bdef7a6f05dc589f4abc16e52cf505644d8 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 30 May 2023 13:24:36 +0200 Subject: [PATCH 16/76] packaging: make BUILDER_REGISTRY configurable This PR is to make an environment variable `BUILDER_REGISTRY` configurable so that those who want to use their own registry for build can set up the registry. Fixes: #6988 Signed-off-by: Hyounggyu Choi --- .../kata-deploy/local-build/kata-deploy-binaries-in-docker.sh | 1 + tools/packaging/scripts/lib.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 9f4f700e9..6464c8722 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -55,6 +55,7 @@ docker run \ -v "${kata_dir}:${kata_dir}" \ --env CI="${CI:-}" \ --env USER=${USER} \ + --env BUILDER_REGISTRY="${BUILDER_REGISTRY:-}" \ --env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" \ --env INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER:-}" \ --env KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER:-}" \ diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index b6874bf64..1f0173e47 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -8,7 +8,7 @@ export GOPATH=${GOPATH:-${HOME}/go} export tests_repo="${tests_repo:-github.com/kata-containers/tests}" export tests_repo_dir="$GOPATH/src/$tests_repo" -export BUILDER_REGISTRY="quay.io/kata-containers/builders" +export BUILDER_REGISTRY="${BUILDER_REGISTRY:-quay.io/kata-containers/builders}" export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From 03027a739952da9576028919368dc7ca0c228802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 30 May 2023 13:26:49 -0700 Subject: [PATCH 17/76] gha: Fix Mariner cluster creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the Mariner Kata host is in preview, we need the `aks-preview` extension to enable the `--workload-runtime KataMshvVmIsolation` flag. Fixes: #6994 Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 71ec7eb39..efe90a37e 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -35,6 +35,8 @@ jobs: - name: Download Azure CLI run: | curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # The aks-preview extension is required while the Mariner Kata host is in preview. + az extension add --name aks-preview - name: Log into the Azure account run: | From 9228815ad2023fca00efebf0cc26f9ffc7a0343a Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 30 May 2023 13:44:58 -0700 Subject: [PATCH 18/76] kernel: Modify build-kernel.sh to accomodate for changes in version.yaml There were recent changes for the tdx kernel in the version.yaml that are not currently accounted for in the build-kernel.sh script. Attempts to setup a tdx kernel to build local changes seemed to not download the tdx kernel. Instead the mainline kernel is downloaded which has no tdx-related changes. The version.yaml has a new entry for tdx kernel. Use that instead for setting up and downloading the tdx kernel. Fixes: #6984 Signed-off-by: Archana Shinde --- tools/packaging/kernel/build-kernel.sh | 16 +++++++++++++--- tools/packaging/kernel/kata_config_version | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 88fb61c18..17ee523e6 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -129,7 +129,13 @@ get_tee_kernel() { mkdir -p ${kernel_path} - [ -z "${kernel_url}" ] && kernel_url=$(get_from_kata_deps "assets.kernel.${tee}.url") + if [ -z "${kernel_url}" ]; then + if [[ "${conf_guest}" == "tdx" ]]; then + kernel_url=$(get_from_kata_deps "assets.kernel-tdx-experimental.url") + else + kernel_url=$(get_from_kata_deps "assets.kernel.${tee}.url") + fi + fi local kernel_tarball="${version}.tar.gz" @@ -611,8 +617,12 @@ main() { kernel_version=$(get_from_kata_deps "assets.kernel-dragonball-experimental.version") elif [[ "${conf_guest}" != "" ]]; then #If specifying a tag for kernel_version, must be formatted version-like to avoid unintended parsing issues - kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) - [ -n "${kernel_version}" ] || kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.tag") + if [[ "${conf_guest}" == "tdx" ]]; then + kernel_version=$(get_from_kata_deps "assets.kernel-tdx-experimental.version" 2>/dev/null || true) + else + kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) + [ -n "${kernel_version}" ] || kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.tag") + fi else kernel_version=$(get_from_kata_deps "assets.kernel.version") fi diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index fe4afb0df..e34885bbc 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -106 +107 From 4b89a6bdac929797270470f31fe62822dcc04fc2 Mon Sep 17 00:00:00 2001 From: SinghWang Date: Wed, 31 May 2023 10:24:45 +0800 Subject: [PATCH 19/76] release: Standardize kata static file name The string representing the architecture aarch64 and x86_64 need to be changed to arm64 and amd64 for the release. Fixes: #6986 Signed-off-by: SinghWang --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index af9b93132..c553ca514 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -83,7 +83,7 @@ jobs: - name: push amd64 static tarball to github run: | tag=$(echo $GITHUB_REF | cut -d/ -f3-) - tarball="kata-static-$tag-x86_64.tar.xz" + tarball="kata-static-$tag-amd64.tar.xz" mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}" pushd $GITHUB_WORKSPACE echo "uploading asset '${tarball}' for tag: ${tag}" @@ -97,7 +97,7 @@ jobs: - name: push arm64 static tarball to github run: | tag=$(echo $GITHUB_REF | cut -d/ -f3-) - tarball="kata-static-$tag-aarch64.tar.xz" + tarball="kata-static-$tag-arm64.tar.xz" mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}" pushd $GITHUB_WORKSPACE echo "uploading asset '${tarball}' for tag: ${tag}" From 0c8282c224f603efc929c2acd852b5470df5a4f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 31 May 2023 05:20:04 +0200 Subject: [PATCH 20/76] gha: aks: Add the host_os as part of the aks cluster's name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to do so, otherwise we'll create two clusters for testing Cloud Hypervisor with exactly the same name, one using Ubuntu, and one using Mariner. Fixes: #6999 Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-aks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index efe90a37e..b939db7be 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -50,7 +50,7 @@ jobs: run: | az aks create \ -g "kataCI" \ - -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \ + -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" \ -s "Standard_D4s_v5" \ --node-count 1 \ --generate-ssh-keys \ From aebd3b47d91257ec6a8e3bc6d2187ad6fda5fa2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 31 May 2023 20:30:17 +0200 Subject: [PATCH 21/76] gha: aks: Ensure host_os is used everywhere needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We added that to create the cluster name, but I forgot to add that to the part we get the k8s config file, or to the part where we delete the AKS cluster. Fixes: #6999 Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-aks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index b939db7be..7ca0842c5 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -67,7 +67,7 @@ jobs: - name: Download credentials for the Kubernetes CLI to use them run: | - az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64 + az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64 - name: Run tests timeout-minutes: 60 @@ -99,6 +99,6 @@ jobs: run: | az aks delete \ -g "kataCI" \ - -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \ + -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" \ --yes \ --no-wait From 26f7520387ea2781ddc8ddc0dd415d20a0a18777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 1 Jun 2023 11:34:44 +0200 Subject: [PATCH 22/76] kata-deploy: Change how we get the Ubuntu k8s key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current method has been failing every now and then, and was reported on https://github.com/kubernetes/release/issues/2862. Ding poked me and suggested to do this change here, so here we go. :-) Fixes: #7006 Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index bb579631b..f8b9edf4c 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -16,9 +16,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN \ apt-get update && \ -apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \ +apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl gpg xz-utils systemd && \ mkdir -p /etc/apt/keyrings/ && \ -curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && \ +curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ apt-get update && \ apt-get install -y --no-install-recommends kubectl && \ From 38868411312e73acc319dd91dfdd0279c6cb2c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 1 Jun 2023 12:17:24 +0200 Subject: [PATCH 23/76] readme: Add status badge for the "Publish Artefacts" job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's start adding the status of our jobs as part of our main page, so folks monitoring those can easily check whether they're okay, or if someone has to be pinged about those. Fixes: #7008 Signed-off-by: Fabiano Fidêncio --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 662899ea6..528e52b6c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ +[![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) + # Kata Containers Welcome to Kata Containers! From 3c1f6d36dca56d80d858bc56226b9d93f68b15dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 1 Jun 2023 12:18:52 +0200 Subject: [PATCH 24/76] readme: Update Kata Containers logo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's use the horizontal logo, as it occupies better the space the we have. The logo comes from: https://openinfra.dev/brand/logos Signed-off-by: Fabiano Fidêncio --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 528e52b6c..05933bb58 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + [![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) From 1c6d22c8034e20595482af3098ab55efb144b506 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Thu, 1 Jun 2023 13:37:17 +0200 Subject: [PATCH 25/76] gha: aks: Use short SHA in cluster name Full SHA is 40 characters, while AKS cluster name has a limit of 63. Trim the SHA to 12 characters, which is widely considered to be unique enough and is short enough to be used in the cluster name Fixes: #7010 Signed-off-by: Jeremi Piotrowski --- .github/workflows/run-k8s-tests-on-aks.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 7ca0842c5..7972f09e7 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -46,11 +46,16 @@ jobs: -p "${{ secrets.AZ_PASSWORD }}" \ --tenant "${{ secrets.AZ_TENANT_ID }}" + - name: Format cluster name + run: | + rev=$(git rev-parse --short=12 HEAD) + echo "cluster_name=${{ github.event.pull_request.number }}-$rev-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" >> $GITHUB_ENV + - name: Create AKS cluster run: | az aks create \ -g "kataCI" \ - -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" \ + -n "${{ env.cluster_name }}" \ -s "Standard_D4s_v5" \ --node-count 1 \ --generate-ssh-keys \ @@ -67,7 +72,7 @@ jobs: - name: Download credentials for the Kubernetes CLI to use them run: | - az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64 + az aks get-credentials -g "kataCI" -n "${{ env.cluster_name }}" - name: Run tests timeout-minutes: 60 @@ -99,6 +104,6 @@ jobs: run: | az aks delete \ -g "kataCI" \ - -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" \ + -n "${{ env.cluster_name }}" \ --yes \ --no-wait From 5f6fc3ed76fb6f156d01bf2ec61250d9240fdfb7 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Thu, 1 Jun 2023 20:25:35 +0800 Subject: [PATCH 26/76] runtime-rs: bugfix: update Cargo.lock When dragonball update dbs-boot crate in commit 64c764c1475fc9230b4836f5e02c81fc6e1410a7, the Cargo.lock in runtime-rs should also be updated. Fixes: #6969 Signed-off-by: xuejun-xj --- src/runtime-rs/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index df550eccb..7f974d744 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -682,9 +682,9 @@ dependencies = [ [[package]] name = "dbs-boot" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108" +checksum = "5466a92f75aa928a9103dcb2088f6d1638ef9da8945fad7389a73864dfa0182c" dependencies = [ "dbs-arch", "kvm-bindings", From ed37715e05844a3d2d0eefc9aad5b6a03f18e748 Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Mon, 29 May 2023 09:45:20 +0000 Subject: [PATCH 27/76] runtime-rs: handle copy files when share_fs is not available In hypervisors that do not support virtiofs we have to copy files in the VM sandbox to properly setup the network (resolv.conf, hosts, and hostname). To do that, we construct the volume as before, with the addition of an extra variable that designates the path where the file will reside in the sandbox. In this case, we issue a `copy_file` agent request *and* we patch the spec to account for this change. Fixes: #6978 Signed-off-by: Anastassios Nanos Signed-off-by: George Pyrros --- .../crates/resource/src/manager_inner.rs | 2 + .../crates/resource/src/volume/mod.rs | 7 +- .../resource/src/volume/share_fs_volume.rs | 70 ++++++++++++++++++- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 16f55a36a..18e74dacf 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -10,6 +10,7 @@ use crate::{network::NetworkConfig, resource_persist::ResourceState}; use agent::{types::Device, Agent, Storage}; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; + use hypervisor::{ device::{device_manager::DeviceManager, DeviceConfig}, BlockConfig, Hypervisor, @@ -255,6 +256,7 @@ impl ResourceManagerInner { spec, self.device_manager.as_ref(), &self.sid, + self.agent.clone(), ) .await } diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index ea7d4f5d6..52b826b2d 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -11,13 +11,13 @@ mod share_fs_volume; mod shm_volume; use async_trait::async_trait; +use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; +use agent::Agent; use anyhow::{Context, Result}; use hypervisor::device::device_manager::DeviceManager; use std::{sync::Arc, vec::Vec}; use tokio::sync::RwLock; -use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; - use self::hugepage::{get_huge_page_limits_map, get_huge_page_option}; const BIND: &str = "bind"; @@ -52,6 +52,7 @@ impl VolumeResource { spec: &oci::Spec, d: &RwLock, sid: &str, + agent: Arc, ) -> Result>> { let mut volumes: Vec> = vec![]; let oci_mounts = &spec.mounts; @@ -85,7 +86,7 @@ impl VolumeResource { ) } else if share_fs_volume::is_share_fs_volume(m) { Arc::new( - share_fs_volume::ShareFsVolume::new(share_fs, m, cid, read_only) + share_fs_volume::ShareFsVolume::new(share_fs, m, cid, read_only, agent.clone()) .await .with_context(|| format!("new share fs volume {:?}", m))?, ) diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs index 0748e374d..098dc399f 100644 --- a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs @@ -5,11 +5,15 @@ // use std::{ + fs::File, + io::Read, + os::unix::fs::MetadataExt, path::{Path, PathBuf}, str::FromStr, sync::Arc, }; +use agent::Agent; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use hypervisor::device::device_manager::DeviceManager; @@ -19,6 +23,9 @@ use super::Volume; use crate::share_fs::{MountedInfo, ShareFs, ShareFsVolumeConfig}; use kata_types::mount; +use crate::share_fs::DEFAULT_KATA_GUEST_SANDBOX_DIR; +use crate::share_fs::PASSTHROUGH_FS_DIR; + const SYS_MOUNT_PREFIX: [&str; 2] = ["/proc", "/sys"]; // copy file to container's rootfs if filesystem sharing is not supported, otherwise @@ -39,6 +46,7 @@ impl ShareFsVolume { m: &oci::Mount, cid: &str, readonly: bool, + agent: Arc, ) -> Result { // The file_name is in the format of "sandbox-{uuid}-{file_name}" let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); @@ -61,11 +69,67 @@ impl ShareFsVolume { Ok(src) => src, }; + // If the mount source is a file, we can copy it to the sandbox if src.is_file() { - // TODO: copy file - debug!(sl!(), "FIXME: copy file {}", &m.source); - } else { + // This is where we set the value for the guest path + let dest = [ + DEFAULT_KATA_GUEST_SANDBOX_DIR, + PASSTHROUGH_FS_DIR, + file_name.clone().as_str(), + ] + .join("/"); + debug!( + sl!(), + "copy local file {:?} to guest {:?}", + &m.source, + dest.clone() + ); + + // Read file metadata + let file_metadata = std::fs::metadata(src.clone()) + .with_context(|| format!("Failed to read metadata from file: {:?}", src))?; + + // Open file + let mut file = File::open(&src) + .with_context(|| format!("Failed to open file: {:?}", src))?; + + // Open read file contents to buffer + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer) + .with_context(|| format!("Failed to read file: {:?}", src))?; + + // Create gRPC request + let r = agent::CopyFileRequest { + path: dest.clone(), + file_size: file_metadata.len() as i64, + uid: file_metadata.uid() as i32, + gid: file_metadata.gid() as i32, + file_mode: file_metadata.mode(), + data: buffer, + ..Default::default() + }; + + debug!(sl!(), "copy_file: {:?} to sandbox {:?}", &src, dest.clone()); + + // Issue gRPC request to agent + agent.copy_file(r).await.with_context(|| { + format!( + "copy file request failed: src: {:?}, dest: {:?}", + file_name, dest + ) + })?; + + // append oci::Mount structure to volume mounts + volume.mounts.push(oci::Mount { + destination: m.destination.clone(), + r#type: "bind".to_string(), + source: dest.clone(), + options: m.options.clone(), + }) + } else { + // If not, we can ignore it. Let's issue a warning so that the user knows. + warn!( sl!(), "Ignoring non-regular file as FS sharing not supported. mount: {:?}", m ); From e4eb664d27f6cf7169c5d10f4383bdd8f3b892f6 Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Wed, 31 May 2023 15:40:45 +0000 Subject: [PATCH 28/76] runtime-rs: update rust to 1.69.0 We are probably hitting this: https://github.com/rust-lang/rust/issues/63033 Seems like it is worth a try to upgrade to 1.69.0 Signed-off-by: Anastassios Nanos --- versions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yaml b/versions.yaml index 38a6fa397..18e51fe83 100644 --- a/versions.yaml +++ b/versions.yaml @@ -324,12 +324,12 @@ languages: rust: description: "Rust language" notes: "'version' is the default minimum version used by this project." - version: "1.68.0" + version: "1.69.0" meta: description: | 'newest-version' is the latest version known to work when building Kata - newest-version: "1.68.0" + newest-version: "1.69.0" golangci-lint: description: "golangci-lint" From aab60309620097aba19ac2c8b70f28d5dee774f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Fri, 26 May 2023 12:46:03 -0700 Subject: [PATCH 29/76] gha: aks: Extract `run` commands to a script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Github Actions reads and runs workflow files from the main branch, rather than from the PR branch. This means that PRs that modify workflow files aren't being tested with the updated workflows coming from the PR, but rather with the old workflows from the main branch. AFAIK, this behavior isn't avoidable for workflow files (but is for other scripts). This makes it very hard to reliably test workflow changes before they're actually merged into main and leads to issues that we have to hotifx (see #6983, #6995). This PR aims to mitigate that by extracting the commands used in workflows to a separate script file. The way our CI is set up, those script files are read from the PR branch and thus changes would be reflected in the CI checks. Fixes: #6971 Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 77 +++-------- .github/workflows/run-k8s-tests-on-sev.yaml | 40 +----- .github/workflows/run-k8s-tests-on-snp.yaml | 40 +----- .github/workflows/run-k8s-tests-on-tdx.yaml | 40 +----- tests/integration/gha-run.sh | 144 ++++++++++++++++++++ 5 files changed, 181 insertions(+), 160 deletions(-) create mode 100755 tests/integration/gha-run.sh diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 7972f09e7..e9f7b55b1 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -27,83 +27,44 @@ jobs: - host_os: cbl-mariner vmm: clh runs-on: ubuntu-latest + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ github.event.pull_request.number }} + KATA_HOST_OS: ${{ matrix.host_os }} + KATA_HYPERVISOR: ${{ matrix.vmm }} steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Download Azure CLI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - # The aks-preview extension is required while the Mariner Kata host is in preview. - az extension add --name aks-preview + run: bash tests/integration/gha-run.sh install-az-cli - name: Log into the Azure account - run: | - az login \ - --service-principal \ - -u "${{ secrets.AZ_APPID }}" \ - -p "${{ secrets.AZ_PASSWORD }}" \ - --tenant "${{ secrets.AZ_TENANT_ID }}" - - - name: Format cluster name - run: | - rev=$(git rev-parse --short=12 HEAD) - echo "cluster_name=${{ github.event.pull_request.number }}-$rev-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" >> $GITHUB_ENV + run: bash tests/integration/gha-run.sh login-azure + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - name: Create AKS cluster - run: | - az aks create \ - -g "kataCI" \ - -n "${{ env.cluster_name }}" \ - -s "Standard_D4s_v5" \ - --node-count 1 \ - --generate-ssh-keys \ - $([ "${{ matrix.host_os == 'cbl-mariner' }}" = "true" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") + run: bash tests/integration/gha-run.sh create-cluster - name: Install `bats` - run: | - sudo apt-get update - sudo apt-get -y install bats + run: bash tests/integration/gha-run.sh install-bats - name: Install `kubectl` - run: | - sudo az aks install-cli + run: bash tests/integration/gha-run.sh install-kubectl - name: Download credentials for the Kubernetes CLI to use them - run: | - az aks get-credentials -g "kataCI" -n "${{ env.cluster_name }}" + run: bash tests/integration/gha-run.sh get-cluster-credentials - name: Run tests timeout-minutes: 60 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 240s - - pushd tests/integration/kubernetes - bash setup.sh - bash run_kubernetes_tests.sh - popd - env: - KATA_HOST_OS: ${{ matrix.host_os }} - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-aks - name: Delete AKS cluster if: always() - run: | - az aks delete \ - -g "kataCI" \ - -n "${{ env.cluster_name }}" \ - --yes \ - --no-wait + run: bash tests/integration/gha-run.sh delete-cluster diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index 98a6db610..52ab7f955 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -21,6 +21,10 @@ jobs: - qemu-sev runs-on: sev env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-sev - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-sev diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 541695e0f..535c6de6d 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -21,6 +21,10 @@ jobs: - qemu-snp runs-on: sev-snp env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-snp - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-snp diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 6c5ba0dc1..886b1c026 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -21,6 +21,10 @@ jobs: - qemu-tdx runs-on: tdx env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /etc/rancher/k3s/k3s.yaml steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-tdx - name: Delete kata-deploy if: always() - run: | - kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - sleep 180s - - kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-tdx diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh new file mode 100755 index 000000000..6238141dd --- /dev/null +++ b/tests/integration/gha-run.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +integration_dir="$(dirname "$(readlink -f "$0")")" +tools_dir="${integration_dir}/../../tools" + +function _print_cluster_name() { + short_sha="$(git rev-parse --short=12 HEAD)" + echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" +} + +function install_azure_cli() { + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # The aks-preview extension is required while the Mariner Kata host is in preview. + az extension add --name aks-preview +} + +function login_azure() { + az login \ + --service-principal \ + -u "${AZ_APPID}" \ + -p "${AZ_PASSWORD}" \ + --tenant "${AZ_TENANT_ID}" +} + +function create_cluster() { + az aks create \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + -s "Standard_D4s_v5" \ + --node-count 1 \ + --generate-ssh-keys \ + $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") +} + +function install_bats() { + sudo apt-get update + sudo apt-get -y install bats +} + +function install_kubectl() { + sudo az aks install-cli +} + +function get_cluster_credentials() { + az aks get-credentials \ + -g "kataCI" \ + -n "$(_print_cluster_name)" +} + +function run_tests() { + platform="${1}" + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" + + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + if [ "${platform}" = "tdx" ]; then + kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" + else + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + fi + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" + + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + if [ "${platform}" = "aks" ]; then + sleep 240s + else + sleep 60s + fi + + pushd "${integration_dir}/kubernetes" + bash setup.sh + bash run_kubernetes_tests.sh + popd +} + +function cleanup() { + platform="${1}" + + if [ "${platform}" = "tdx" ]; then + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + else + deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + fi + + kubectl delete ${deploy_spec} + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}" || die "Failed to setup the tests image" + kubectl apply ${cleanup_spec} + sleep 180s + + kubectl delete ${cleanup_spec} + kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" +} + +function delete_cluster() { + az aks delete \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + --yes \ + --no-wait +} + +function main() { + action="${1:-}" + + case "${action}" in + install-azure-cli) install_azure_cli ;; + login-azure) login_azure ;; + create-cluster) create_cluster ;; + install-bats) install_bats ;; + install-kubectl) install_kubectl ;; + get-cluster-credentials) get_cluster_credentials ;; + run-tests-aks) run_tests "aks" ;; + run-tests-sev) run_tests "sev" ;; + run-tests-snp) run_tests "snp" ;; + run-tests-tdx) run_tests "tdx" ;; + cleanup-sev) cleanup "sev" ;; + cleanup-snp) cleanup "snp" ;; + cleanup-tdx) cleanup "tdx" ;; + delete-cluster) delete_cluster ;; + *) >&2 echo "Invalid action"; exit 2 ;; + esac +} + +main "$@" From 77519fd12016bd2ac41736bdffeceb19a5f572de Mon Sep 17 00:00:00 2001 From: Jayant Singh Date: Fri, 28 Apr 2023 23:54:49 +0000 Subject: [PATCH 30/76] kata-ctl: Switch to slog logging; add --log-level, --json-logging args Fixes: #5401, #6654 - Switch kata-ctl from eprintln!()/println!() to structured logging via the logging library which uses slog. - Adds a new create_term_logger() library call which enables printing log messages to the terminal via a less verbose / more human readable terminal format with colors. - Adds --log-level argument to select the minimum log level of printed messages. - Adds --json-logging argument to switch to logging in JSON format. Co-authored-by: Byron Marohn Co-authored-by: Luke Phillips Signed-off-by: Jayant Singh Signed-off-by: Byron Marohn Signed-off-by: Luke Phillips Signed-off-by: Kelby Madal-Hellmuth Signed-off-by: Liz Lawrens --- src/agent/Cargo.lock | 59 ++++++++++++ src/libs/logging/Cargo.toml | 1 + src/libs/logging/src/lib.rs | 22 +++++ src/tools/kata-ctl/Cargo.lock | 100 +++++++++++++++++++++ src/tools/kata-ctl/Cargo.toml | 4 + src/tools/kata-ctl/src/arch/aarch64/mod.rs | 16 +++- src/tools/kata-ctl/src/arch/s390x/mod.rs | 13 ++- src/tools/kata-ctl/src/arch/x86_64/mod.rs | 24 ++--- src/tools/kata-ctl/src/args.rs | 19 ++++ src/tools/kata-ctl/src/check.rs | 36 ++++++-- src/tools/kata-ctl/src/main.rs | 33 ++++++- src/tools/kata-ctl/src/ops/check_ops.rs | 16 ++-- src/tools/kata-ctl/src/ops/exec_ops.rs | 8 +- src/tools/kata-ctl/src/ops/volume_ops.rs | 9 +- 14 files changed, 321 insertions(+), 39 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index d5a0dcbe7..4f6df27c6 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -422,6 +422,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.7" @@ -433,6 +443,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "either" version = "1.6.1" @@ -949,6 +970,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -1762,6 +1784,12 @@ dependencies = [ "zbus", ] +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + [[package]] name = "ryu" version = "1.0.10" @@ -1951,6 +1979,19 @@ dependencies = [ "slog-scope", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.11", +] + [[package]] name = "smallvec" version = "1.8.0" @@ -2031,6 +2072,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -2102,8 +2154,15 @@ dependencies = [ "itoa", "libc", "num_threads", + "time-macros", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tokio" version = "1.28.1" diff --git a/src/libs/logging/Cargo.toml b/src/libs/logging/Cargo.toml index c7cac4d7b..4d19d90c6 100644 --- a/src/libs/logging/Cargo.toml +++ b/src/libs/logging/Cargo.toml @@ -15,6 +15,7 @@ serde_json = "1.0.73" # (by stopping the compiler from removing log calls). slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] } slog-json = "2.4.0" +slog-term = "2.9.0" slog-async = "2.7.0" slog-scope = "4.4.0" diff --git a/src/libs/logging/src/lib.rs b/src/libs/logging/src/lib.rs index 29325c6bd..b66217459 100644 --- a/src/libs/logging/src/lib.rs +++ b/src/libs/logging/src/lib.rs @@ -39,6 +39,28 @@ const LOG_LEVELS: &[(&str, slog::Level)] = &[ const DEFAULT_SUBSYSTEM: &str = "root"; +// Creates a logger which prints output as human readable text to the terminal +pub fn create_term_logger(level: slog::Level) -> (slog::Logger, slog_async::AsyncGuard) { + let term_drain = slog_term::term_compact().fuse(); + + // Ensure only a unique set of key/value fields is logged + let unique_drain = UniqueDrain::new(term_drain).fuse(); + + // Allow runtime filtering of records by log level + let filter_drain = RuntimeLevelFilter::new(unique_drain, level).fuse(); + + // Ensure the logger is thread-safe + let (async_drain, guard) = slog_async::Async::new(filter_drain) + .thread_name("slog-async-logger".into()) + .build_with_guard(); + + // Add some "standard" fields + let logger = slog::Logger::root(async_drain.fuse(), o!("subsystem" => DEFAULT_SUBSYSTEM)); + + (logger, guard) +} + +// Creates a logger which prints output as JSON // XXX: 'writer' param used to make testing possible. pub fn create_logger( name: &str, diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index 3056071be..e1ebd762e 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -96,6 +96,17 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -280,6 +291,27 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "either" version = "1.8.1" @@ -460,6 +492,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glob" version = "0.3.1" @@ -506,6 +549,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -728,6 +780,7 @@ dependencies = [ "hyper", "kata-types", "libc", + "logging", "micro_http", "nix 0.25.1", "privdrop", @@ -821,6 +874,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -961,6 +1015,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "oci" version = "0.1.0" @@ -1284,6 +1347,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.7.1" @@ -1600,6 +1674,19 @@ dependencies = [ "slog", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -1705,6 +1792,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "test-utils" version = "0.1.0" @@ -1749,6 +1847,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml index 141a90a0c..a0ba95adb 100644 --- a/src/tools/kata-ctl/Cargo.toml +++ b/src/tools/kata-ctl/Cargo.toml @@ -36,6 +36,10 @@ serial_test = "0.5.1" vmm-sys-util = "0.11.0" epoll = "4.0.1" libc = "0.2.138" + +# Note: this crate sets the slog 'max_*' features which allows the log level +# to be modified at runtime. +logging = { path = "../../libs/logging" } slog = "2.7.0" slog-scope = "4.4.0" hyper = "0.14.20" diff --git a/src/tools/kata-ctl/src/arch/aarch64/mod.rs b/src/tools/kata-ctl/src/arch/aarch64/mod.rs index 7eaf3ccb4..0f4012a36 100644 --- a/src/tools/kata-ctl/src/arch/aarch64/mod.rs +++ b/src/tools/kata-ctl/src/arch/aarch64/mod.rs @@ -11,6 +11,7 @@ mod arch_specific { use crate::types::*; use crate::utils; use anyhow::Result; + use slog::{info, o, warn}; use std::path::Path; const KVM_DEV: &str = "/dev/kvm"; @@ -19,6 +20,12 @@ mod arch_specific { #[allow(dead_code)] pub const ARCH_CPU_MODEL_FIELD: &str = "CPU architecture"; + macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "aarch64")) + }; + } + // List of check functions static CHECK_LIST: &[CheckItem] = &[CheckItem { name: CheckType::Cpu, @@ -28,11 +35,14 @@ mod arch_specific { }]; pub fn check(_args: &str) -> Result<()> { - println!("INFO: check: aarch64"); + info!(sl!(), "check: aarch64"); if Path::new(KVM_DEV).exists() { - println!("Kata Containers can run on this host\n"); + info!(sl!(), "Kata Containers can run on this host\n"); } else { - eprintln!("WARNING: Kata Containers can't run on this host as lack of virtulization support\n"); + warn!( + sl!(), + "Kata Containers can't run on this host as lack of virtulization support\n" + ); } Ok(()) diff --git a/src/tools/kata-ctl/src/arch/s390x/mod.rs b/src/tools/kata-ctl/src/arch/s390x/mod.rs index 929e79c92..d3efe6f3c 100644 --- a/src/tools/kata-ctl/src/arch/s390x/mod.rs +++ b/src/tools/kata-ctl/src/arch/s390x/mod.rs @@ -13,6 +13,7 @@ mod arch_specific { use crate::utils; use anyhow::{anyhow, Result}; use nix::unistd::Uid; + use slog::{info, o, warn}; use std::collections::HashMap; use std::io::BufRead; use std::io::BufReader; @@ -21,6 +22,12 @@ mod arch_specific { const CPUINFO_FEATURES_TAG: &str = "features"; const CPU_FEATURES_REQ: &[&str] = &["sie"]; + macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "s390x")) + }; + } + #[allow(dead_code)] pub const ARCH_CPU_VENDOR_FIELD: &str = check::GENERIC_CPU_VENDOR_FIELD; #[allow(dead_code)] @@ -28,7 +35,7 @@ mod arch_specific { // check cpu fn check_cpu() -> Result<()> { - println!("INFO: check CPU: s390x"); + info!(sl!(), "check CPU: s390x"); let cpu_info = check::get_single_cpu_info(check::PROC_CPUINFO, CPUINFO_DELIMITER)?; @@ -42,14 +49,14 @@ mod arch_specific { let missing_cpu_features = check::check_cpu_flags(&cpu_features, CPU_FEATURES_REQ)?; if !missing_cpu_features.is_empty() { - eprintln!("WARNING: Missing CPU flags {:?}", missing_cpu_features); + warn!(sl!(), "Missing CPU flags {:?}", missing_cpu_features); } Ok(()) } pub fn check(_args: &str) -> Result<()> { - println!("INFO: check: s390x"); + info!(sl!(), "check: s390x"); let _cpu_result = check_cpu(); diff --git a/src/tools/kata-ctl/src/arch/x86_64/mod.rs b/src/tools/kata-ctl/src/arch/x86_64/mod.rs index 206c447a9..53779104d 100644 --- a/src/tools/kata-ctl/src/arch/x86_64/mod.rs +++ b/src/tools/kata-ctl/src/arch/x86_64/mod.rs @@ -15,6 +15,7 @@ mod arch_specific { use crate::utils; use anyhow::{anyhow, Context, Result}; use nix::unistd::Uid; + use slog::{info, o, warn}; use std::fs; use std::path::Path; @@ -27,6 +28,12 @@ mod arch_specific { pub const ARCH_CPU_VENDOR_FIELD: &str = check::GENERIC_CPU_VENDOR_FIELD; pub const ARCH_CPU_MODEL_FIELD: &str = check::GENERIC_CPU_MODEL_FIELD; + macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "x86_64")) + }; + } + // List of check functions static CHECK_LIST: &[CheckItem] = &[ CheckItem { @@ -72,7 +79,7 @@ mod arch_specific { // check cpu fn check_cpu(_args: &str) -> Result<()> { - println!("INFO: check CPU: x86_64"); + info!(sl!(), "check CPU: x86_64"); let cpu_info = check::get_single_cpu_info(check::PROC_CPUINFO, CPUINFO_DELIMITER)?; @@ -89,14 +96,11 @@ mod arch_specific { // TODO: Add more information to output (see kata-check in go tool); adjust formatting let missing_cpu_attributes = check::check_cpu_attribs(&cpu_info, CPU_ATTRIBS_INTEL)?; if !missing_cpu_attributes.is_empty() { - eprintln!( - "WARNING: Missing CPU attributes {:?}", - missing_cpu_attributes - ); + warn!(sl!(), "Missing CPU attributes {:?}", missing_cpu_attributes); } let missing_cpu_flags = check::check_cpu_flags(&cpu_flags, CPU_FLAGS_INTEL)?; if !missing_cpu_flags.is_empty() { - eprintln!("WARNING: Missing CPU flags {:?}", missing_cpu_flags); + warn!(sl!(), "Missing CPU flags {:?}", missing_cpu_flags); } Ok(()) @@ -122,7 +126,7 @@ mod arch_specific { // check if kvm is usable fn check_kvm_is_usable(_args: &str) -> Result<()> { - println!("INFO: check if kvm is usable: x86_64"); + info!(sl!(), "check if kvm is usable: x86_64"); let result = check::check_kvm_is_usable_generic(); @@ -275,7 +279,7 @@ mod arch_specific { } fn check_kernel_modules(_args: &str) -> Result<()> { - println!("INFO: check kernel modules for: x86_64"); + info!(sl!(), "check kernel modules for: x86_64"); for module in MODULE_LIST { let module_loaded = @@ -291,12 +295,12 @@ mod arch_specific { ); match parameter_check { - Ok(_v) => println!("{} Ok", module.name), + Ok(_v) => info!(sl!(), "{} Ok", module.name), Err(e) => return Err(e), } } Err(err) => { - eprintln!("WARNING {:}", err.replace('\n', "")) + warn!(sl!(), "{:}", err.replace('\n', "")) } } } diff --git a/src/tools/kata-ctl/src/args.rs b/src/tools/kata-ctl/src/args.rs index cc2dc0513..8049ae31b 100644 --- a/src/tools/kata-ctl/src/args.rs +++ b/src/tools/kata-ctl/src/args.rs @@ -12,6 +12,25 @@ use thiserror::Error; pub struct KataCtlCli { #[clap(subcommand)] pub command: Commands, + #[clap(short, long, value_enum, value_parser = parse_log_level)] + /// Sets the minimum log level required for log messages to be displayed. Default is 'info'. + /// Valid values are: trace, debug, info, warning, error, critical + pub log_level: Option, + #[clap(short, long, action)] + /// If enabled, log messages will be JSON formatted for easier machine parsing + pub json_logging: bool, +} + +fn parse_log_level(arg: &str) -> Result { + match arg { + "trace" => Ok(slog::Level::Trace), + "debug" => Ok(slog::Level::Debug), + "info" => Ok(slog::Level::Info), + "warning" => Ok(slog::Level::Warning), + "error" => Ok(slog::Level::Error), + "critical" => Ok(slog::Level::Critical), + _ => Err("Must be one of [trace, debug, info, warning, error, critical]".to_string()), + } } #[derive(Debug, Subcommand)] diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs index ef1007f86..d531425c2 100644 --- a/src/tools/kata-ctl/src/check.rs +++ b/src/tools/kata-ctl/src/check.rs @@ -12,6 +12,7 @@ use nix::unistd::close; use nix::{ioctl_write_int_bad, request_code_none}; use reqwest::header::{CONTENT_TYPE, USER_AGENT}; use serde::{Deserialize, Serialize}; +use slog::{info, o}; use std::fmt; use thiserror::Error; @@ -51,6 +52,12 @@ pub const GENERIC_CPU_MODEL_FIELD: &str = "model name"; #[allow(dead_code)] pub const PROC_CPUINFO: &str = "/proc/cpuinfo"; +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "check")) + }; +} + fn read_file_contents(file_path: &str) -> Result { let contents = std::fs::read_to_string(file_path)?; Ok(contents) @@ -276,14 +283,20 @@ pub fn check_all_releases() -> Result<()> { for release in releases { if !release.prerelease { - println!( + info!( + sl!(), "Official : Release {:15}; created {} ; {}", - release.tag_name, release.created_at, release.tarball_url + release.tag_name, + release.created_at, + release.tarball_url ); } else { - println!( + info!( + sl!(), "PreRelease: Release {:15}; created {} ; {}", - release.tag_name, release.created_at, release.tarball_url + release.tag_name, + release.created_at, + release.tarball_url ); } } @@ -294,12 +307,15 @@ pub fn check_official_releases() -> Result<()> { let releases: Vec = get_kata_all_releases_by_url(KATA_GITHUB_RELEASE_URL).map_err(handle_reqwest_error)?; - println!("Official Releases..."); + info!(sl!(), "Official Releases..."); for release in releases { if !release.prerelease { - println!( + info!( + sl!(), "Release {:15}; created {} ; {}", - release.tag_name, release.created_at, release.tarball_url + release.tag_name, + release.created_at, + release.tarball_url ); } } @@ -392,6 +408,7 @@ pub fn check_kernel_module_loaded(module: &str, parameter: &str) -> Result { + slog_scope::logger().new(o!("subsystem" => "kata-ctl_main")) + }; +} fn real_main() -> Result<()> { let args = KataCtlCli::parse(); - match args.command { + let log_level = args.log_level.unwrap_or(slog::Level::Info); + + let (logger, _guard) = if args.json_logging { + logging::create_logger(crate_name!(), crate_name!(), log_level, io::stdout()) + } else { + logging::create_term_logger(log_level) + }; + + let _guard = slog_scope::set_global_logger(logger); + + let res = match args.command { Commands::Check(args) => handle_check(args), Commands::DirectVolume(args) => handle_direct_volume(args), Commands::Exec(args) => handle_exec(args), @@ -35,12 +53,19 @@ fn real_main() -> Result<()> { Commands::Iptables(args) => handle_iptables(args), Commands::Metrics(args) => handle_metrics(args), Commands::Version => handle_version(), + }; + + // Log errors here, then let the logger go out of scope in main() to ensure + // the asynchronous drain flushes all messages before exit() + if let Err(e) = &res { + error!(sl!(), "{:#?}", e); } + + res } fn main() { - if let Err(e) = real_main() { - eprintln!("ERROR: {:#?}", e); + if let Err(_e) = real_main() { exit(1); } } diff --git a/src/tools/kata-ctl/src/ops/check_ops.rs b/src/tools/kata-ctl/src/ops/check_ops.rs index ed418169d..8b19e0d8f 100644 --- a/src/tools/kata-ctl/src/ops/check_ops.rs +++ b/src/tools/kata-ctl/src/ops/check_ops.rs @@ -15,8 +15,16 @@ use crate::types::*; use anyhow::{anyhow, Result}; +use slog::{info, o, warn}; + const NAME: &str = "kata-ctl"; +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "check_ops")) + }; +} + // This function retrieves the cmd function passes as argument fn get_builtin_check_func(name: CheckType) -> Result { if let Some(check_list) = get_checks() { @@ -55,14 +63,12 @@ fn print_check_list() -> Result<()> { let cmds = get_client_cmd_details(); if cmds.is_empty() { - println!("Checks not found!\n"); + warn!(sl!(), "Checks not found!\n"); return Ok(()); } - cmds.iter().for_each(|n| println!(" - {}", n)); - - println!(); + cmds.iter().for_each(|n| info!(sl!(), " - {}", n)); Ok(()) } @@ -125,6 +131,6 @@ pub fn handle_metrics(_args: MetricsCommand) -> Result<()> { pub fn handle_version() -> Result<()> { let version = version::get().unwrap(); - println!("{} version {:?} (type: rust)", NAME, version); + info!(sl!(), "{} version {:?} (type: rust)", NAME, version); Ok(()) } diff --git a/src/tools/kata-ctl/src/ops/exec_ops.rs b/src/tools/kata-ctl/src/ops/exec_ops.rs index 9e1d4549a..51651188b 100644 --- a/src/tools/kata-ctl/src/ops/exec_ops.rs +++ b/src/tools/kata-ctl/src/ops/exec_ops.rs @@ -19,7 +19,7 @@ use std::{ use anyhow::{anyhow, Context}; use nix::sys::socket::{connect, socket, AddressFamily, SockFlag, SockType, VsockAddr}; use reqwest::StatusCode; -use slog::debug; +use slog::{debug, error, o}; use vmm_sys_util::terminal::Terminal; use crate::args::ExecArguments; @@ -40,7 +40,7 @@ type Result = std::result::Result; #[macro_export] macro_rules! sl { () => { - slog_scope::logger() + slog_scope::logger().new(o!("subsystem" => "exec_ops")) }; } @@ -142,7 +142,7 @@ impl EpollContext { return Ok(()); } Err(e) => { - println!("error with errno {:?} while reading stdin", e); + error!(sl!(), "errno {:?} while reading stdin", e); return Ok(()); } Ok(count) => { @@ -159,7 +159,7 @@ impl EpollContext { return Ok(()); } Err(e) => { - println!("error with errno {:?} while reading server", e); + error!(sl!(), "errno {:?} while reading server", e); return Ok(()); } Ok(count) => { diff --git a/src/tools/kata-ctl/src/ops/volume_ops.rs b/src/tools/kata-ctl/src/ops/volume_ops.rs index 0b037bf81..94243743c 100644 --- a/src/tools/kata-ctl/src/ops/volume_ops.rs +++ b/src/tools/kata-ctl/src/ops/volume_ops.rs @@ -13,6 +13,7 @@ use kata_types::mount::{ }; use nix; use reqwest::StatusCode; +use slog::{info, o}; use std::{fs, time::Duration}; use url; @@ -25,6 +26,12 @@ use shim_interface::shim_mgmt::{ const TIMEOUT: Duration = Duration::from_millis(2000); const CONTENT_TYPE_JSON: &str = "application/json"; +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "volume_ops")) + }; +} + pub fn handle_direct_volume(vol_cmd: DirectVolumeCommand) -> Result<()> { if !nix::unistd::Uid::effective().is_root() { return Err(anyhow!( @@ -41,7 +48,7 @@ pub fn handle_direct_volume(vol_cmd: DirectVolumeCommand) -> Result<()> { } }; if let Some(cmd_result) = cmd_result { - println!("{:?}", cmd_result); + info!(sl!(), "{:?}", cmd_result); } Ok(()) From 410bc1814394051efb60e95eedc28c814e6d2548 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Mon, 5 Jun 2023 21:03:20 +0800 Subject: [PATCH 31/76] agent-ctl: fix the compile error When the version of libc is upgraded to 0.2.145, older getrandom could not adapt to new API, and this will make agent-ctl fail to compile. We upgrade the version of `rand`, so the low version of getrandom will no longer need. Fixes: #7032 Signed-off-by: Yushuo --- src/agent/Cargo.lock | 76 +-- src/libs/Cargo.lock | 70 +-- src/libs/kata-sys-util/Cargo.toml | 2 +- src/runtime-rs/Cargo.lock | 2 +- src/tools/agent-ctl/Cargo.lock | 937 ++++++++++++++++-------------- src/tools/kata-ctl/Cargo.lock | 19 - src/tools/runk/Cargo.lock | 82 +-- 7 files changed, 549 insertions(+), 639 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index d5a0dcbe7..5cc002f20 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -495,7 +495,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" dependencies = [ "lazy_static", "log", - "rand 0.8.5", + "rand", ] [[package]] @@ -633,17 +633,6 @@ dependencies = [ "slab", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.7" @@ -861,7 +850,7 @@ dependencies = [ "nix 0.24.2", "oci", "once_cell", - "rand 0.7.3", + "rand", "serde_json", "slog", "slog-scope", @@ -1184,7 +1173,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project", - "rand 0.8.5", + "rand", "serde", "thiserror", "tokio", @@ -1573,19 +1562,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -1593,18 +1569,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -1614,16 +1580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -1632,16 +1589,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.7", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "getrandom", ] [[package]] @@ -1659,7 +1607,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.7", + "getrandom", "redox_syscall", "thiserror", ] @@ -2383,12 +2331,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -2651,7 +2593,7 @@ dependencies = [ "nix 0.23.1", "once_cell", "ordered-stream", - "rand 0.8.5", + "rand", "serde", "serde_repr", "sha1", diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 6415f8d1e..5da16917f 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -186,7 +186,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" dependencies = [ "lazy_static", "log", - "rand 0.8.5", + "rand", ] [[package]] @@ -299,17 +299,6 @@ dependencies = [ "slab", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.6" @@ -495,7 +484,7 @@ dependencies = [ "num_cpus", "oci", "once_cell", - "rand 0.7.3", + "rand", "serde_json", "serial_test", "slog", @@ -929,19 +918,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -949,18 +925,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -970,16 +936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -988,16 +945,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.6", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "getrandom", ] [[package]] @@ -1428,12 +1376,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 20497bfa6..40381b4d0 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -24,7 +24,7 @@ serde_json = "1.0.73" slog = "2.5.2" slog-scope = "4.4.0" subprocess = "0.2.8" -rand = "0.7.2" +rand = "0.8.5" thiserror = "1.0.30" kata-types = { path = "../kata-types" } diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 7f974d744..9759e0e54 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -1460,7 +1460,7 @@ dependencies = [ "nix 0.24.3", "oci", "once_cell", - "rand 0.7.3", + "rand 0.8.5", "serde_json", "slog", "slog-scope", diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index bfa0177fb..8f536f130 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -4,13 +4,19 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.18" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -31,15 +37,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arc-swap" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-broadcast" @@ -54,9 +60,9 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ "concurrent-queue", "event-listener", @@ -65,43 +71,43 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" dependencies = [ + "async-lock", "async-task", "concurrent-queue", "fastrand", "futures-lite", - "once_cell", "slab", ] [[package]] name = "async-io" -version = "1.9.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ + "async-lock", "autocfg", + "cfg-if 1.0.0", "concurrent-queue", "futures-lite", - "libc", "log", - "once_cell", "parking", "polling", + "rustix", "slab", "socket2", "waker-fn", - "winapi", ] [[package]] name = "async-lock" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", ] @@ -114,24 +120,24 @@ checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] name = "async-task" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" +checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] @@ -165,9 +171,9 @@ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitmask-enum" @@ -176,14 +182,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2" dependencies = [ "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byte-unit" @@ -199,42 +205,36 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" - -[[package]] -name = "cache-padded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "capctl" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eea0d91a34c56f0a0779e1cc2ec7040fa7f672819c4d3fe7d9dd4af3d2e78aca" +checksum = "fbdc32a78afc325d71a48d13084f1c3ddf67cc5dc06c6e5439a8630b14612cad" dependencies = [ "bitflags", + "cfg-if 1.0.0", "libc", ] [[package]] name = "caps" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61bf7211aad104ce2769ec05efcdfabf85ee84ac92461d142f22cf8badd0e54c" +checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" dependencies = [ - "errno", "libc", "thiserror", ] [[package]] name = "cc" -version = "1.0.72" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -263,15 +263,15 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", - "time", + "time 0.1.45", "wasm-bindgen", "winapi", ] @@ -299,24 +299,24 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" [[package]] name = "concurrent-queue" -version = "1.2.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" dependencies = [ - "cache-padded", + "crossbeam-utils", ] [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "crossbeam-channel" -version = "0.5.1" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -324,12 +324,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if 1.0.0", - "lazy_static", ] [[package]] @@ -340,7 +339,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] @@ -351,7 +350,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] @@ -376,15 +375,15 @@ dependencies = [ [[package]] name = "either" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "enumflags2" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75d4cd21b95383444831539909fbb14b9dc3fdceb2a6f5d36577329a1f55ccb" +checksum = "c041f5090df68b32bcd905365fd51769c8b9d553fe87fde0b683534f10c01bd2" dependencies = [ "enumflags2_derive", "serde", @@ -392,24 +391,24 @@ dependencies = [ [[package]] name = "enumflags2_derive" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae" +checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "errno" -version = "0.2.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -436,14 +435,14 @@ checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" dependencies = [ "log", "once_cell", - "rand 0.8.4", + "rand", ] [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -456,9 +455,9 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "futures" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -471,9 +470,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -481,15 +480,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -498,15 +497,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-lite" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ "fastrand", "futures-core", @@ -519,36 +518,33 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "autocfg", - "proc-macro-hack", "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "futures-sink" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ - "autocfg", "futures-channel", "futures-core", "futures-io", @@ -558,31 +554,18 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] [[package]] name = "getrandom" -version = "0.1.16" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -624,6 +607,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -638,15 +627,25 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.46" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" dependencies = [ "android_system_properties", "core-foundation-sys", + "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", ] [[package]] @@ -691,25 +690,36 @@ dependencies = [ ] [[package]] -name = "itertools" -version = "0.10.3" +name = "io-lifetimes" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.58" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -726,11 +736,11 @@ dependencies = [ "lazy_static", "libc", "logging", - "nix 0.23.1", + "nix 0.23.2", "oci", "protobuf 3.2.0", "protocols", - "rand 0.8.4", + "rand", "rustjail", "serde", "serde_json", @@ -751,10 +761,10 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix 0.24.2", + "nix 0.24.3", "oci", "once_cell", - "rand 0.7.3", + "rand", "serde_json", "slog", "slog-scope", @@ -775,6 +785,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -791,15 +802,21 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.132" +version = "0.2.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "fc86cde3ff845662b8f4ef6cb50ea0e20c524eb3d29ae048287e06a1b3fa6a81" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -807,12 +824,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "logging" @@ -827,9 +841,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -842,24 +856,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.14" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", ] [[package]] @@ -870,9 +873,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "nix" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ "bitflags", "cc", @@ -883,9 +886,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -905,30 +908,11 @@ dependencies = [ "libc", ] -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi", -] - -[[package]] -name = "num-integer" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -955,9 +939,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.14.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-stream" @@ -971,9 +955,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" [[package]] name = "parking_lot" @@ -987,15 +971,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1041,52 +1025,41 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "polling" -version = "2.3.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", + "bitflags", "cfg-if 1.0.0", + "concurrent-queue", "libc", "log", - "wepoll-ffi", - "winapi", + "pin-project-lite", + "windows-sys 0.48.0", ] [[package]] name = "ppv-lite86" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-crate" -version = "1.2.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "thiserror", - "toml", + "toml_edit", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -1129,7 +1102,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] @@ -1222,46 +1195,22 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] [[package]] name = "rand" -version = "0.7.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", -] - -[[package]] -name = "rand" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.3", - "rand_hc 0.3.1", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -1271,43 +1220,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core", ] [[package]] name = "rand_core" -version = "0.5.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.1.16", -] - -[[package]] -name = "rand_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" -dependencies = [ - "getrandom 0.2.3", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core 0.6.3", + "getrandom", ] [[package]] @@ -1319,22 +1241,31 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.3", - "redox_syscall", + "getrandom", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.5.6" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -1343,18 +1274,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.26" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "rlimit" @@ -1365,6 +1287,20 @@ dependencies = [ "libc", ] +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + [[package]] name = "rustjail" version = "0.1.0" @@ -1381,7 +1317,7 @@ dependencies = [ "kata-sys-util", "lazy_static", "libc", - "nix 0.24.2", + "nix 0.24.3", "oci", "path-absolutize", "protobuf 3.2.0", @@ -1402,9 +1338,16 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] [[package]] name = "scan_fmt" @@ -1423,29 +1366,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "serde_json" -version = "1.0.73" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1454,13 +1397,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] @@ -1480,18 +1423,21 @@ checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.5" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] [[package]] name = "slash-formatter" @@ -1519,14 +1465,14 @@ dependencies = [ [[package]] name = "slog-json" -version = "2.4.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9b96fb6b5e80e371423b4aca6656eb537661ce8f82c2697e619f8ca85d043" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" dependencies = [ - "chrono", "serde", "serde_json", "slog", + "time 0.3.21", ] [[package]] @@ -1542,15 +1488,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1580,20 +1526,20 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.82" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] name = "syn" -version = "2.0.14" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -1608,16 +1554,15 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "tempfile" -version = "3.2.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if 1.0.0", - "libc", - "rand 0.8.4", - "redox_syscall", - "remove_dir_all", - "winapi", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", ] [[package]] @@ -1631,85 +1576,131 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if 1.0.0", "once_cell", ] [[package]] name = "time" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] +[[package]] +name = "time" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", +] + [[package]] name = "tokio" -version = "1.15.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ + "autocfg", "bytes", "libc", - "memchr", "mio", - "once_cell", "pin-project-lite", "signal-hook-registry", + "socket2", "tokio-macros", - "winapi", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.7.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "toml" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] [[package]] -name = "tracing" -version = "0.1.36" +name = "toml_datetime" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" + +[[package]] +name = "toml_edit" +version = "0.19.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", @@ -1719,20 +1710,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -1746,7 +1737,7 @@ dependencies = [ "byteorder", "libc", "log", - "nix 0.23.1", + "nix 0.23.2", "protobuf 3.2.0", "protobuf-codegen 3.2.0", "thiserror", @@ -1791,27 +1782,21 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-segmentation" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" - -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "vec_map" @@ -1827,21 +1812,21 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -1849,24 +1834,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1874,41 +1859,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" - -[[package]] -name = "wepoll-ffi" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" -dependencies = [ - "cc", -] +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "which" -version = "4.2.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", - "lazy_static", "libc", + "once_cell", ] [[package]] @@ -1934,47 +1910,154 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.36.1" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows-targets 0.48.0", ] [[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.36.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.36.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "winnow" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +dependencies = [ + "memchr", +] [[package]] name = "xattr" @@ -2009,10 +2092,10 @@ dependencies = [ "futures-util", "hex", "lazy_static", - "nix 0.23.1", + "nix 0.23.2", "once_cell", "ordered-stream", - "rand 0.8.4", + "rand", "serde", "serde_repr", "sha1", @@ -2035,14 +2118,14 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] name = "zbus_names" -version = "2.2.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41a408fd8a352695690f53906dc7fd036be924ec51ea5e05666ff42685ed0af5" +checksum = "82441e6033be0a741157a72951a3e4957d519698f3a824439cc131c5ba77ac2a" dependencies = [ "serde", "static_assertions", @@ -2051,9 +2134,9 @@ dependencies = [ [[package]] name = "zvariant" -version = "3.6.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bd68e4e6432ef19df47d7e90e2e72b5e7e3d778e0ae3baddf12b951265cc758" +checksum = "622cc473f10cef1b0d73b7b34a266be30ebdcfaea40ec297dd8cbda088f9f93c" dependencies = [ "byteorder", "enumflags2", @@ -2065,12 +2148,24 @@ dependencies = [ [[package]] name = "zvariant_derive" -version = "3.6.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08e977eaa3af652f63d479ce50d924254ad76722a6289ec1a1eac3231ca30430" +checksum = "5d9c1b57352c25b778257c661f3c4744b7cefb7fc09dd46909a153cce7773da2" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7234f0d811589db492d16893e3f21e8e2fd282e6d01b0cddee310322062cc200" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index 3056071be..59d454f62 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -764,7 +764,6 @@ dependencies = [ "lazy_static", "num_cpus", "oci", - "proc-mounts", "regex", "safe-path", "serde", @@ -1047,15 +1046,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "partition-identity" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa925f9becb532d758b0014b472c576869910929cf4c3f8054b386f19ab9e21" -dependencies = [ - "thiserror", -] - [[package]] name = "percent-encoding" version = "2.2.0" @@ -1129,15 +1119,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "proc-mounts" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d652f8435d0ab70bf4f3590a6a851d59604831a458086541b95238cc51ffcf2" -dependencies = [ - "partition-identity", -] - [[package]] name = "prost" version = "0.8.0" diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock index b280b2a27..ed239100a 100644 --- a/src/tools/runk/Cargo.lock +++ b/src/tools/runk/Cargo.lock @@ -511,7 +511,7 @@ checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" dependencies = [ "log", "once_cell", - "rand 0.8.5", + "rand", ] [[package]] @@ -649,17 +649,6 @@ dependencies = [ "slab", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.7" @@ -796,7 +785,7 @@ dependencies = [ "nix 0.24.2", "oci", "once_cell", - "rand 0.7.3", + "rand", "serde_json", "slog", "slog-scope", @@ -817,6 +806,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -1363,19 +1353,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -1383,18 +1360,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -1404,16 +1371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -1422,16 +1380,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.7", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "getrandom", ] [[package]] @@ -1449,7 +1398,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.7", + "getrandom", "redox_syscall", "thiserror", ] @@ -1569,6 +1518,13 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] + [[package]] name = "scan_fmt" version = "0.2.6" @@ -2034,12 +1990,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -2357,7 +2307,7 @@ dependencies = [ "nix 0.23.1", "once_cell", "ordered-stream", - "rand 0.8.5", + "rand", "serde", "serde_repr", "sha1", From 980d084f47f73afbf01f7c6af79629ac5f4872f6 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 5 Jun 2023 15:59:52 +0000 Subject: [PATCH 32/76] log-parser: Update log parser link at README This PR updates the link to the correspondent Developer Guide at the enabling full containerd debug that we have for kata 2.0 documentation. Fixes #7034 Signed-off-by: Gabriela Cervantes --- src/tools/log-parser-rs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/log-parser-rs/README.md b/src/tools/log-parser-rs/README.md index a641b0030..18df4feeb 100644 --- a/src/tools/log-parser-rs/README.md +++ b/src/tools/log-parser-rs/README.md @@ -48,7 +48,7 @@ For a comprehensive (and guaranteed up to date) list, please run `log-parser-rs ## Usage -1. Make sure containerd is in [debug mode](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#enabling-full-containerd-debug) +1. Make sure containerd is in [debug mode](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enabling-full-containerd-debug) 1. Make sure you are running runtime-rs: ``` $ containerd-shim-kata-v2 --version|grep -qi rust && echo rust || echo golang From 31c0ad207653c15998a3ef41316528231ec44312 Mon Sep 17 00:00:00 2001 From: Arron Wang Date: Wed, 4 May 2022 16:35:49 +0800 Subject: [PATCH 33/76] packaging: Add cryptsetup support in Guest kernel and rootfs Add required kernel config for dm-crypt/dm-integrity/dm-verity and related crypto config. Add userspace command line tools for disk encryption support and ext4 file system utilities. Fixes: #6674 Signed-off-by: Arron Wang --- tools/osbuilder/rootfs-builder/rootfs.sh | 2 ++ .../osbuilder/rootfs-builder/ubuntu/config.sh | 1 + .../kata-deploy-binaries-in-docker.sh | 1 + tools/packaging/kernel/build-kernel.sh | 4 ++++ .../confidential_containers/cryptsetup.conf | 21 +++++++++++++++++++ .../configs/fragments/x86_64/crypto.conf | 3 +++ tools/packaging/kernel/kata_config_version | 2 +- 7 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tools/packaging/kernel/configs/fragments/common/confidential_containers/cryptsetup.conf create mode 100644 tools/packaging/kernel/configs/fragments/x86_64/crypto.conf diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index a494cb440..22940ee99 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -16,6 +16,7 @@ AGENT_VERSION=${AGENT_VERSION:-} RUST_VERSION="null" AGENT_BIN=${AGENT_BIN:-kata-agent} AGENT_INIT=${AGENT_INIT:-no} +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""} OSBUILDER_VERSION="unknown" DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc} @@ -434,6 +435,7 @@ build_rootfs_distro() --env AGENT_INIT="${AGENT_INIT}" \ --env ARCH="${ARCH}" \ --env CI="${CI}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ --env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \ --env LIBC="${LIBC}" \ --env EXTRA_PKGS="${EXTRA_PKGS}" \ diff --git a/tools/osbuilder/rootfs-builder/ubuntu/config.sh b/tools/osbuilder/rootfs-builder/ubuntu/config.sh index ec5dfafd6..02d93e7dd 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/config.sh +++ b/tools/osbuilder/rootfs-builder/ubuntu/config.sh @@ -7,6 +7,7 @@ OS_NAME=ubuntu OS_VERSION=${OS_VERSION:-focal} PACKAGES="chrony iptables dbus" [ "$AGENT_INIT" = no ] && PACKAGES+=" init" +[ "$MEASURED_ROOTFS" = yes ] && PACKAGES+=" cryptsetup-bin e2fsprogs" [ "$SECCOMP" = yes ] && PACKAGES+=" libseccomp2" REPO_URL=http://ports.ubuntu.com diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 6464c8722..11589c88a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -64,6 +64,7 @@ docker run \ --env SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER:-}" \ --env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \ --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ --rm \ -w ${script_dir} \ build-kata-deploy "${kata_deploy_create}" $@ diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 17ee523e6..6b42be119 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -270,6 +270,10 @@ get_kernel_frag_path() { fi if [[ "${conf_guest}" != "" ]];then + info "Enabling config for confidential guest trust storage protection" + local cryptsetup_configs="$(ls ${common_path}/confidential_containers/cryptsetup.conf)" + all_configs="${all_configs} ${cryptsetup_configs}" + info "Enabling config for '${conf_guest}' confidential guest protection" local conf_configs="$(ls ${arch_path}/${conf_guest}/*.conf)" all_configs="${all_configs} ${conf_configs}" diff --git a/tools/packaging/kernel/configs/fragments/common/confidential_containers/cryptsetup.conf b/tools/packaging/kernel/configs/fragments/common/confidential_containers/cryptsetup.conf new file mode 100644 index 000000000..a3e04e9b1 --- /dev/null +++ b/tools/packaging/kernel/configs/fragments/common/confidential_containers/cryptsetup.conf @@ -0,0 +1,21 @@ +CONFIG_MD=y +CONFIG_BLK_DEV_DM_BUILTIN=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_DM_INTEGRITY=y +CONFIG_CRYPTO_AEAD=y +CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_AKCIPHER2=y +CONFIG_CRYPTO_KPP2=y +CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_MANAGER2=y +CONFIG_CRYPTO_USER=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_NULL2=y +CONFIG_CRYPTO_CRYPTD=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_ESSIV=y +CONFIG_CRYPTO_XTS=y +CONFIG_CRYPTO_HMAC=y diff --git a/tools/packaging/kernel/configs/fragments/x86_64/crypto.conf b/tools/packaging/kernel/configs/fragments/x86_64/crypto.conf new file mode 100644 index 000000000..5cd7070f3 --- /dev/null +++ b/tools/packaging/kernel/configs/fragments/x86_64/crypto.conf @@ -0,0 +1,3 @@ +# x86 cryptographic instructions to improve AES encryption and SHA256 hashing. +CONFIG_CRYPTO_SHA256_SSSE3=y +CONFIG_CRYPTO_AES_NI_INTEL=y diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index e34885bbc..3b20426c0 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -107 +108 From 5cb02a80670e6f19da4da937f5f26f8f54051ff0 Mon Sep 17 00:00:00 2001 From: "Wang, Arron" Date: Tue, 23 Aug 2022 17:21:07 +0800 Subject: [PATCH 34/76] image-build: generate root hash as an separate partition for rootfs Generate rootfs hash data during creating the kata rootfs, current kata image only have one partition, we add another partition as hash device to save hash data of rootfs data blocks. Fixes: #6674 Signed-off-by: Wang, Arron --- tools/osbuilder/image-builder/Dockerfile | 1 + .../osbuilder/image-builder/image_builder.sh | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tools/osbuilder/image-builder/Dockerfile b/tools/osbuilder/image-builder/Dockerfile index 87f1e570f..fcfa5ac20 100644 --- a/tools/osbuilder/image-builder/Dockerfile +++ b/tools/osbuilder/image-builder/Dockerfile @@ -14,5 +14,6 @@ RUN ([ -n "$http_proxy" ] && \ gdisk \ parted \ qemu-img \ + veritysetup \ xfsprogs && \ dnf clean all diff --git a/tools/osbuilder/image-builder/image_builder.sh b/tools/osbuilder/image-builder/image_builder.sh index 2ae656b94..3e7f0babc 100755 --- a/tools/osbuilder/image-builder/image_builder.sh +++ b/tools/osbuilder/image-builder/image_builder.sh @@ -11,6 +11,7 @@ set -o errexit set -o pipefail DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc} +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} readonly script_name="${0##*/}" readonly script_dir=$(dirname "$(readlink -f "$0")") @@ -185,6 +186,7 @@ build_with_container() { --env BLOCK_SIZE="${block_size}" \ --env ROOT_FREE_SPACE="${root_free_space}" \ --env NSDAX_BIN="${nsdax_bin}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ --env SELINUX="${SELINUX}" \ --env DEBUG="${DEBUG}" \ -v /dev:/dev \ @@ -391,9 +393,21 @@ create_disk() { # Kata runtime expect an image with just one partition # The partition is the rootfs content info "Creating partitions" + + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + info "Creating partitions with hash device" + # The hash data will take less than one percent disk space to store + hash_start=$(echo $img_size | awk '{print $1 * 0.99}' |cut -d $(locale decimal_point) -f 1) + partition_param="mkpart primary ${fs_type} ${part_start}M ${hash_start}M " + partition_param+="mkpart primary ${fs_type} ${hash_start}M ${rootfs_end}M " + partition_param+="set 1 boot on" + else + partition_param="mkpart primary ${fs_type} ${part_start}M ${rootfs_end}M" + fi + parted -s -a optimal "${image}" -- \ mklabel msdos \ - mkpart primary "${fs_type}" "${part_start}"M "${rootfs_end}"M + "${partition_param}" OK "Partitions created" } @@ -490,6 +504,12 @@ create_rootfs_image() { fsck.ext4 -D -y "${device}p1" fi + if [ "${MEASURED_ROOTFS}" == "yes" ] && [ -b "${device}p2" ]; then + info "veritysetup format rootfs device: ${device}p1, hash device: ${device}p2" + local image_dir=$(dirname "${image}") + veritysetup format "${device}p1" "${device}p2" > "${image_dir}"/root_hash.txt 2>&1 + fi + losetup -d "${device}" rm -rf "${mount_dir}" } From 28b2645624e86abedc9158c6261f1859a70611c8 Mon Sep 17 00:00:00 2001 From: "Wang, Arron" Date: Thu, 1 Sep 2022 13:02:22 +0800 Subject: [PATCH 35/76] initramfs: Add build script to generate initramfs The init.sh in initramfs will parse the verity scheme, roothash, root device and setup the root device accordingly. Fixes: #6674 Signed-off-by: Wang, Arron --- .../local-build/kata-deploy-binaries.sh | 7 +++ .../static-build/initramfs/Dockerfile | 38 +++++++++++++ .../static-build/initramfs/build-initramfs.sh | 55 +++++++++++++++++++ .../packaging/static-build/initramfs/build.sh | 46 ++++++++++++++++ .../packaging/static-build/initramfs/init.sh | 44 +++++++++++++++ .../static-build/initramfs/initramfs.list | 21 +++++++ versions.yaml | 10 ++++ 7 files changed, 221 insertions(+) create mode 100644 tools/packaging/static-build/initramfs/Dockerfile create mode 100755 tools/packaging/static-build/initramfs/build-initramfs.sh create mode 100755 tools/packaging/static-build/initramfs/build.sh create mode 100755 tools/packaging/static-build/initramfs/init.sh create mode 100644 tools/packaging/static-build/initramfs/initramfs.list diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 877576ca6..e80972f35 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -24,6 +24,7 @@ readonly versions_yaml="${repo_root_dir}/versions.yaml" readonly clh_builder="${static_build_dir}/cloud-hypervisor/build-static-clh.sh" readonly firecracker_builder="${static_build_dir}/firecracker/build-static-firecracker.sh" +readonly initramfs_builder="${static_build_dir}/initramfs/build.sh" readonly kernel_builder="${static_build_dir}/kernel/build.sh" readonly ovmf_builder="${static_build_dir}/ovmf/build.sh" readonly qemu_builder="${static_build_dir}/qemu/build-static-qemu.sh" @@ -38,6 +39,7 @@ readonly jenkins_url="http://jenkins.katacontainers.io" readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" ARCH=$(uname -m) +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} workdir="${WORKDIR:-$PWD}" @@ -241,6 +243,11 @@ install_kernel_helper() { install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0 + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + info "build initramfs for cc kernel" + "${initramfs_builder}" + fi + info "build ${kernel_name}" info "Kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -v "${kernel_version}" ${extra_cmd} diff --git a/tools/packaging/static-build/initramfs/Dockerfile b/tools/packaging/static-build/initramfs/Dockerfile new file mode 100644 index 000000000..3ebe0f347 --- /dev/null +++ b/tools/packaging/static-build/initramfs/Dockerfile @@ -0,0 +1,38 @@ +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +from ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=UTC +RUN apt-get update &&\ + apt-get --no-install-recommends install -y software-properties-common &&\ + add-apt-repository ppa:git-core/ppa -y &&\ + apt-get update && apt-get upgrade -y && \ + apt-get --no-install-recommends install -y \ + apt-utils \ + asciidoctor \ + autoconf \ + autopoint \ + automake \ + busybox-static \ + ca-certificates \ + curl \ + gcc \ + gettext \ + git \ + libaio-dev \ + libblkid-dev \ + libselinux1-dev \ + libtool \ + libpopt-dev \ + libjson-c-dev \ + libssl-dev \ + make \ + ninja-build \ + pkg-config \ + uuid-dev \ + libseccomp-dev \ + libseccomp2 \ + zlib1g-dev &&\ + apt-get clean && rm -rf /var/lib/apt/lists/ diff --git a/tools/packaging/static-build/initramfs/build-initramfs.sh b/tools/packaging/static-build/initramfs/build-initramfs.sh new file mode 100755 index 000000000..a011e9822 --- /dev/null +++ b/tools/packaging/static-build/initramfs/build-initramfs.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Copyright (c) 2022 Intel +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${script_dir}/../../scripts/lib.sh" +install_dir="${1:-.}" + +cryptsetup_repo="${cryptsetup_repo:-}" +cryptsetup_version="${cryptsetup_version:-}" +lvm2_repo="${lvm2_repo:-}" +lvm2_version="${lvm2_version:-}" + +[ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo" +[ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version" +[ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" +[ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" + +build_root=$(mktemp -d) +pushd ${build_root} + +info "Build ${lvm2_repo} version: ${lvm2_version}" +git clone --depth 1 --branch "${lvm2_version}" "${lvm2_repo}" lvm2 +pushd lvm2 +./configure --enable-static_link --disable-selinux +make && make install +cp ./libdm/libdevmapper.pc /usr/lib/pkgconfig/devmapper.pc +popd #lvm2 + +info "Build ${cryptsetup_repo} version: ${cryptsetup_version}" +git clone --depth 1 --branch "${cryptsetup_version}" "${cryptsetup_repo}" cryptsetup +pushd cryptsetup +./autogen.sh +./configure --enable-static --enable-static-cryptsetup --disable-udev --disable-external-tokens --disable-ssh-token +make && make install +strip /usr/sbin/veritysetup.static +popd #cryptsetup + +info "Build gen_init_cpio tool" +git clone --depth 1 --filter=blob:none --sparse https://github.com/torvalds/linux.git +pushd linux +git sparse-checkout add usr && cd usr && make gen_init_cpio +install gen_init_cpio /usr/sbin/ +popd #linux + +popd #${build_root} + +install "${script_dir}/init.sh" /usr/sbin/ +gen_init_cpio "${script_dir}/initramfs.list" | gzip -9 -n > "${install_dir}"/initramfs.cpio.gz diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh new file mode 100755 index 000000000..489ddd5e1 --- /dev/null +++ b/tools/packaging/static-build/initramfs/build.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 Intel +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root_dir="$(cd "${script_dir}/../../../.." && pwd)" +readonly initramfs_builder="${script_dir}/build-initramfs.sh" +readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)" + +source "${script_dir}/../../scripts/lib.sh" + +container_image="kata-initramfs-builder" +kata_version="${kata_version:-}" +cryptsetup_repo="${cryptsetup_repo:-}" +cryptsetup_version="${cryptsetup_version:-}" +lvm2_repo="${lvm2_repo:-}" +lvm2_version="${lvm2_version:-}" +package_output_dir="${package_output_dir:-}" + +[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url" "${kata_version}") +[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version" "${kata_version}") +[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url" "${kata_version}") +[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version" "${kata_version}") + +[ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo" +[ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version" +[ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" +[ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" + +sudo docker build \ + -t "${container_image}" "${script_dir}" + +sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ + -w "${PWD}" \ + --env cryptsetup_repo="${cryptsetup_repo}" \ + --env cryptsetup_version="${cryptsetup_version}" \ + --env lvm2_repo="${lvm2_repo}" \ + --env lvm2_version="${lvm2_version}" \ + "${container_image}" \ + bash -c "${initramfs_builder} ${default_install_dir}" diff --git a/tools/packaging/static-build/initramfs/init.sh b/tools/packaging/static-build/initramfs/init.sh new file mode 100755 index 000000000..d6a68d62f --- /dev/null +++ b/tools/packaging/static-build/initramfs/init.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# +# Copyright (c) 2022 Intel +# +# SPDX-License-Identifier: Apache-2.0 + +[ -d /dev ] || mkdir -m 0755 /dev +[ -d /root ] || mkdir -m 0700 /root +[ -d /sys ] || mkdir /sys +[ -d /proc ] || mkdir /proc +[ -d /mnt ] || mkdir /mnt +[ -d /tmp ] || mkdir /tmp + +mount -t sysfs -o nodev,noexec,nosuid sysfs /sys +mount -t proc -o nodev,noexec,nosuid proc /proc + +echo "/sbin/mdev" > /proc/sys/kernel/hotplug +mdev -s + +get_option() { + local value + value=" $(cat /proc/cmdline) " + value="${value##* ${1}=}" + value="${value%% *}" + [ "${value}" != "" ] && echo "${value}" +} + +rootfs_verifier=$(get_option rootfs_verity.scheme) +rootfs_hash=$(get_option rootfs_verity.hash) +root_device=$(get_option root) +hash_device=${root_device%?}2 + +if [ -e ${root_device} ] && [ -e ${hash_device} ] && [ "${rootfs_verifier}" = "dm-verity" ] +then + veritysetup open "${root_device}" root "${hash_device}" "${rootfs_hash}" + mount /dev/mapper/root /mnt +else + echo "No LUKS device found" + mount "${root_device}" /mnt +fi + +umount /proc +umount /sys +exec switch_root /mnt /sbin/init diff --git a/tools/packaging/static-build/initramfs/initramfs.list b/tools/packaging/static-build/initramfs/initramfs.list new file mode 100644 index 000000000..90c8af8c9 --- /dev/null +++ b/tools/packaging/static-build/initramfs/initramfs.list @@ -0,0 +1,21 @@ +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# initramfs to setup verified boot for rootfs +dir /dev 0755 0 0 +dir /root 0700 0 0 +dir /sbin 0755 0 0 +dir /bin 0755 0 0 +dir /run 0755 0 0 +dir /mnt 0755 0 0 +file /init /usr/sbin/init.sh 0755 0 0 +file /sbin/busybox /usr/bin/busybox 0755 0 0 +file /sbin/veritysetup /usr/sbin/veritysetup.static 0755 0 0 +slink /bin/sh /sbin/busybox 0755 0 0 +slink /sbin/mount /sbin/busybox 0755 0 0 +slink /bin/mkdir /sbin/busybox 0755 0 0 +slink /sbin/mdev /sbin/busybox 0755 0 0 +slink /sbin/switch_root /sbin/busybox 0755 0 0 +slink /sbin/umount /sbin/busybox 0755 0 0 +slink /sbin/cat /sbin/busybox 0755 0 0 diff --git a/versions.yaml b/versions.yaml index 18e51fe83..3455221a8 100644 --- a/versions.yaml +++ b/versions.yaml @@ -228,11 +228,21 @@ externals: url: "https://github.com/kubernetes-sigs/cri-tools" version: "1.23.0" + cryptsetup: + description: "A utility used to setup disk encryption, integrity protection" + url: "https://gitlab.com/cryptsetup/cryptsetup" + version: "v2.5.0" + gperf: description: "GNU gperf is a perfect hash function generator" url: "http://ftp.gnu.org/pub/gnu/gperf/" version: "3.1" + lvm2: + description: "LVM2 and device-mapper tools and libraries" + url: "https://github.com/lvmteam/lvm2" + version: "v2_03_16" + kubernetes: description: "Kubernetes project container manager" url: "https://github.com/kubernetes/kubernetes" From 0080588075f1a09d6ed38f6e109a312905bfbefb Mon Sep 17 00:00:00 2001 From: "Wang, Arron" Date: Mon, 5 Sep 2022 17:36:42 +0800 Subject: [PATCH 36/76] kernel: Integrate initramfs into Guest kernel Integrate initramfs into guest kernel as one binary, which will be measured by the firmware together. Fixes: #6674 Signed-off-by: Wang, Arron --- tools/packaging/kernel/build-kernel.sh | 18 +++++++++++++++++- .../confidential_containers/initramfs.conf | 1 + tools/packaging/static-build/kernel/build.sh | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tools/packaging/kernel/configs/fragments/common/confidential_containers/initramfs.conf diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 6b42be119..c1f89cfeb 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -31,6 +31,7 @@ readonly default_kernel_config_dir="${script_dir}/configs" # Default path to search for kernel config fragments readonly default_config_frags_dir="${script_dir}/configs/fragments" readonly default_config_whitelist="${script_dir}/configs/fragments/whitelist.conf" +readonly default_initramfs="${script_dir}/initramfs.cpio.gz" # GPU vendor readonly GV_INTEL="intel" readonly GV_NVIDIA="nvidia" @@ -64,6 +65,8 @@ kernel_url="" #Linux headers for GPU guest fs module building linux_headers="" +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} + packaging_scripts_dir="${script_dir}/../scripts" source "${packaging_scripts_dir}/lib.sh" @@ -269,11 +272,19 @@ get_kernel_frag_path() { fi fi - if [[ "${conf_guest}" != "" ]];then + if [ "${MEASURED_ROOTFS}" == "yes" ]; then info "Enabling config for confidential guest trust storage protection" local cryptsetup_configs="$(ls ${common_path}/confidential_containers/cryptsetup.conf)" all_configs="${all_configs} ${cryptsetup_configs}" + if [ -f "${default_initramfs}" ]; then + info "Enabling config for confidential guest measured boot" + local initramfs_configs="$(ls ${common_path}/confidential_containers/initramfs.conf)" + all_configs="${all_configs} ${initramfs_configs}" + fi + fi + + if [[ "${conf_guest}" != "" ]];then info "Enabling config for '${conf_guest}' confidential guest protection" local conf_configs="$(ls ${arch_path}/${conf_guest}/*.conf)" all_configs="${all_configs} ${conf_configs}" @@ -422,6 +433,11 @@ setup_kernel() { [ -n "${hypervisor_target}" ] || hypervisor_target="kvm" [ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}" "${kernel_path}") + if [ "${MEASURED_ROOTFS}" == "yes" ] && [ -f "${default_initramfs}" ]; then + info "Copying initramfs from: ${default_initramfs}" + cp "${default_initramfs}" ./ + fi + info "Copying config file from: ${kernel_config_path}" cp "${kernel_config_path}" ./.config make oldconfig diff --git a/tools/packaging/kernel/configs/fragments/common/confidential_containers/initramfs.conf b/tools/packaging/kernel/configs/fragments/common/confidential_containers/initramfs.conf new file mode 100644 index 000000000..35e662a65 --- /dev/null +++ b/tools/packaging/kernel/configs/fragments/common/confidential_containers/initramfs.conf @@ -0,0 +1 @@ +CONFIG_INITRAMFS_SOURCE="initramfs.cpio.gz" diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index 091f76cef..b36f06cff 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -25,6 +25,7 @@ sudo docker pull ${container_image} || \ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ "${container_image}" \ bash -c "${kernel_builder} $* setup" From f62b2670c04a2a09ca33b95b7a3365a1c22f5f92 Mon Sep 17 00:00:00 2001 From: "Wang, Arron" Date: Thu, 15 Sep 2022 10:17:55 +0800 Subject: [PATCH 37/76] config: Add root hash value and measure config to kernel params After we have a guest kernel with builtin initramfs which provide the rootfs measurement capability and Kata rootfs image with hash device, we need set related root hash value and measure config to the kernel params in kata configuration file. Fixes: #6674 Signed-off-by: Wang, Arron --- src/runtime/Makefile | 4 ++++ tools/packaging/guest-image/build_image.sh | 3 +++ .../local-build/kata-deploy-binaries.sh | 14 +++++++++++++- tools/packaging/static-build/shim-v2/build.sh | 6 ++++-- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index e937b741e..80c423612 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -133,6 +133,10 @@ FIRMWARETDVFVOLUMEPATH := FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd +ROOTMEASURECONFIG ?= "" +KERNELPARAMS += $(ROOTMEASURECONFIG) +KERNELTDXPARAMS += $(ROOTMEASURECONFIG) + # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index d602b85c0..3eac34c61 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -54,6 +54,9 @@ build_image() { IMG_OS_VERSION="${img_os_version}" \ ROOTFS_BUILD_DEST="${builddir}/rootfs-image" mv -f "kata-containers.img" "${install_dir}/${image_name}" + if [ -e "root_hash.txt" ]; then + cp root_hash.txt "${install_dir}/" + fi ( cd "${install_dir}" ln -sf "${image_name}" kata-containers.img diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index e80972f35..c09eb7501 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -485,7 +485,19 @@ install_shimv2() { export GO_VERSION export RUST_VERSION - DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" + + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + extra_opts="DEFSERVICEOFFLOAD=true" + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder//root_hash.txt") + root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" + fi + + DESTDIR="${destdir}" PREFIX="${prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" + else + DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" + fi } install_ovmf() { diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index d948ae1e8..933044564 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -21,6 +21,8 @@ DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}" +EXTRA_OPTS="${EXTRA_OPTS:-""}" + sudo docker pull ${container_image} || \ (sudo docker build \ --build-arg GO_VERSION="${GO_VERSION}" \ @@ -47,12 +49,12 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \ "${container_image}" \ - bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}" + bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch} ${EXTRA_OPTS}" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \ "${container_image}" \ - bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install" + bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" ${EXTRA_OPTS} install" for vmm in ${VMM_CONFIGS}; do config_file="${DESTDIR}/${PREFIX}/share/defaults/kata-containers/configuration-${vmm}.toml" From f6afae9c7335578c52b221920f2b3499a9d8e4c8 Mon Sep 17 00:00:00 2001 From: "Wang, Arron" Date: Sun, 21 May 2023 16:50:00 +0800 Subject: [PATCH 38/76] packaging: Add rootfs-image-tdx-tarball target Add rootfs-image-tdx target: ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh --build=rootfs-image-tdx ./opt/kata/share/kata-containers/kata-containers-tdx.img ./opt/kata/share/kata-containers/kata-ubuntu-latest-tdx.image Fixes: #6674 Signed-off-by: Wang, Arron --- .../build-kata-static-tarball-amd64.yaml | 1 + tools/packaging/guest-image/build_image.sh | 9 ++++++++- .../packaging/kata-deploy/local-build/Makefile | 4 ++++ .../local-build/kata-deploy-binaries.sh | 18 +++++++++++++++--- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index ade4eb9e7..09ddfd05d 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -32,6 +32,7 @@ jobs: - qemu-snp-experimental - qemu-tdx-experimental - rootfs-image + - rootfs-image-tdx - rootfs-initrd - rootfs-initrd-sev - shim-v2 diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index 3eac34c61..230538d1c 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -22,6 +22,7 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" export GOPATH=${GOPATH:-${HOME}/go} arch_target="$(uname -m)" +final_image_name="kata-containers" final_initrd_name="kata-containers-initrd" image_initrd_extension=".img" @@ -59,7 +60,7 @@ build_image() { fi ( cd "${install_dir}" - ln -sf "${image_name}" kata-containers.img + ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}" ) } @@ -86,6 +87,7 @@ main() { image_type=image destdir="$PWD" prefix="/opt/kata" + image_suffix="" image_initrd_suffix="" builddir="${PWD}" while getopts "h-:" opt; do @@ -113,6 +115,11 @@ main() { initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version") initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}" final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" + elif [ "${image_initrd_suffix}" == "tdx" ]; then + img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name") + img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version") + image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}" + final_image_name="${final_image_name}-${image_initrd_suffix}" fi ;; prefix=*) diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 091e3dc21..eb161e82b 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -42,6 +42,7 @@ all: serial-targets \ serial-targets: ${MAKE} -f $(MK_PATH) -j 1 V= \ rootfs-image-tarball \ + rootfs-image-tdx-tarball \ rootfs-initrd-sev-tarball \ rootfs-initrd-tarball \ cloud-hypervisor-tarball @@ -100,6 +101,9 @@ qemu-tdx-experimental-tarball: rootfs-image-tarball: ${MAKE} $@-build +rootfs-image-tdx-tarball: kernel-tdx-experimental-tarball + ${MAKE} $@-build + rootfs-initrd-sev-tarball: kernel-sev-tarball ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index c09eb7501..c65f4cd80 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -95,6 +95,7 @@ options: qemu-snp-experimental qemu-tdx-experimental rootfs-image + rootfs-image-tdx rootfs-initrd rootfs-initrd-sev shim-v2 @@ -135,8 +136,10 @@ install_cached_tarball_component() { #Install guest image install_image() { - local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-image-$(uname -m)/${cached_artifacts_path}" - local component="rootfs-image" + local image_type="${1:-"image"}" + local initrd_suffix="${2:-""}" + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-${image_type}" local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" @@ -156,7 +159,12 @@ install_image() { && return 0 info "Create image" - "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" + "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}" +} + +#Install guest image for tdx +install_image_tdx() { + install_image "image-tdx" "tdx" } #Install guest initrd @@ -310,6 +318,8 @@ install_kernel_experimental() { install_kernel_tdx_experimental() { local kernel_url="$(get_from_kata_deps assets.kernel-tdx-experimental.url)" + export MEASURED_ROOTFS=yes + install_kernel_helper \ "assets.kernel-tdx-experimental.version" \ "kernel-tdx-experimental" \ @@ -600,6 +610,8 @@ handle_build() { rootfs-image) install_image ;; + rootfs-image-tdx) install_image_tdx ;; + rootfs-initrd) install_initrd ;; rootfs-initrd-sev) install_initrd_sev ;; From f487199edff8132075cbb9d1b359880cdd0c6467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 6 Jun 2023 11:51:18 -0700 Subject: [PATCH 39/76] gha: aks: Fix argument in call to gha-run.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: #7047 Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 2 +- tests/integration/gha-run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index e9f7b55b1..a39c2bbcd 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -40,7 +40,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Download Azure CLI - run: bash tests/integration/gha-run.sh install-az-cli + run: bash tests/integration/gha-run.sh install-azure-cli - name: Log into the Azure account run: bash tests/integration/gha-run.sh login-azure diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh index 6238141dd..7544c1518 100755 --- a/tests/integration/gha-run.sh +++ b/tests/integration/gha-run.sh @@ -137,7 +137,7 @@ function main() { cleanup-snp) cleanup "snp" ;; cleanup-tdx) cleanup "tdx" ;; delete-cluster) delete_cluster ;; - *) >&2 echo "Invalid action"; exit 2 ;; + *) >&2 echo "Invalid argument"; exit 2 ;; esac } From 69668ce87f09201716ad0d42f6ad2614432008cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 6 Jun 2023 11:54:43 -0700 Subject: [PATCH 40/76] tests: gha-run: Use correct env variable for repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit s/DOCKER_IMAGE/DOCKER_REPO Signed-off-by: Aurélien Bombo --- tests/integration/gha-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh index 7544c1518..103ce2cda 100755 --- a/tests/integration/gha-run.sh +++ b/tests/integration/gha-run.sh @@ -100,9 +100,9 @@ function cleanup() { kubectl delete ${deploy_spec} kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" - cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}" || die "Failed to setup the tests image" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" kubectl apply ${cleanup_spec} sleep 180s From abae11404663ea429d1780a6d0ababc983c6cf9d Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Thu, 8 Jun 2023 08:47:08 +0800 Subject: [PATCH 41/76] runtime-rs: refactor device manager implementation The key aspects of the DM implementation refactoring as below: 1. reduce duplicated code Many scenarios have similar steps when adding devices. so to reduce duplicated code, we should create a common method abstracted and use it in various scenarios. do_handle_device: (1) new_device with DeviceConfig and return device_id; (2) try_add_device with device_id and do really add device; (3) return device info of device's info; 2. return full info of Device Trait get_device_info replace the original type DeviceConfig with full info DeviceType. 3. refactor find_device method. Fixes: #5656 Signed-off-by: alex.lyn --- .../hypervisor/src/device/device_manager.rs | 158 ++++++++++++------ .../hypervisor/src/device/driver/vfio.rs | 4 +- .../src/device/driver/vhost_user.rs | 4 +- .../src/device/driver/virtio_blk.rs | 6 +- .../crates/hypervisor/src/device/mod.rs | 2 +- .../crates/resource/src/manager_inner.rs | 38 ++--- 6 files changed, 130 insertions(+), 82 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index c633cf918..62d9f7e52 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -8,19 +8,21 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::{anyhow, Context, Result}; use kata_sys_util::rand::RandomBytes; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, RwLock}; +use super::{ + util::{get_host_path, get_virt_drive_name}, + Device, DeviceConfig, DeviceType, +}; use crate::{ BlockConfig, BlockDevice, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, }; -use super::{ - util::{get_host_path, get_virt_drive_name}, - Device, DeviceConfig, -}; pub type ArcMutexDevice = Arc>; +const DEVICE_TYPE_BLOCK: &str = "b"; + /// block_index and released_block_index are used to search an available block index /// in Sandbox. /// @@ -75,35 +77,27 @@ impl DeviceManager { }) } - pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result { - let device_id = if let Some(dev) = self.find_device(device_config).await { - dev - } else { - self.create_device(device_config) - .await - .context("failed to create device")? - }; - Ok(device_id) - } - - pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> { + async fn try_add_device(&mut self, device_id: &str) -> Result<()> { // find the device let device = self .devices .get(device_id) .context("failed to find device")?; - let mut device_guard = device.lock().await; + // attach device + let mut device_guard = device.lock().await; let result = device_guard.attach(self.hypervisor.as_ref()).await; + // handle attach error if let Err(e) = result { - if let DeviceConfig::BlockCfg(config) = device_guard.get_device_info().await { - self.shared_info.release_device_index(config.index); + if let DeviceType::Block(device) = device_guard.get_device_info().await { + self.shared_info.release_device_index(device.config.index); }; drop(device_guard); self.devices.remove(device_id); return Err(e); } + Ok(()) } @@ -120,66 +114,97 @@ impl DeviceManager { } Err(e) => Err(e), }; + + // if detach success, remove it from device manager if result.is_ok() { drop(device_guard); - // if detach success, remove it from device manager self.devices.remove(device_id); } + return result; } + Err(anyhow!( "device with specified ID hasn't been created. {}", device_id )) } - pub async fn get_device_info(&self, device_id: &str) -> Result { + async fn get_device_info(&self, device_id: &str) -> Result { if let Some(dev) = self.devices.get(device_id) { return Ok(dev.lock().await.get_device_info().await); } + Err(anyhow!( "device with specified ID hasn't been created. {}", device_id )) } - async fn find_device(&self, device_config: &DeviceConfig) -> Option { + async fn find_device(&self, host_path: String) -> Option { for (device_id, dev) in &self.devices { match dev.lock().await.get_device_info().await { - DeviceConfig::BlockCfg(config) => match device_config { - DeviceConfig::BlockCfg(ref config_new) => { - if config_new.path_on_host == config.path_on_host { - return Some(device_id.to_string()); - } + DeviceType::Block(device) => { + if device.config.path_on_host == host_path { + return Some(device_id.to_string()); } - _ => { - continue; - } - }, + } _ => { // TODO: support find other device type continue; } } } + None } - async fn create_device(&mut self, device_config: &DeviceConfig) -> Result { + fn get_dev_virt_path(&mut self, dev_type: &str) -> Result> { + let virt_path = if dev_type == DEVICE_TYPE_BLOCK { + // generate virt path + let current_index = self.shared_info.declare_device_index()?; + let drive_name = get_virt_drive_name(current_index as i32)?; + let virt_path_name = format!("/dev/{}", drive_name); + + Some((current_index, virt_path_name)) + } else { + // only dev_type is block, otherwise, it's useless. + None + }; + + Ok(virt_path) + } + + async fn new_device(&mut self, device_config: &DeviceConfig) -> Result { // device ID must be generated by manager instead of device itself // in case of ID collision let device_id = self.new_device_id()?; let dev: ArcMutexDevice = match device_config { - DeviceConfig::BlockCfg(config) => self - .create_block_device(config, device_id.clone()) - .await - .context("failed to create device")?, + DeviceConfig::BlockCfg(config) => { + // try to find the device, found and just return id. + if let Some(dev_id_matched) = self.find_device(config.path_on_host.clone()).await { + info!( + sl!(), + "device with host path:{:?} found. just return device id: {:?}", + config.path_on_host.clone(), + dev_id_matched + ); + + return Ok(dev_id_matched); + } + + self.create_block_device(config, device_id.clone()) + .await + .context("failed to create device")? + } _ => { return Err(anyhow!("invliad device type")); } }; + // register device to devices self.devices.insert(device_id.clone(), dev.clone()); + Ok(device_id) } @@ -204,17 +229,23 @@ impl DeviceManager { _ => "".to_string(), }; block_config.driver_option = block_driver; - // generate virt path - let current_index = self.shared_info.declare_device_index()?; - block_config.index = current_index; - let drive_name = get_virt_drive_name(current_index as i32)?; - block_config.virt_path = format!("/dev/{}", drive_name); - // if the path on host is empty, we need to get device host path from the device major and minor number - // Otherwise, it might be rawfile based block device, the host path is already passed from the runtime, so we don't need to do anything here - if block_config.path_on_host.is_empty() { - block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor) - .context("failed to get host path")?; + + // generate block device index and virt path + // safe here, Block device always has virt_path. + if let Some(virt_path) = self.get_dev_virt_path(DEVICE_TYPE_BLOCK)? { + block_config.index = virt_path.0; + block_config.virt_path = virt_path.1; } + + // if the path on host is empty, we need to get device host path from the device major and minor number + // Otherwise, it might be rawfile based block device, the host path is already passed from the runtime, + // so we don't need to do anything here + if block_config.path_on_host.is_empty() { + block_config.path_on_host = + get_host_path(DEVICE_TYPE_BLOCK.to_owned(), config.major, config.minor) + .context("failed to get host path")?; + } + Ok(Arc::new(Mutex::new(BlockDevice::new( device_id, block_config, @@ -237,3 +268,36 @@ impl DeviceManager { Err(anyhow!("ID are exhausted")) } } + +// Many scenarios have similar steps when adding devices. so to reduce duplicated code, +// we should create a common method abstracted and use it in various scenarios. +// do_handle_device: +// (1) new_device with DeviceConfig and return device_id; +// (2) try_add_device with device_id and do really add device; +// (3) return device info of device's info; +pub async fn do_handle_device( + d: &RwLock, + dev_info: &DeviceConfig, +) -> Result { + let device_id = d + .write() + .await + .new_device(dev_info) + .await + .context("failed to create deviec")?; + + d.write() + .await + .try_add_device(&device_id) + .await + .context("failed to add deivce")?; + + let device_info = d + .read() + .await + .get_device_info(&device_id) + .await + .context("failed to get device info")?; + + Ok(device_info) +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs index ff3a579a8..63fe40022 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs @@ -7,7 +7,7 @@ use std::{fs, path::Path, process::Command}; use crate::device::Device; -use crate::device::DeviceConfig; +use crate::device::DeviceType; use crate::Hypervisor as hypervisor; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use anyhow::anyhow; @@ -166,7 +166,7 @@ impl Device for VfioConfig { todo!() } - async fn get_device_info(&self) -> DeviceConfig { + async fn get_device_info(&self) -> DeviceType { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs index d778c4459..a105672d5 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs @@ -5,7 +5,7 @@ // use crate::device::Device; -use crate::device::DeviceConfig; +use crate::device::DeviceType; use crate::Hypervisor as hypervisor; use anyhow::Result; use async_trait::async_trait; @@ -47,7 +47,7 @@ impl Device for VhostUserConfig { todo!() } - async fn get_device_info(&self) -> DeviceConfig { + async fn get_device_info(&self) -> DeviceType { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs index 2ff98a1e7..da5d50ea7 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs @@ -6,7 +6,7 @@ pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio"; use crate::device::Device; -use crate::device::{DeviceConfig, DeviceType}; +use crate::device::DeviceType; use crate::Hypervisor as hypervisor; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -98,8 +98,8 @@ impl Device for BlockDevice { Ok(Some(self.config.index)) } - async fn get_device_info(&self) -> DeviceConfig { - DeviceConfig::BlockCfg(self.config.clone()) + async fn get_device_info(&self) -> DeviceType { + DeviceType::Block(self.clone()) } async fn increase_attach_count(&mut self) -> Result { diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs index d341d9a12..d4996a3e6 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs @@ -53,7 +53,7 @@ pub trait Device: Send + Sync { // detach is to unplug device from VM async fn detach(&mut self, h: &dyn hypervisor) -> Result>; // get_device_info returns device config - async fn get_device_info(&self) -> DeviceConfig; + async fn get_device_info(&self) -> DeviceType; // increase_attach_count is used to increase the attach count for a device // return values: // * true: no need to do real attach when current attach count is zero, skip following actions. diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 18e74dacf..0c77ec142 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -12,7 +12,10 @@ use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; use hypervisor::{ - device::{device_manager::DeviceManager, DeviceConfig}, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, DeviceType, + }, BlockConfig, Hypervisor, }; use kata_types::config::TomlConfig; @@ -266,42 +269,23 @@ impl ResourceManagerInner { for d in linux.devices.iter() { match d.r#type.as_str() { "b" => { - let device_info = DeviceConfig::BlockCfg(BlockConfig { + let dev_info = DeviceConfig::BlockCfg(BlockConfig { major: d.major, minor: d.minor, ..Default::default() }); - let device_id = self - .device_manager - .write() - .await - .new_device(&device_info) - .await - .context("failed to create deviec")?; - self.device_manager - .write() + let device_info = do_handle_device(&self.device_manager, &dev_info) .await - .try_add_device(&device_id) - .await - .context("failed to add deivce")?; - - // get complete device information - let dev_info = self - .device_manager - .read() - .await - .get_device_info(&device_id) - .await - .context("failed to get device info")?; + .context("do handle device")?; // create agent device - if let DeviceConfig::BlockCfg(config) = dev_info { + if let DeviceType::Block(device) = device_info { let agent_device = Device { - id: device_id.clone(), + id: device.device_id.clone(), container_path: d.path.clone(), - field_type: config.driver_option, - vm_path: config.virt_path, + field_type: device.config.driver_option, + vm_path: device.config.virt_path, ..Default::default() }; devices.push(agent_device); From a8e0f51c52391ba60417450378e0a82c7cb8465c Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Tue, 7 Feb 2023 11:46:56 +0800 Subject: [PATCH 42/76] dragonball: extend DeviceOpContext In order to support virtio-mem and virtio-balloon devices, we need to extend DeviceOpContext with VmConfigInfo and InstanceInfo. Fixes: #6719 Signed-off-by: Helin Guo --- src/dragonball/src/device_manager/mod.rs | 46 ++++++++++++++++++++---- src/dragonball/src/vm/mod.rs | 4 +-- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 49e9666ac..324cf41fc 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::io; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; @@ -45,9 +45,10 @@ use dbs_upcall::{ use dbs_virtio_devices::vsock::backend::VsockInnerConnector; use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::api::v1::InstanceInfo; use crate::error::StartMicroVmError; use crate::resource_manager::ResourceManager; -use crate::vm::{KernelConfigInfo, Vm}; +use crate::vm::{KernelConfigInfo, Vm, VmConfigInfo}; use crate::IoManagerCached; /// Virtual machine console device manager. @@ -248,6 +249,8 @@ pub struct DeviceOpContext { upcall_client: Option>>, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec>, + vm_config: Option, + shared_info: Arc>, } impl DeviceOpContext { @@ -257,6 +260,8 @@ impl DeviceOpContext { vm_as: Option, address_space: Option, is_hotplug: bool, + vm_config: Option, + shared_info: Arc>, ) -> Self { let irq_manager = device_mgr.irq_manager.clone(); let res_manager = device_mgr.res_manager.clone(); @@ -282,11 +287,21 @@ impl DeviceOpContext { upcall_client: None, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec::new(), + vm_config, + shared_info, } } pub(crate) fn create_boot_ctx(vm: &Vm, epoll_mgr: Option) -> Self { - Self::new(epoll_mgr, vm.device_manager(), None, None, false) + Self::new( + epoll_mgr, + vm.device_manager(), + None, + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ) } pub(crate) fn get_vm_as(&self) -> Result { @@ -386,6 +401,8 @@ impl DeviceOpContext { Some(vm_as), vm.vm_address_space().cloned(), true, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); ctx.upcall_client = vm.upcall_client().clone(); ctx @@ -463,7 +480,7 @@ pub struct DeviceManager { res_manager: Arc, vm_fd: Arc, pub(crate) logger: slog::Logger, - + pub(crate) shared_info: Arc>, pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, #[cfg(target_arch = "aarch64")] @@ -490,6 +507,7 @@ impl DeviceManager { res_manager: Arc, epoll_manager: EpollManager, logger: &slog::Logger, + shared_info: Arc>, ) -> Self { DeviceManager { io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))), @@ -498,6 +516,7 @@ impl DeviceManager { res_manager, vm_fd, logger: logger.new(slog::o!()), + shared_info, con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, @@ -636,9 +655,9 @@ impl DeviceManager { vm_as: GuestAddressSpaceImpl, epoll_mgr: EpollManager, kernel_config: &mut KernelConfigInfo, - com1_sock_path: Option, dmesg_fifo: Option>, address_space: Option<&AddressSpace>, + vm_config: &VmConfigInfo, ) -> std::result::Result<(), StartMicroVmError> { let mut ctx = DeviceOpContext::new( Some(epoll_mgr), @@ -646,8 +665,12 @@ impl DeviceManager { Some(vm_as), address_space.cloned(), false, + Some(vm_config.clone()), + self.shared_info.clone(), ); + let com1_sock_path = vm_config.serial_path.clone(); + self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; @@ -710,6 +733,8 @@ impl DeviceManager { Some(vm_as), address_space.cloned(), true, + None, + self.shared_info.clone(), ); #[cfg(feature = "virtio-blk")] @@ -1030,6 +1055,10 @@ mod tests { let epoll_manager = EpollManager::default(); let res_manager = Arc::new(ResourceManager::new(None)); let logger = slog_scope::logger().new(slog::o!()); + let shared_info = Arc::new(RwLock::new(InstanceInfo::new( + String::from("dragonball"), + String::from("1"), + ))); DeviceManager { vm_fd: Arc::clone(&vm_fd), @@ -1052,6 +1081,7 @@ mod tests { mmio_device_info: HashMap::new(), logger, + shared_info, } } } @@ -1091,7 +1121,7 @@ mod tests { }, vpmu_feature: 0, }; - vm.set_vm_config(vm_config); + vm.set_vm_config(vm_config.clone()); vm.init_guest_memory().unwrap(); vm.setup_interrupt_controller().unwrap(); let vm_as = vm.vm_as().cloned().unwrap(); @@ -1117,8 +1147,8 @@ mod tests { event_mgr.epoll_manager(), &mut cmdline, None, - None, address_space.as_ref(), + &vm_config, ) .unwrap(); let guard = mgr.io_manager.load(); @@ -1142,6 +1172,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), vm.vm_address_space().cloned(), true, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); let guest_addr = GuestAddress(0x200000000000); diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 852d78c2b..6e2545c6f 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -222,6 +222,7 @@ impl Vm { resource_manager.clone(), epoll_manager.clone(), &logger, + api_shared_info.clone(), ); Ok(Vm { @@ -453,7 +454,6 @@ impl Vm { ) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: initializing devices ..."); - let com1_sock_path = self.vm_config.serial_path.clone(); let kernel_config = self .kernel_config .as_mut() @@ -475,9 +475,9 @@ impl Vm { vm_as.clone(), epoll_manager, kernel_config, - com1_sock_path, self.dmesg_fifo.take(), self.address_space.address_space(), + &self.vm_config, )?; info!(self.logger, "VM: start devices"); From 776a15e092c2ca6ce72bf1bb55575163f4eb95bb Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Thu, 8 Jun 2023 14:51:11 +0800 Subject: [PATCH 43/76] runtime-rs: add support direct volume. As block/direct volume use similar steps of device adding, so making full use of block volume code is a better way to handle direct volume. the only different point is that direct volume will use DirectVolume and get_volume_mount_info to parse mountinfo.json from the direct volume path. That's to say, direct volume needs the help of `kata-ctl direct-volume ...`. Details seen at Advanced Topics: [How to run Kata Containers with kinds of Block Volumes] docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md Fixes: #5656 Signed-off-by: alex.lyn --- docs/how-to/README.md | 1 + ...-containers-with-kinds-of-Block-Volumes.md | 78 ++++++++ src/runtime-rs/Cargo.lock | 90 ++++++++- .../resource/src/rootfs/block_rootfs.rs | 35 ++-- .../resource/src/volume/block_volume.rs | 181 +++++++++++------- .../crates/resource/src/volume/mod.rs | 14 +- .../crates/resource/src/volume/utils.rs | 65 +++++++ 7 files changed, 364 insertions(+), 100 deletions(-) create mode 100644 docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md create mode 100644 src/runtime-rs/crates/resource/src/volume/utils.rs diff --git a/docs/how-to/README.md b/docs/how-to/README.md index df6a060fc..874efc01a 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -45,3 +45,4 @@ - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md) - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md) - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md) +- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md) \ No newline at end of file diff --git a/docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md b/docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md new file mode 100644 index 000000000..d4fd048ec --- /dev/null +++ b/docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md @@ -0,0 +1,78 @@ +# A new way for Kata Containers to use Kinds of Block Volumes + +> **Note:** This guide is only available for runtime-rs with default Hypervisor Dragonball. +> Now, other hypervisors are still ongoing, and it'll be updated when they're ready. + + +## Background + +Currently, there is no widely applicable and convenient method available for users to use some kinds of backend storages, such as File on host based block volume, SPDK based volume or VFIO device based volume for Kata Containers, so we adopt [Proposal: Direct Block Device Assignment](https://github.com/kata-containers/kata-containers/blob/main/docs/design/direct-blk-device-assignment.md) to address it. + +## Solution + +According to the proposal, it requires to use the `kata-ctl direct-volume` command to add a direct assigned block volume device to the Kata Containers runtime. + +And then with the help of method [get_volume_mount_info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L95), get information from JSON file: `(mountinfo.json)` and parse them into structure [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70) which is used to save device-related information. + +We only fill the `mountinfo.json`, such as `device` ,`volume_type`, `fs_type`, `metadata` and `options`, which correspond to the fields in [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70), to describe a device. + +The JSON file `mountinfo.json` placed in a sub-path `/kubelet/kata-test-vol-001/volume001` which under fixed path `/run/kata-containers/shared/direct-volumes/`. +And the full path looks like: `/run/kata-containers/shared/direct-volumes/kubelet/kata-test-vol-001/volume001`, But for some security reasons. it is +encoded as `/run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx`. + +Finally, when running a Kata Containers witch `ctr run --mount type=X, src=Y, dst=Z,,options=rbind:rw`, the `type=X` should be specified a proprietary type specifically designed for some kind of volume. + +Now, supported types: + +- `directvol` for direct volume +- `spdkvol` for SPDK volume (TBD) +- `vfiovol` for VFIO device based volume (TBD) + + +## Setup Device and Run a Kata-Containers + +### Direct Block Device Based Volume + +#### create raw block based backend storage + +> **Tips:** raw block based backend storage MUST be formatted with `mkfs`. + +```bash +$ sudo dd if=/dev/zero of=/tmp/stor/rawdisk01.20g bs=1M count=20480 +$ sudo mkfs.ext4 /tmp/stor/rawdisk01.20g +``` + +#### setup direct block device for kata-containers + +```json +{ + "device": "/tmp/stor/rawdisk01.20g", + "volume_type": "directvol", + "fs_type": "ext4", + "metadata":"{}", + "options": [] +} +``` + +```bash +$ sudo ./kata-ctl direct-volume add /kubelet/kata-direct-vol-002/directvol002 "{\"device\": \"/tmp/stor/rawdisk01.20g\", \"volume_type\": \"directvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}" +$# /kubelet/kata-direct-vol-002/directvol002 <==> /run/kata-containers/shared/direct-volumes/W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx +$ cat W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx/mountInfo.json +{"volume_type":"directvol","device":"/tmp/stor/rawdisk01.20g","fs_type":"ext4","metadata":{},"options":[]} +``` + +#### Run a Kata container with direct block device volume + +```bash +$ # type=disrectvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw +$sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=directvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw "$image" kata-direct-vol-xx05302045 /bin/bash +``` + + +### SPDK Device Based Volume + +TBD + +### VFIO Device Based Volume + +TBD \ No newline at end of file diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 9759e0e54..0b5cd7ca9 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -241,6 +241,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -827,6 +838,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dlv-list" version = "0.3.0" @@ -1212,6 +1244,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -1619,6 +1660,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -1846,7 +1888,16 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ "libc", ] @@ -2434,6 +2485,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.8", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.7.1" @@ -2845,6 +2907,19 @@ dependencies = [ "slog-scope", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.20", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -2963,6 +3038,17 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.2.0" @@ -3043,6 +3129,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs index 0e161fcb3..bcb31d957 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs @@ -10,7 +10,10 @@ use agent::Storage; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use hypervisor::{ - device::{device_manager::DeviceManager, DeviceConfig}, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, DeviceType, + }, BlockConfig, }; use kata_types::mount::Mount; @@ -46,18 +49,10 @@ impl BlockRootfs { ..Default::default() }; - let device_id = d - .write() + // create and insert block device into Kata VM + let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone())) .await - .new_device(&DeviceConfig::BlockCfg(block_device_config.clone())) - .await - .context("failed to create deviec")?; - - d.write() - .await - .try_add_device(device_id.as_str()) - .await - .context("failed to add deivce")?; + .context("do handle device failed.")?; let mut storage = Storage { fs_type: rootfs.fs_type.clone(), @@ -66,17 +61,11 @@ impl BlockRootfs { ..Default::default() }; - // get complete device information - let dev_info = d - .read() - .await - .get_device_info(device_id.as_str()) - .await - .context("failed to get device info")?; - - if let DeviceConfig::BlockCfg(config) = dev_info { - storage.driver = config.driver_option; - storage.source = config.virt_path; + let mut device_id: String = "".to_owned(); + if let DeviceType::Block(device) = device_info { + storage.driver = device.config.driver_option; + storage.source = device.config.virt_path; + device_id = device.device_id; } Ok(Self { diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs index 14f09fc81..ad1c01f17 100644 --- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs @@ -4,29 +4,32 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::Result; +use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use std::{collections::HashMap, fs, path::Path}; +use nix::sys::{stat, stat::SFlag}; +use tokio::sync::RwLock; -use crate::share_fs::{do_get_guest_path, do_get_host_path}; - -use super::{share_fs_volume::generate_mount_path, Volume}; -use agent::Storage; -use anyhow::{anyhow, Context}; +use super::Volume; +use crate::volume::utils::{ + generate_shared_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, KATA_DIRECT_VOLUME_TYPE, + KATA_MOUNT_BIND_TYPE, +}; use hypervisor::{ - device::{device_manager::DeviceManager, DeviceConfig}, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, DeviceType, + }, BlockConfig, }; -use nix::sys::stat::{self, SFlag}; -use tokio::sync::RwLock; -#[derive(Debug)] + +#[derive(Clone)] pub(crate) struct BlockVolume { storage: Option, mount: oci::Mount, device_id: String, } -/// BlockVolume: block device volume +/// BlockVolume for bind-mount block volume and direct block volume impl BlockVolume { pub(crate) async fn new( d: &RwLock, @@ -35,54 +38,71 @@ impl BlockVolume { cid: &str, sid: &str, ) -> Result { - let fstat = stat::stat(m.source.as_str()).context(format!("stat {}", m.source))?; - info!(sl!(), "device stat: {:?}", fstat); - let mut options = HashMap::new(); - if read_only { - options.insert("read_only".to_string(), "true".to_string()); - } + let mnt_src: &str = &m.source; + // default block device fs type: ext4. + let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string(); - let block_device_config = &mut BlockConfig { - major: stat::major(fstat.st_rdev) as i64, - minor: stat::minor(fstat.st_rdev) as i64, - ..Default::default() + let block_device_config = match m.r#type.as_str() { + KATA_MOUNT_BIND_TYPE => { + let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?; + + BlockConfig { + major: stat::major(fstat.st_rdev) as i64, + minor: stat::minor(fstat.st_rdev) as i64, + ..Default::default() + } + } + KATA_DIRECT_VOLUME_TYPE => { + // get volume mountinfo from mountinfo.json + let v = volume_mount_info(mnt_src) + .context("deserde information from mountinfo.json")?; + // check volume type + if v.volume_type != KATA_DIRECT_VOLUME_TYPE { + return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); + } + + let fstat = stat::stat(v.device.as_str()) + .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; + if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG + && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK + { + return Err(anyhow!( + "invalid volume device {:?} for volume type {:?}", + v.device, + v.volume_type + )); + } + + blk_dev_fstype = v.fs_type.clone(); + + BlockConfig { + path_on_host: v.device, + ..Default::default() + } + } + _ => { + return Err(anyhow!( + "unsupport direct block volume r#type: {:?}", + m.r#type.as_str() + )) + } }; - let device_id = d - .write() + // create and insert block device into Kata VM + let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone())) .await - .new_device(&DeviceConfig::BlockCfg(block_device_config.clone())) - .await - .context("failed to create deviec")?; + .context("do handle device failed.")?; - d.write() + // generate host guest shared path + let guest_path = generate_shared_path(m.destination.clone(), read_only, cid, sid) .await - .try_add_device(device_id.as_str()) - .await - .context("failed to add deivce")?; - - let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); - let file_name = generate_mount_path(cid, file_name); - let guest_path = do_get_guest_path(&file_name, cid, true, false); - let host_path = do_get_host_path(&file_name, sid, cid, true, read_only); - fs::create_dir_all(&host_path) - .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?; - - // get complete device information - let dev_info = d - .read() - .await - .get_device_info(&device_id) - .await - .context("failed to get device info")?; + .context("generate host-guest shared path failed")?; // storage - let mut storage = Storage::default(); - - if let DeviceConfig::BlockCfg(config) = dev_info { - storage.driver = config.driver_option; - storage.source = config.virt_path; - } + let mut storage = agent::Storage { + mount_point: guest_path.clone(), + ..Default::default() + }; storage.options = if read_only { vec!["ro".to_string()] @@ -90,21 +110,32 @@ impl BlockVolume { Vec::new() }; - storage.mount_point = guest_path.clone(); - - // If the volume had specified the filesystem type, use it. Otherwise, set it - // to ext4 since but right now we only support it. - if m.r#type != "bind" { - storage.fs_type = m.r#type.clone(); - } else { - storage.fs_type = "ext4".to_string(); + // As the true Block Device wrapped in DeviceType, we need to + // get it out from the wrapper, and the device_id will be for + // BlockVolume. + // safe here, device_info is correct and only unwrap it. + let mut device_id = String::new(); + if let DeviceType::Block(device) = device_info { + // blk, mmioblk + storage.driver = device.config.driver_option; + // /dev/vdX + storage.source = device.config.virt_path; + device_id = device.device_id; + } + + // In some case, dest is device /dev/xxx + if m.destination.clone().starts_with("/dev") { + storage.fs_type = "bind".to_string(); + storage.options.append(&mut m.options.clone()); + } else { + // usually, the dest is directory. + storage.fs_type = blk_dev_fstype; } - // mount let mount = oci::Mount { destination: m.destination.clone(), - r#type: m.r#type.clone(), - source: guest_path.clone(), + r#type: storage.fs_type.clone(), + source: guest_path, options: m.options.clone(), }; @@ -128,6 +159,7 @@ impl Volume for BlockVolume { } else { vec![] }; + Ok(s) } @@ -144,13 +176,22 @@ impl Volume for BlockVolume { } } -pub(crate) fn is_block_volume(m: &oci::Mount) -> bool { - if m.r#type != "bind" { - return false; +pub(crate) fn is_block_volume(m: &oci::Mount) -> Result { + let vol_types = vec![KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; + if !vol_types.contains(&m.r#type.as_str()) { + return Ok(false); } - if let Ok(fstat) = stat::stat(m.source.as_str()).context(format!("stat {}", m.source)) { - info!(sl!(), "device stat: {:?}", fstat); - return SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK; + + let fstat = + stat::stat(m.source.as_str()).context(format!("stat mount source {} failed.", m.source))?; + let s_flag = SFlag::from_bits_truncate(fstat.st_mode); + + match m.r#type.as_str() { + // case: mount bind and block device + KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true), + // case: directvol and directory + KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true), + // else: unsupported or todo for other volume type. + _ => Ok(false), } - false } diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 52b826b2d..6fc6e3bc8 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -9,16 +9,18 @@ mod default_volume; pub mod hugepage; mod share_fs_volume; mod shm_volume; -use async_trait::async_trait; +pub mod utils; -use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; -use agent::Agent; -use anyhow::{Context, Result}; -use hypervisor::device::device_manager::DeviceManager; use std::{sync::Arc, vec::Vec}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; use tokio::sync::RwLock; use self::hugepage::{get_huge_page_limits_map, get_huge_page_option}; +use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; +use agent::Agent; +use hypervisor::device::device_manager::DeviceManager; const BIND: &str = "bind"; @@ -66,7 +68,7 @@ impl VolumeResource { shm_volume::ShmVolume::new(m, shm_size) .with_context(|| format!("new shm volume {:?}", m))?, ) - } else if is_block_volume(m) { + } else if is_block_volume(m).context("block volume type")? { // handle block volume Arc::new( block_volume::BlockVolume::new(d, m, read_only, cid, sid) diff --git a/src/runtime-rs/crates/resource/src/volume/utils.rs b/src/runtime-rs/crates/resource/src/volume/utils.rs new file mode 100644 index 000000000..892724e1e --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/utils.rs @@ -0,0 +1,65 @@ +// Copyright (c) 2022-2023 Alibaba Cloud +// Copyright (c) 2022-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{fs, path::Path}; + +use anyhow::{anyhow, Context, Result}; + +use crate::{ + share_fs::{do_get_guest_path, do_get_host_path}, + volume::share_fs_volume::generate_mount_path, +}; +use kata_sys_util::eother; +use kata_types::mount::{get_volume_mount_info, DirectVolumeMountInfo}; + +pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4"; +pub const KATA_MOUNT_BIND_TYPE: &str = "bind"; +pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol"; +pub const KATA_VFIO_VOLUME_TYPE: &str = "vfiovol"; +pub const KATA_SPDK_VOLUME_TYPE: &str = "spdkvol"; + +// volume mount info load infomation from mountinfo.json +pub fn volume_mount_info(volume_path: &str) -> Result { + get_volume_mount_info(volume_path) +} + +pub fn get_file_name>(src: P) -> Result { + let file_name = src + .as_ref() + .file_name() + .map(|v| v.to_os_string()) + .ok_or_else(|| { + eother!( + "failed to get file name of path {}", + src.as_ref().to_string_lossy() + ) + })? + .into_string() + .map_err(|e| anyhow!("failed to convert to string {:?}", e))?; + + Ok(file_name) +} + +pub(crate) async fn generate_shared_path( + dest: String, + read_only: bool, + cid: &str, + sid: &str, +) -> Result { + let file_name = get_file_name(&dest).context("failed to get file name.")?; + let mount_name = generate_mount_path(cid, file_name.as_str()); + let guest_path = do_get_guest_path(&mount_name, cid, true, false); + let host_path = do_get_host_path(&mount_name, sid, cid, true, read_only); + + if dest.starts_with("/dev") { + fs::File::create(&host_path).context(format!("failed to create file {:?}", &host_path))?; + } else { + std::fs::create_dir_all(&host_path) + .map_err(|e| anyhow!("failed to create dir {}: {:?}", host_path, e))?; + } + + Ok(guest_path) +} From 7ed9494973b8b7e4f393f38b00fdd7fb18082165 Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Wed, 31 May 2023 10:13:09 +0800 Subject: [PATCH 44/76] dragonball: introduce virtio-mem device We introduce virtio-mem device to support memory resize. virtio-mem device could hot-plug more memory blocks to guest and could also hot-unplug them from guest. Fixes: #6719 Signed-off-by: Helin Guo --- src/dragonball/Cargo.lock | 15 +- src/dragonball/Cargo.toml | 7 +- src/dragonball/src/api/v1/vmm_action.rs | 83 ++ .../src/device_manager/blk_dev_mgr.rs | 4 + .../src/device_manager/mem_dev_mgr.rs | 733 ++++++++++++++++++ .../device_manager/memory_region_handler.rs | 1 + src/dragonball/src/device_manager/mod.rs | 52 ++ 7 files changed, 885 insertions(+), 10 deletions(-) create mode 100644 src/dragonball/src/device_manager/mem_dev_mgr.rs diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock index f2e087213..3cc177fe1 100644 --- a/src/dragonball/Cargo.lock +++ b/src/dragonball/Cargo.lock @@ -209,11 +209,12 @@ dependencies = [ [[package]] name = "dbs-address-space" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc" +checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322" dependencies = [ "arc-swap", + "lazy_static", "libc", "nix 0.23.2", "thiserror", @@ -300,9 +301,9 @@ dependencies = [ [[package]] name = "dbs-upcall" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f" +checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb" dependencies = [ "anyhow", "dbs-utils", @@ -330,9 +331,9 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668" +checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8" dependencies = [ "byteorder", "caps", @@ -346,7 +347,7 @@ dependencies = [ "kvm-ioctls", "libc", "log", - "nix 0.23.2", + "nix 0.24.3", "nydus-api", "nydus-blobfs", "nydus-rafs", diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index f70463266..0c94530ca 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -12,16 +12,16 @@ edition = "2018" [dependencies] arc-swap = "1.5.0" bytes = "1.1.0" -dbs-address-space = "0.2.0" +dbs-address-space = "0.3.0" dbs-allocator = "0.1.0" dbs-arch = "0.2.0" dbs-boot = "0.4.0" dbs-device = "0.2.0" dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] } dbs-legacy-devices = "0.1.0" -dbs-upcall = { version = "0.2.0", optional = true } +dbs-upcall = { version = "0.3.0", optional = true } dbs-utils = "0.2.0" -dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] } +dbs-virtio-devices = { version = "0.3.1", optional = true, features = ["virtio-mmio"] } kvm-bindings = "0.6.0" kvm-ioctls = "0.12.0" lazy_static = "1.2" @@ -55,3 +55,4 @@ virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] # virtio-fs only work on atomic-guest-memory virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] +virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"] diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 6ad7bfcb7..6243b82a3 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -27,6 +27,8 @@ pub use crate::device_manager::blk_dev_mgr::{ pub use crate::device_manager::fs_dev_mgr::{ FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo, }; +#[cfg(feature = "virtio-mem")] +pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError}; #[cfg(feature = "virtio-net")] pub use crate::device_manager::virtio_net_dev_mgr::{ VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, @@ -97,6 +99,15 @@ pub enum VmmActionError { /// The action `ResizeVcpu` Failed #[error("vcpu resize error : {0}")] ResizeVcpu(#[source] VcpuResizeError), + + /// Cannot access address space. + #[error("Cannot access address space.")] + AddressSpaceNotInitialized, + + #[cfg(feature = "virtio-mem")] + /// Mem device related errors. + #[error("virtio-mem device error: {0}")] + Mem(#[source] MemDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -172,6 +183,10 @@ pub enum VmmAction { #[cfg(feature = "hotplug")] /// Resize Vcpu number in the guest. ResizeVcpu(VcpuResizeInfo), + + #[cfg(feature = "virtio-mem")] + /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input. + InsertMemDevice(MemDeviceConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -274,6 +289,8 @@ impl VmmService { } #[cfg(feature = "hotplug")] VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg), + #[cfg(feature = "virtio-mem")] + VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg), }; debug!("send vmm response: {:?}", response); @@ -648,6 +665,32 @@ impl VmmService { Ok(VmmData::Empty) } + + #[cfg(feature = "virtio-mem")] + fn add_mem_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: MemDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; + + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady + } else { + VmmActionError::StartMicroVm(e) + } + })?; + + vm.device_manager_mut() + .mem_manager + .insert_or_update_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Mem) + } } fn handle_cpu_topology( @@ -1456,4 +1499,44 @@ mod tests { t.check_request(); } } + + #[cfg(feature = "virtio-mem")] + #[test] + fn test_vmm_action_insert_mem_device() { + skip_if_not_root!(); + + let tests = &mut [ + // hotplug unready + TestData::new( + VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()), + InstanceState::Running, + &|result| { + assert!(matches!( + result, + Err(VmmActionError::StartMicroVm( + StartMicroVmError::UpcallMissVsock + )) + )); + let err_string = format!("{}", result.unwrap_err()); + let expected_err = String::from( + "failed to boot the VM: \ + the upcall client needs a virtio-vsock device for communication", + ); + assert_eq!(err_string, expected_err); + }, + ), + // success + TestData::new( + VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()), + InstanceState::Uninitialized, + &|result| { + assert!(result.is_ok()); + }, + ), + ]; + + for t in tests.iter_mut() { + t.check_request(); + } + } } diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index f7cdfa9b0..0fe10cf24 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -871,6 +871,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), None, false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); let dummy_file = TempFile::new().unwrap(); @@ -907,6 +909,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), None, false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); vm.device_manager_mut() diff --git a/src/dragonball/src/device_manager/mem_dev_mgr.rs b/src/dragonball/src/device_manager/mem_dev_mgr.rs new file mode 100644 index 000000000..2bb68ae80 --- /dev/null +++ b/src/dragonball/src/device_manager/mem_dev_mgr.rs @@ -0,0 +1,733 @@ +// Copyright 2020 Alibaba Cloud. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::io; +use std::sync::{Arc, Mutex}; + +use dbs_address_space::{ + AddressSpace, AddressSpaceError, AddressSpaceRegion, MPOL_MF_MOVE, MPOL_PREFERRED, USABLE_END, +}; +use dbs_utils::epoll_manager::EpollManager; +use dbs_virtio_devices as virtio; +use kvm_bindings::kvm_userspace_memory_region; +use kvm_ioctls::VmFd; +use nix::sys::mman; +use serde_derive::{Deserialize, Serialize}; +use slog::{debug, error, info, warn}; +use virtio::mem::{Mem, MemRegionFactory}; +use virtio::Error as VirtIoError; +use vm_memory::{ + Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestRegionMmap, GuestUsize, MmapRegion, +}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::DbsMmioV2Device; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::vm::VmConfigInfo; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = false; + +const HUGE_PAGE_2M: usize = 0x200000; + +// max numa node ids on host +const MAX_NODE: u32 = 64; + +/// Errors associated with `MemDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum MemDeviceError { + /// The mem device was already used. + #[error("the virtio-mem ID was already added to a different device")] + MemDeviceAlreadyExists, + + /// Cannot perform the requested operation after booting the microVM. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// insert mem device error + #[error("cannot add virtio-mem device, {0}")] + InsertDeviceFailed(#[source] DeviceMgrError), + + /// create mem device error + #[error("cannot create virito-mem device, {0}")] + CreateMemDevice(#[source] DeviceMgrError), + + /// create mmio device error + #[error("cannot create virito-mem mmio device, {0}")] + CreateMmioDevice(#[source] DeviceMgrError), + + /// resize mem device error + #[error("failure while resizing virtio-mem device, {0}")] + ResizeFailed(#[source] VirtIoError), + + /// mem device does not exist + #[error("mem device does not exist")] + DeviceNotExist, + + /// address space region error + #[error("address space region error, {0}")] + AddressSpaceRegion(#[source] AddressSpaceError), + + /// Cannot initialize a mem device or add a device to the MMIO Bus. + #[error("failure while registering mem device: {0}")] + RegisterMemDevice(#[source] DeviceMgrError), + + /// The mem device id doesn't exist. + #[error("invalid mem device id '{0}'")] + InvalidDeviceId(String), + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a virtio-mem device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +pub struct MemDeviceConfigInfo { + /// Unique identifier of the pmem device + pub mem_id: String, + /// Memory size mib + pub size_mib: u64, + /// Memory capacity mib + pub capacity_mib: u64, + /// Use multi_region or not + pub multi_region: bool, + /// host numa node id + pub host_numa_node_id: Option, + /// guest numa node id + pub guest_numa_node_id: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl ConfigItem for MemDeviceConfigInfo { + type Err = MemDeviceError; + + fn id(&self) -> &str { + &self.mem_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> { + if self.mem_id.as_str() == other.mem_id.as_str() { + Err(MemDeviceError::MemDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Mem Device Info +pub type MemDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for MemDeviceInfo { + type Err = MemDeviceError; + + fn id(&self) -> &str { + &self.config.mem_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> { + if self.config.mem_id.as_str() == other.config.mem_id.as_str() { + Err(MemDeviceError::MemDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Mem Devices Configs +#[derive(Clone)] +pub struct MemDeviceMgr { + /// A list of `MemDeviceConfig` objects. + info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl MemDeviceMgr { + /// Inserts `mem_cfg` in the virtio-mem device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + pub fn insert_or_update_device( + &mut self, + mut ctx: DeviceOpContext, + mem_cfg: MemDeviceConfigInfo, + ) -> std::result::Result<(), MemDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + error!(ctx.logger(), "hotplug feature has been disabled."; + "subsystem" => "virito-mem"); + return Err(MemDeviceError::UpdateNotAllowedPostBoot); + } + + let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?; + + // If the id of the drive already exists in the list, the operation is update. + if let Some(index) = self.get_index_of_mem_dev(&mem_cfg.mem_id) { + // Update an existing memory device + if ctx.is_hotplug { + info!( + ctx.logger(), + "update memory device: {}, size: 0x{:x}MB.", + mem_cfg.mem_id, + mem_cfg.size_mib; + "subsystem" => "virito-mem" + ); + self.update_memory_size(index, mem_cfg.size_mib)?; + } + self.info_list.insert_or_update(&mem_cfg)?; + } else { + // Create a new memory device + if !ctx.is_hotplug { + self.info_list.insert_or_update(&mem_cfg)?; + return Ok(()); + } + + info!( + ctx.logger(), + "hot-add memory device: {}, size: 0x{:x}MB.", mem_cfg.mem_id, mem_cfg.size_mib; + "subsystem" => "virito-mem" + ); + + let device = Self::create_memory_device(&mem_cfg, &ctx, &epoll_mgr) + .map_err(MemDeviceError::CreateMemDevice)?; + let mmio_device = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + &mut ctx, + mem_cfg.use_shared_irq.unwrap_or(self.use_shared_irq), + mem_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(MemDeviceError::CreateMmioDevice)?; + + #[cfg(not(test))] + ctx.insert_hotplug_mmio_device(&mmio_device, None) + .map_err(|e| { + error!( + ctx.logger(), + "failed to hot-add virtio-mem device {}, {}", &mem_cfg.mem_id, e; + "subsystem" => "virito-mem" + ); + MemDeviceError::InsertDeviceFailed(e) + })?; + + let index = self.info_list.insert_or_update(&mem_cfg)?; + self.info_list[index].set_device(mmio_device); + } + + Ok(()) + } + + /// Attaches all virtio-mem devices from the MemDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), MemDeviceError> { + let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?; + + for info in self.info_list.iter_mut() { + let config = &info.config; + info!( + ctx.logger(), + "attach virtio-mem device {}, size 0x{:x}.", config.mem_id, config.size_mib; + "subsystem" => "virito-mem" + ); + // Ignore virtio-mem device with zero memory capacity. + if config.size_mib == 0 { + debug!( + ctx.logger(), + "ignore zero-sizing memory device {}.", config.mem_id; + "subsystem" => "virito-mem" + ); + continue; + } + + let device = Self::create_memory_device(config, ctx, &epoll_mgr) + .map_err(MemDeviceError::CreateMemDevice)?; + let mmio_device = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + ctx, + config.use_shared_irq.unwrap_or(self.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(MemDeviceError::RegisterMemDevice)?; + + info.set_device(mmio_device); + } + + Ok(()) + } + + fn get_index_of_mem_dev(&self, mem_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.mem_id.eq(mem_id)) + } + + fn create_memory_device( + config: &MemDeviceConfigInfo, + ctx: &DeviceOpContext, + epoll_mgr: &EpollManager, + ) -> std::result::Result, DeviceMgrError> { + let factory = Arc::new(Mutex::new(MemoryRegionFactory::new( + ctx, + config.mem_id.clone(), + config.host_numa_node_id, + )?)); + + let mut capacity_mib = config.capacity_mib; + if capacity_mib == 0 { + capacity_mib = *USABLE_END >> 20; + } + // get boot memory size for calculate alignment + let boot_mem_size = { + let boot_size = (ctx.get_vm_config()?.mem_size_mib << 20) as u64; + // increase 1G memory because of avoiding mmio hole + match boot_size { + x if x > dbs_boot::layout::MMIO_LOW_START => x + (1 << 30), + _ => boot_size, + } + }; + + virtio::mem::Mem::new( + config.mem_id.clone(), + capacity_mib, + config.size_mib, + config.multi_region, + config.guest_numa_node_id, + epoll_mgr.clone(), + factory, + boot_mem_size, + ) + .map_err(DeviceMgrError::Virtio) + } + + /// Removes all virtio-mem devices + pub fn remove_devices(&self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + for info in self.info_list.iter() { + if let Some(device) = &info.device { + DeviceManager::destroy_mmio_virtio_device(device.clone(), ctx)?; + } + } + + Ok(()) + } + + fn update_memory_size( + &self, + index: usize, + size_mib: u64, + ) -> std::result::Result<(), MemDeviceError> { + let device = self.info_list[index] + .device + .as_ref() + .ok_or_else(|| MemDeviceError::DeviceNotExist)?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(mem_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return mem_dev + .set_requested_size(size_mib) + .map_err(MemDeviceError::ResizeFailed); + } + } + Ok(()) + } +} + +impl Default for MemDeviceMgr { + /// Create a new `MemDeviceMgr` object.. + fn default() -> Self { + MemDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +struct MemoryRegionFactory { + mem_id: String, + vm_as: GuestAddressSpaceImpl, + address_space: AddressSpace, + vm_config: VmConfigInfo, + vm_fd: Arc, + logger: Arc, + host_numa_node_id: Option, + instance_id: String, +} + +impl MemoryRegionFactory { + fn new( + ctx: &DeviceOpContext, + mem_id: String, + host_numa_node_id: Option, + ) -> Result { + let vm_as = ctx.get_vm_as()?; + let address_space = ctx.get_address_space()?; + let vm_config = ctx.get_vm_config()?; + let logger = Arc::new(ctx.logger().new(slog::o!())); + + let shared_info = ctx.shared_info.read().unwrap(); + let instance_id = shared_info.id.clone(); + + Ok(MemoryRegionFactory { + mem_id, + vm_as, + address_space, + vm_config, + vm_fd: ctx.vm_fd.clone(), + logger, + host_numa_node_id, + instance_id, + }) + } + + fn configure_anon_mem(&self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> { + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_DONTFORK, + ) + } + .map_err(VirtIoError::Madvise)?; + + Ok(()) + } + + fn configure_numa(&self, mmap_reg: &MmapRegion, node_id: u32) -> Result<(), VirtIoError> { + let nodemask = 1_u64 + .checked_shl(node_id) + .ok_or(VirtIoError::InvalidInput)?; + let res = unsafe { + libc::syscall( + libc::SYS_mbind, + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + MPOL_PREFERRED, + &nodemask as *const u64, + MAX_NODE, + MPOL_MF_MOVE, + ) + }; + if res < 0 { + warn!( + self.logger, + "failed to mbind memory to host_numa_node_id {}: this may affect performance", + node_id; + "subsystem" => "virito-mem" + ); + } + Ok(()) + } + + fn configure_thp(&mut self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> { + debug!( + self.logger, + "Setting MADV_HUGEPAGE on AddressSpaceRegion addr {:x?} len {:x?}", + mmap_reg.as_ptr(), + mmap_reg.size(); + "subsystem" => "virito-mem" + ); + + // Safe because we just create the MmapRegion + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_HUGEPAGE, + ) + } + .map_err(VirtIoError::Madvise)?; + + Ok(()) + } + + fn map_to_kvm( + &mut self, + slot: u32, + reg: &Arc, + mmap_reg: &MmapRegion, + ) -> Result<(), VirtIoError> { + let host_addr = mmap_reg.as_ptr() as u64; + + let flags = 0u32; + + let mem_region = kvm_userspace_memory_region { + slot, + guest_phys_addr: reg.start_addr().raw_value(), + memory_size: reg.len(), + userspace_addr: host_addr, + flags, + }; + + // Safe because the user mem region is just created, and kvm slot is allocated + // by resource allocator. + unsafe { self.vm_fd.set_user_memory_region(mem_region) } + .map_err(VirtIoError::SetUserMemoryRegion)?; + + Ok(()) + } +} + +impl MemRegionFactory for MemoryRegionFactory { + fn create_region( + &mut self, + guest_addr: GuestAddress, + region_len: GuestUsize, + kvm_slot: u32, + ) -> std::result::Result, VirtIoError> { + // create address space region + let mem_type = self.vm_config.mem_type.as_str(); + let mut mem_file_path = self.vm_config.mem_file_path.clone(); + let mem_file_name = format!( + "/virtiomem_{}_{}", + self.instance_id.as_str(), + self.mem_id.as_str() + ); + mem_file_path.push_str(mem_file_name.as_str()); + let region = Arc::new( + AddressSpaceRegion::create_default_memory_region( + guest_addr, + region_len, + self.host_numa_node_id, + mem_type, + mem_file_path.as_str(), + false, + true, + ) + .map_err(|e| { + error!(self.logger, "failed to insert address space region: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + guest_addr.0, region_len + ), + )) + })?, + ); + info!( + self.logger, + "VM: mem_type: {} mem_file_path: {}, numa_node_id: {:?} file_offset: {:?}", + mem_type, + mem_file_path, + self.host_numa_node_id, + region.file_offset(); + "subsystem" => "virito-mem" + ); + + let mmap_region = MmapRegion::build( + region.file_offset().cloned(), + region_len as usize, + region.prot_flags(), + region.perm_flags(), + ) + .map_err(VirtIoError::NewMmapRegion)?; + let host_addr: u64 = mmap_region.as_ptr() as u64; + + // thp + if mem_type == "hugeanon" || mem_type == "hugeshmem" { + self.configure_thp(&mmap_region)?; + } + + // Handle numa + if let Some(numa_node_id) = self.host_numa_node_id { + self.configure_numa(&mmap_region, numa_node_id)?; + } + + // add to guest memory mapping + self.map_to_kvm(kvm_slot, ®ion, &mmap_region)?; + + info!( + self.logger, + "kvm set user memory region: slot: {}, flags: {}, guest_phys_addr: {:X}, memory_size: {}, userspace_addr: {:X}", + kvm_slot, + 0, + guest_addr.raw_value(), + region_len, + host_addr; + "subsystem" => "virito-mem" + ); + + // All value should be valid. + let memory_region = Arc::new( + GuestRegionMmap::new(mmap_region, guest_addr).map_err(VirtIoError::InsertMmap)?, + ); + + let vm_as_new = self + .vm_as + .memory() + .insert_region(memory_region.clone()) + .map_err(VirtIoError::InsertMmap)?; + self.vm_as.lock().unwrap().replace(vm_as_new); + self.address_space.insert_region(region).map_err(|e| { + error!(self.logger, "failed to insert address space region: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + guest_addr.0, region_len + ), + )) + })?; + + Ok(memory_region) + } + + fn restore_region_addr( + &self, + guest_addr: GuestAddress, + ) -> std::result::Result<*mut u8, VirtIoError> { + let memory = self.vm_as.memory(); + // NOTE: We can't clone `GuestRegionMmap` reference directly!!! + // + // Since an important role of the member `mapping` (type is + // `MmapRegion`) in `GuestRegionMmap` is to mmap the memory during + // construction and munmap the memory during drop. However, when the + // life time of cloned data is over, the drop operation will be + // performed, which will munmap the origional mmap memory, which will + // cause some memory in dragonall to be inaccessable. And remember the + // data structure that was cloned is still alive now, when its life time + // is over, it will perform the munmap operation again, which will cause + // a memory exception! + memory + .get_host_address(guest_addr) + .map_err(VirtIoError::GuestMemory) + } + + fn get_host_numa_node_id(&self) -> Option { + self.host_numa_node_id + } + + fn set_host_numa_node_id(&mut self, host_numa_node_id: Option) { + self.host_numa_node_id = host_numa_node_id; + } +} + +#[cfg(test)] +mod tests { + use vm_memory::GuestMemoryRegion; + + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + impl Default for MemDeviceConfigInfo { + fn default() -> Self { + MemDeviceConfigInfo { + mem_id: "".to_string(), + size_mib: 0, + capacity_mib: 1024, + multi_region: true, + host_numa_node_id: None, + guest_numa_node_id: None, + use_generic_irq: None, + use_shared_irq: None, + } + } + } + + #[test] + fn test_mem_config_check_conflicts() { + let config = MemDeviceConfigInfo::default(); + let mut config2 = MemDeviceConfigInfo::default(); + assert!(config.check_conflicts(&config2).is_err()); + config2.mem_id = "dummy_mem".to_string(); + assert!(config.check_conflicts(&config2).is_ok()); + } + + #[test] + fn test_create_mem_devices_configs() { + let mgr = MemDeviceMgr::default(); + assert_eq!(mgr.info_list.len(), 0); + assert_eq!(mgr.get_index_of_mem_dev(""), None); + } + + #[test] + fn test_mem_insert_or_update_device() { + // Init vm for test. + let mut vm = create_vm_for_test(); + + // We don't need to use virtio-mem before start vm + // Test for standard config with hotplug + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + true, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + + let dummy_mem_device = MemDeviceConfigInfo::default(); + vm.device_manager_mut() + .mem_manager + .insert_or_update_device(device_op_ctx, dummy_mem_device) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + } + + #[test] + fn test_mem_attach_device() { + // Init vm and insert mem config for test. + let mut vm = create_vm_for_test(); + let dummy_mem_device = MemDeviceConfigInfo::default(); + vm.device_manager_mut() + .mem_manager + .info_list + .insert_or_update(&dummy_mem_device) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + + // Test for standard config + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + false, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + vm.device_manager_mut() + .mem_manager + .attach_devices(&mut device_op_ctx) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + } + + #[test] + fn test_mem_create_region() { + let vm = create_vm_for_test(); + let ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + true, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + let mem_id = String::from("mem0"); + let guest_addr = GuestAddress(0x1_0000_0000); + let region_len = 0x1000_0000; + let kvm_slot = 2; + + // no vfio manager, no numa node + let mut factory = MemoryRegionFactory::new(&ctx, mem_id, None).unwrap(); + let region_opt = factory.create_region(guest_addr, region_len, kvm_slot); + assert_eq!(region_opt.unwrap().len(), region_len); + } +} diff --git a/src/dragonball/src/device_manager/memory_region_handler.rs b/src/dragonball/src/device_manager/memory_region_handler.rs index 2be149ef9..fbf5aa20c 100644 --- a/src/dragonball/src/device_manager/memory_region_handler.rs +++ b/src/dragonball/src/device_manager/memory_region_handler.rs @@ -55,6 +55,7 @@ impl DeviceVirtioRegionHandler { None, file_offset, region.flags(), + region.prot(), false, )); diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 324cf41fc..69112e3ae 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -90,6 +90,12 @@ mod memory_region_handler; #[cfg(feature = "virtio-fs")] pub use self::memory_region_handler::*; +#[cfg(feature = "virtio-mem")] +/// Device manager for virtio-mem devices. +pub mod mem_dev_mgr; +#[cfg(feature = "virtio-mem")] +use self::mem_dev_mgr::MemDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -311,6 +317,27 @@ impl DeviceOpContext { } } + pub(crate) fn get_vm_config(&self) -> Result { + match self.vm_config.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn get_address_space(&self) -> Result { + match self.address_space.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn get_epoll_mgr(&self) -> Result { + match self.epoll_mgr.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + pub(crate) fn logger(&self) -> &slog::Logger { &self.logger } @@ -498,6 +525,9 @@ pub struct DeviceManager { #[cfg(feature = "virtio-fs")] fs_manager: Arc>, + + #[cfg(feature = "virtio-mem")] + pub(crate) mem_manager: MemDeviceMgr, } impl DeviceManager { @@ -530,6 +560,8 @@ impl DeviceManager { virtio_net_manager: VirtioNetDeviceMgr::default(), #[cfg(feature = "virtio-fs")] fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), + #[cfg(feature = "virtio-mem")] + mem_manager: MemDeviceMgr::default(), } } @@ -899,6 +931,24 @@ impl DeviceManager { ) } + /// Create an Virtio MMIO transport layer device for the virtio backend device with configure + /// change notification enabled. + pub fn create_mmio_virtio_device_with_device_change_notification( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + let features = DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY; + DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(features), + use_shared_irq, + use_generic_irq, + ) + } + /// Create an Virtio MMIO transport layer device for the virtio backend device with specified /// features. pub fn create_mmio_virtio_device_with_features( @@ -1077,6 +1127,8 @@ mod tests { virtio_net_manager: VirtioNetDeviceMgr::default(), #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), + #[cfg(feature = "virtio-mem")] + mem_manager: MemDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), From 8fb7ab75186b3077e556c566e2beb951571c4035 Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Sun, 29 Jan 2023 16:18:06 +0800 Subject: [PATCH 45/76] dragonball: introduce virtio-balloon device We introduce virtio-balloon device to support memory resize. virtio-balloon device could reclaim memory from guest to host. Fixes: #6719 Signed-off-by: Helin Guo --- src/dragonball/Cargo.lock | 124 ++++-- src/dragonball/Cargo.toml | 1 + src/dragonball/src/api/v1/vmm_action.rs | 87 +++- .../src/device_manager/balloon_dev_mgr.rs | 419 ++++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 13 + src/dragonball/src/error.rs | 5 + src/dragonball/src/vm/mod.rs | 13 + 7 files changed, 632 insertions(+), 30 deletions(-) create mode 100644 src/dragonball/src/device_manager/balloon_dev_mgr.rs diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock index 3cc177fe1..0ed990c0b 100644 --- a/src/dragonball/Cargo.lock +++ b/src/dragonball/Cargo.lock @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -445,13 +445,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -483,7 +483,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -619,9 +619,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -671,7 +671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -842,7 +842,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1045,7 +1045,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1121,16 +1121,16 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1168,18 +1168,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" dependencies = [ "proc-macro2", "quote", @@ -1188,9 +1188,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1423,7 +1423,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1611,7 +1611,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -1620,13 +1629,28 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -1635,42 +1659,84 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "xattr" version = "0.2.3" diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 0c94530ca..b0e03047d 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -56,3 +56,4 @@ virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] # virtio-fs only work on atomic-guest-memory virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"] +virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"] diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 6243b82a3..247daf10d 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -19,6 +19,8 @@ use crate::vmm::Vmm; use self::VmConfigError::*; use self::VmmActionError::MachineConfig; +#[cfg(feature = "virtio-balloon")] +pub use crate::device_manager::balloon_dev_mgr::{BalloonDeviceConfigInfo, BalloonDeviceError}; #[cfg(feature = "virtio-blk")] pub use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, @@ -36,7 +38,6 @@ pub use crate::device_manager::virtio_net_dev_mgr::{ }; #[cfg(feature = "virtio-vsock")] pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; - #[cfg(feature = "hotplug")] pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo}; @@ -108,6 +109,11 @@ pub enum VmmActionError { /// Mem device related errors. #[error("virtio-mem device error: {0}")] Mem(#[source] MemDeviceError), + + #[cfg(feature = "virtio-balloon")] + /// Balloon device related errors. + #[error("virtio-balloon device error: {0}")] + Balloon(#[source] BalloonDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -187,6 +193,11 @@ pub enum VmmAction { #[cfg(feature = "virtio-mem")] /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input. InsertMemDevice(MemDeviceConfigInfo), + + #[cfg(feature = "virtio-balloon")] + /// Add a new balloon device or update one that already exists using the `BalloonDeviceConfig` + /// as input. + InsertBalloonDevice(BalloonDeviceConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -291,6 +302,10 @@ impl VmmService { VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg), #[cfg(feature = "virtio-mem")] VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg), + #[cfg(feature = "virtio-balloon")] + VmmAction::InsertBalloonDevice(balloon_cfg) => { + self.add_balloon_device(vmm, event_mgr, balloon_cfg) + } }; debug!("send vmm response: {:?}", response); @@ -691,6 +706,36 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::Mem) } + + #[cfg(feature = "virtio-balloon")] + fn add_balloon_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: BalloonDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; + + if config.size_mib != 0 { + info!("add_balloon_device: wait prealloc"); + vm.stop_prealloc().map_err(VmmActionError::StartMicroVm)?; + } + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady + } else { + VmmActionError::StartMicroVm(e) + } + })?; + + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Balloon) + } } fn handle_cpu_topology( @@ -1539,4 +1584,44 @@ mod tests { t.check_request(); } } + + #[cfg(feature = "virtio-balloon")] + #[test] + fn test_vmm_action_insert_balloon_device() { + skip_if_not_root!(); + + let tests = &mut [ + // hotplug unready + TestData::new( + VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()), + InstanceState::Running, + &|result| { + assert!(matches!( + result, + Err(VmmActionError::StartMicroVm( + StartMicroVmError::UpcallMissVsock + )) + )); + let err_string = format!("{}", result.unwrap_err()); + let expected_err = String::from( + "failed to boot the VM: \ + the upcall client needs a virtio-vsock device for communication", + ); + assert_eq!(err_string, expected_err); + }, + ), + // success + TestData::new( + VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()), + InstanceState::Uninitialized, + &|result| { + assert!(result.is_ok()); + }, + ), + ]; + + for t in tests.iter_mut() { + t.check_request(); + } + } } diff --git a/src/dragonball/src/device_manager/balloon_dev_mgr.rs b/src/dragonball/src/device_manager/balloon_dev_mgr.rs new file mode 100644 index 000000000..b0ee2bd37 --- /dev/null +++ b/src/dragonball/src/device_manager/balloon_dev_mgr.rs @@ -0,0 +1,419 @@ +// Copyright 2020 Alibaba Cloud. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dbs_virtio_devices as virtio; +use serde_derive::{Deserialize, Serialize}; +use slog::{error, info}; +use virtio::balloon::{Balloon, BalloonConfig}; +use virtio::Error as VirtIoError; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::DbsMmioV2Device; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = false; + +/// Errors associated with `BalloonDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum BalloonDeviceError { + /// The balloon device was already used. + #[error("the virtio-balloon ID was already added to a different device")] + BalloonDeviceAlreadyExists, + + /// Cannot perform the requested operation after booting the microVM. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// guest memory error + #[error("failed to access guest memory, {0}")] + GuestMemoryError(#[source] vm_memory::mmap::Error), + + /// create balloon device error + #[error("failed to create virtio-balloon device, {0}")] + CreateBalloonDevice(#[source] virtio::Error), + + /// hotplug balloon device error + #[error("cannot hotplug virtio-balloon device, {0}")] + HotplugDeviceFailed(#[source] DeviceMgrError), + + /// create mmio device error + #[error("cannot create virtio-balloon mmio device, {0}")] + CreateMmioDevice(#[source] DeviceMgrError), + + /// Cannot initialize a balloon device or add a device to the MMIO Bus. + #[error("failure while registering balloon device: {0}")] + RegisterBalloonDevice(#[source] DeviceMgrError), + + /// resize balloon device error + #[error("failure while resizing virtio-balloon device, {0}")] + ResizeFailed(#[source] VirtIoError), + + /// The balloon device id doesn't exist. + #[error("invalid balloon device id '{0}'")] + InvalidDeviceId(String), + + /// balloon device does not exist + #[error("balloon device does not exist")] + NotExist, + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a virtio-balloon device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +pub struct BalloonDeviceConfigInfo { + /// Unique identifier of the balloon device + pub balloon_id: String, + /// Resize balloon size in mib + pub size_mib: u64, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, + /// VIRTIO_BALLOON_F_DEFLATE_ON_OOM + pub f_deflate_on_oom: bool, + /// VIRTIO_BALLOON_F_REPORTING + pub f_reporting: bool, +} + +impl ConfigItem for BalloonDeviceConfigInfo { + type Err = BalloonDeviceError; + + fn id(&self) -> &str { + &self.balloon_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> { + if self.balloon_id.as_str() == other.balloon_id.as_str() { + Err(BalloonDeviceError::BalloonDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Balloon Device Info +pub type BalloonDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for BalloonDeviceInfo { + type Err = BalloonDeviceError; + + fn id(&self) -> &str { + &self.config.balloon_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> { + if self.config.balloon_id.as_str() == other.config.balloon_id.as_str() { + Err(BalloonDeviceError::BalloonDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Balloon Devices Configs +#[derive(Clone)] +pub struct BalloonDeviceMgr { + /// A list of `BalloonDeviceConfig` objects. + info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl BalloonDeviceMgr { + /// Inserts `balloon_cfg` in the virtio-balloon device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + pub fn insert_or_update_device( + &mut self, + mut ctx: DeviceOpContext, + balloon_cfg: BalloonDeviceConfigInfo, + ) -> std::result::Result<(), BalloonDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + error!(ctx.logger(), "hotplug feature has been disabled."; + "subsystem" => "balloon_dev_mgr",); + return Err(BalloonDeviceError::UpdateNotAllowedPostBoot); + } + + let epoll_mgr = ctx + .get_epoll_mgr() + .map_err(BalloonDeviceError::DeviceManager)?; + + // If the id of the drive already exists in the list, the operation is update. + if let Some(index) = self.get_index_of_balloon_dev(&balloon_cfg.balloon_id) { + // Update an existing balloon device + if ctx.is_hotplug { + info!(ctx.logger(), "resize virtio balloon size to {:?}", balloon_cfg.size_mib; "subsystem" => "balloon_dev_mgr"); + self.update_balloon_size(index, balloon_cfg.size_mib)?; + } + self.info_list.insert_or_update(&balloon_cfg)?; + } else { + // Create a new balloon device + if !self.info_list.is_empty() { + error!(ctx.logger(), "only support one balloon device!"; "subsystem" => "balloon_dev_mgr"); + return Err(BalloonDeviceError::BalloonDeviceAlreadyExists); + } + + if !ctx.is_hotplug { + self.info_list.insert_or_update(&balloon_cfg)?; + return Ok(()); + } + + info!(ctx.logger(), "hotplug balloon device: {}", balloon_cfg.balloon_id; "subsystem" => "balloon_dev_mgr"); + let device = Box::new( + virtio::balloon::Balloon::new( + epoll_mgr, + BalloonConfig { + f_deflate_on_oom: balloon_cfg.f_deflate_on_oom, + f_reporting: balloon_cfg.f_reporting, + }, + ) + .map_err(BalloonDeviceError::CreateBalloonDevice)?, + ); + + let mmio_dev = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + device, + &mut ctx, + balloon_cfg.use_shared_irq.unwrap_or(self.use_shared_irq), + balloon_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BalloonDeviceError::CreateMmioDevice)?; + ctx.insert_hotplug_mmio_device(&mmio_dev, None) + .map_err(|e| { + error!( + ctx.logger(), + "hotplug balloon device {} error: {}", + &balloon_cfg.balloon_id, e; + "subsystem" => "balloon_dev_mgr" + ); + BalloonDeviceError::HotplugDeviceFailed(e) + })?; + let index = self.info_list.insert_or_update(&balloon_cfg)?; + self.info_list[index].set_device(mmio_dev); + } + Ok(()) + } + + /// Attaches all virtio-balloon devices from the BalloonDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), BalloonDeviceError> { + let epoll_mgr = ctx + .get_epoll_mgr() + .map_err(BalloonDeviceError::DeviceManager)?; + + for info in self.info_list.iter_mut() { + info!(ctx.logger(), "attach balloon device: {}", info.config.balloon_id; "subsystem" => "balloon_dev_mgr"); + + let device = Balloon::new( + epoll_mgr.clone(), + BalloonConfig { + f_deflate_on_oom: info.config.f_deflate_on_oom, + f_reporting: info.config.f_reporting, + }, + ) + .map_err(BalloonDeviceError::CreateBalloonDevice)?; + let mmio_dev = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BalloonDeviceError::RegisterBalloonDevice)?; + info.set_device(mmio_dev); + } + + Ok(()) + } + + fn update_balloon_size( + &self, + index: usize, + size_mib: u64, + ) -> std::result::Result<(), BalloonDeviceError> { + let device = self.info_list[index] + .device + .as_ref() + .ok_or_else(|| BalloonDeviceError::NotExist)?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(balloon_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return balloon_dev + .set_size(size_mib) + .map_err(BalloonDeviceError::ResizeFailed); + } + } + Ok(()) + } + + fn get_index_of_balloon_dev(&self, balloon_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.balloon_id.eq(balloon_id)) + } +} + +impl Default for BalloonDeviceMgr { + /// Create a new `BalloonDeviceMgr` object.. + fn default() -> Self { + BalloonDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + impl Default for BalloonDeviceConfigInfo { + fn default() -> Self { + BalloonDeviceConfigInfo { + balloon_id: "".to_string(), + size_mib: 0, + use_generic_irq: None, + use_shared_irq: None, + f_deflate_on_oom: false, + f_reporting: false, + } + } + } + + #[test] + fn test_balloon_config_check_conflicts() { + let config = BalloonDeviceConfigInfo::default(); + let mut config2 = BalloonDeviceConfigInfo::default(); + assert!(config.check_conflicts(&config2).is_err()); + config2.balloon_id = "dummy_balloon".to_string(); + assert!(config.check_conflicts(&config2).is_ok()); + } + + #[test] + fn test_create_balloon_devices_configs() { + let mgr = BalloonDeviceMgr::default(); + assert_eq!(mgr.info_list.len(), 0); + assert_eq!(mgr.get_index_of_balloon_dev(""), None); + } + + #[test] + fn test_balloon_insert_or_update_device() { + //Init vm for test. + let mut vm = create_vm_for_test(); + + // Test for standard config + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + } + + #[test] + fn test_balloon_attach_device() { + //Init vm and insert balloon config for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + // Test for standard config + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + assert!(vm + .device_manager_mut() + .balloon_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + } + + #[test] + fn test_balloon_update_device() { + //Init vm for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + assert!(vm + .device_manager_mut() + .balloon_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + assert!(vm + .device_manager() + .balloon_manager + .update_balloon_size(0, 200) + .is_ok()); + } +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 69112e3ae..b007270c9 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -96,6 +96,12 @@ pub mod mem_dev_mgr; #[cfg(feature = "virtio-mem")] use self::mem_dev_mgr::MemDeviceMgr; +#[cfg(feature = "virtio-balloon")] +/// Device manager for virtio-balloon devices. +pub mod balloon_dev_mgr; +#[cfg(feature = "virtio-balloon")] +use self::balloon_dev_mgr::BalloonDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -528,6 +534,9 @@ pub struct DeviceManager { #[cfg(feature = "virtio-mem")] pub(crate) mem_manager: MemDeviceMgr, + + #[cfg(feature = "virtio-balloon")] + pub(crate) balloon_manager: BalloonDeviceMgr, } impl DeviceManager { @@ -562,6 +571,8 @@ impl DeviceManager { fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), #[cfg(feature = "virtio-mem")] mem_manager: MemDeviceMgr::default(), + #[cfg(feature = "virtio-balloon")] + balloon_manager: BalloonDeviceMgr::default(), } } @@ -1129,6 +1140,8 @@ mod tests { vsock_manager: VsockDeviceMgr::default(), #[cfg(feature = "virtio-mem")] mem_manager: MemDeviceMgr::default(), + #[cfg(feature = "virtio-balloon")] + balloon_manager: BalloonDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 66a24f562..e0aab17fb 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -188,6 +188,11 @@ pub enum StartMicroVmError { /// Virtio-fs errors. #[error("virtio-fs errors: {0}")] FsDeviceError(#[source] device_manager::fs_dev_mgr::FsDeviceError), + + #[cfg(feature = "virtio-balloon")] + /// Virtio-balloon errors. + #[error("virtio-balloon errors: {0}")] + BalloonDeviceError(#[source] device_manager::balloon_dev_mgr::BalloonDeviceError), } /// Errors associated with starting the instance. diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 6e2545c6f..9af046c43 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -386,6 +386,19 @@ impl Vm { (dragonball_version, instance_id) } + + pub(crate) fn stop_prealloc(&mut self) -> std::result::Result<(), StartMicroVmError> { + if self.address_space.is_initialized() { + return self + .address_space + .wait_prealloc(true) + .map_err(StartMicroVmError::AddressManagerError); + } + + Err(StartMicroVmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + )) + } } impl Vm { From f28a62164a2180ed13e378e228cd747db060a1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Fri, 2 Jun 2023 13:39:43 -0700 Subject: [PATCH 46/76] gha: Add `cloud-hypervisor-glibc` build target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the glibc flavor of CLH to the list of assets as preparation for #6839. Mariner Kata is only tested with glibc. Fixes: #7026 Signed-off-by: Aurélien Bombo --- .github/workflows/build-kata-static-tarball-amd64.yaml | 9 +++++++++ .github/workflows/release-amd64.yaml | 2 ++ .github/workflows/release-arm64.yaml | 2 ++ .github/workflows/release-s390x.yaml | 2 ++ tools/packaging/kata-deploy/local-build/Makefile | 6 +++++- 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 09ddfd05d..0b7618e87 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -2,6 +2,10 @@ name: CI | Build kata-static tarball for amd64 on: workflow_call: inputs: + stage: + required: false + type: string + default: test tarball-suffix: required: false type: string @@ -15,8 +19,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + stage: ${{ inputs.stage }} asset: - cloud-hypervisor + - cloud-hypervisor-glibc - firecracker - kernel - kernel-sev @@ -38,6 +44,9 @@ jobs: - shim-v2 - tdvf - virtiofsd + exclude: + - stage: release + asset: cloud-hypervisor-glibc steps: - name: Login to Kata Containers quay.io if: ${{ inputs.push-to-registry == 'yes' }} diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml index 8d48b956b..6ff990696 100644 --- a/.github/workflows/release-amd64.yaml +++ b/.github/workflows/release-amd64.yaml @@ -9,6 +9,8 @@ on: jobs: build-kata-static-tarball-amd64: uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml + with: + stage: release kata-deploy: needs: build-kata-static-tarball-amd64 diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml index 2b5e810a3..cd7db8fdf 100644 --- a/.github/workflows/release-arm64.yaml +++ b/.github/workflows/release-arm64.yaml @@ -9,6 +9,8 @@ on: jobs: build-kata-static-tarball-arm64: uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml + with: + stage: release kata-deploy: needs: build-kata-static-tarball-arm64 diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml index ef436b7b8..5f3aaf05f 100644 --- a/.github/workflows/release-s390x.yaml +++ b/.github/workflows/release-s390x.yaml @@ -9,6 +9,8 @@ on: jobs: build-kata-static-tarball-s390x: uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml + with: + stage: release kata-deploy: needs: build-kata-static-tarball-s390x diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index eb161e82b..8cb740b94 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -45,7 +45,8 @@ serial-targets: rootfs-image-tdx-tarball \ rootfs-initrd-sev-tarball \ rootfs-initrd-tarball \ - cloud-hypervisor-tarball + cloud-hypervisor-tarball \ + cloud-hypervisor-glibc-tarball %-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh $(call BUILD,$*) @@ -53,6 +54,9 @@ serial-targets: cloud-hypervisor-tarball: ${MAKE} $@-build +cloud-hypervisor-glibc-tarball: + exit 0 + firecracker-tarball: ${MAKE} $@-build From 9f7a45996c468354cc3986bb27ab3a3ee66571e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Fri, 2 Jun 2023 13:45:33 -0700 Subject: [PATCH 47/76] gha: Add `rootfs-initrd-mariner` build target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the Mariner guest image build target to the list of assets as preparation for #6839. Signed-off-by: Aurélien Bombo --- .github/workflows/build-kata-static-tarball-amd64.yaml | 1 + tools/packaging/kata-deploy/local-build/Makefile | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 0b7618e87..1ef136ff1 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -40,6 +40,7 @@ jobs: - rootfs-image - rootfs-image-tdx - rootfs-initrd + - rootfs-initrd-mariner - rootfs-initrd-sev - shim-v2 - tdvf diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 8cb740b94..c23f6b04e 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -43,6 +43,7 @@ serial-targets: ${MAKE} -f $(MK_PATH) -j 1 V= \ rootfs-image-tarball \ rootfs-image-tdx-tarball \ + rootfs-initrd-mariner-tarball \ rootfs-initrd-sev-tarball \ rootfs-initrd-tarball \ cloud-hypervisor-tarball \ @@ -108,6 +109,9 @@ rootfs-image-tarball: rootfs-image-tdx-tarball: kernel-tdx-experimental-tarball ${MAKE} $@-build +rootfs-initrd-mariner-tarball: + exit 0 + rootfs-initrd-sev-tarball: kernel-sev-tarball ${MAKE} $@-build From 2137739987f62aa4e13d7d25b50bd81db367e37b Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Mon, 12 Jun 2023 11:23:44 +0800 Subject: [PATCH 48/76] runtime-rs: update Cargo.lock After we support memory resize in Dragonball, we need to update Cargo.lock in runtime-rs. Fixes: #6719 Signed-off-by: Helin Guo --- src/runtime-rs/Cargo.lock | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 0b5cd7ca9..358776fab 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -655,11 +655,12 @@ dependencies = [ [[package]] name = "dbs-address-space" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc" +checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322" dependencies = [ "arc-swap", + "lazy_static", "libc", "nix 0.23.2", "thiserror", @@ -746,9 +747,9 @@ dependencies = [ [[package]] name = "dbs-upcall" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f" +checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb" dependencies = [ "anyhow", "dbs-utils", @@ -776,9 +777,9 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668" +checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8" dependencies = [ "byteorder", "caps", @@ -792,7 +793,7 @@ dependencies = [ "kvm-ioctls", "libc", "log", - "nix 0.23.2", + "nix 0.24.3", "nydus-api", "nydus-blobfs", "nydus-rafs", From 8cb4238b4681f7a5752ebffd43d7cc3520324171 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 12 Jun 2023 09:24:09 +0100 Subject: [PATCH 49/76] packaging: Remove snap package Nobody has volunteered to maintain the (currently broken) snap build, so remove it. Fixes: #6769. Signed-off-by: James O. D. Hunt --- .github/workflows/snap-release.yaml | 53 ------ .github/workflows/snap.yaml | 37 ---- README.md | 2 - docs/install/README.md | 22 --- ...ers-3.0-rust-runtime-installation-guide.md | 3 - docs/install/snap-installation-guide.md | 82 --------- snap/local/README.md | 101 ----------- snap/local/snap-common.sh | 114 ------------ snap/snapcraft.yaml | 170 ------------------ src/runtime/README.md | 2 - tools/packaging/.gitignore | 2 - tools/packaging/Makefile | 10 +- tools/packaging/README.md | 4 - tools/packaging/kernel/README.md | 3 +- 14 files changed, 2 insertions(+), 603 deletions(-) delete mode 100644 .github/workflows/snap-release.yaml delete mode 100644 .github/workflows/snap.yaml delete mode 100644 docs/install/snap-installation-guide.md delete mode 100644 snap/local/README.md delete mode 100644 snap/local/snap-common.sh delete mode 100644 snap/snapcraft.yaml diff --git a/.github/workflows/snap-release.yaml b/.github/workflows/snap-release.yaml deleted file mode 100644 index 4a12ce8d7..000000000 --- a/.github/workflows/snap-release.yaml +++ /dev/null @@ -1,53 +0,0 @@ -name: Release Kata in snapcraft store -on: - push: - tags: - - '[0-9]+.[0-9]+.[0-9]+*' - -env: - SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }} - -jobs: - release-snap: - runs-on: ubuntu-20.04 - steps: - - name: Check out Git repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Snapcraft - run: | - # Required to avoid snapcraft install failure - sudo chown root:root / - - # "--classic" is needed for the GitHub action runner - # environment. - sudo snap install snapcraft --classic - - # Allow other parts to access snap binaries - echo /snap/bin >> "$GITHUB_PATH" - - - name: Build snap - run: | - # Removing man-db, workflow kept failing, fixes: #4480 - sudo apt -y remove --purge man-db - sudo apt-get update - sudo apt-get install -y git git-extras - kata_url="https://github.com/kata-containers/kata-containers" - latest_version=$(git ls-remote --tags ${kata_url} | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1) - current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)" - # Check semantic versioning format (x.y.z) and if the current tag is the latest tag - if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then - # Current version is the latest version, build it - snapcraft snap --debug --destructive-mode - fi - - - name: Upload snap - run: | - snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)" - snap_file="kata-containers_${snap_version}_amd64.snap" - # Upload the snap if it exists - if [ -f ${snap_file} ]; then - snapcraft upload --release=stable ${snap_file} - fi diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml deleted file mode 100644 index ac163fa2e..000000000 --- a/.github/workflows/snap.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: snap CI -on: - pull_request: - types: - - opened - - synchronize - - reopened - - edited - paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ] - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - name: Check out - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Snapcraft - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - # Required to avoid snapcraft install failure - sudo chown root:root / - - # "--classic" is needed for the GitHub action runner - # environment. - sudo snap install snapcraft --classic - - # Allow other parts to access snap binaries - echo /snap/bin >> "$GITHUB_PATH" - - - name: Build snap - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - snapcraft snap --debug --destructive-mode diff --git a/README.md b/README.md index 05933bb58..b53125194 100644 --- a/README.md +++ b/README.md @@ -146,8 +146,6 @@ The table below lists the remaining parts of the project: Kata Containers is now [available natively for most distributions](docs/install/README.md#packaged-installation-methods). -However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See -the [components](#components) section for further details. ## Glossary of Terms diff --git a/docs/install/README.md b/docs/install/README.md index 8a86bc7c8..5fb1c3a03 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -19,7 +19,6 @@ Packaged installation methods uses your distribution's native package format (su |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------| | [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | -| ~~[Using snap](#snap-installation)~~ | ~~Easy to install~~ | ~~yes~~ | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~ | | [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | | [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | | [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | @@ -42,27 +41,6 @@ Kata packages are provided by official distribution repositories for: | [CentOS](centos-installation-guide.md) | 8 | | [Fedora](fedora-installation-guide.md) | 34 | -### Snap Installation - -> **WARNING:** -> -> The Snap package method is **unmaintained** and only provides an old -> version of Kata Containers: -> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) -> provides Kata Containers -> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) -> but the latest stable Kata Containers release at the time of writing is -> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). -> -> We recommend strongly that you switch to an alternative Kata Containers installation method. -> -> See: https://github.com/kata-containers/kata-containers/issues/6769 -> for further details. - -~~The snap installation is available for all distributions which support `snapd`.~~ - -~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~ - ### Automatic Installation [Use `kata-manager`](/utils/README.md) to automatically install a working Kata Containers system. diff --git a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md index 4cfcb392d..bb1732292 100644 --- a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md +++ b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md @@ -26,7 +26,6 @@ architectures: |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- | | [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes | | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | No | -| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | No | | [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | No | | [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | No | | [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | Yes | @@ -36,8 +35,6 @@ architectures: Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md). ### Official packages `ToDo` -### Snap Installation -`ToDo` ### Automatic Installation `ToDo` ### Manual Installation diff --git a/docs/install/snap-installation-guide.md b/docs/install/snap-installation-guide.md deleted file mode 100644 index acf2e637e..000000000 --- a/docs/install/snap-installation-guide.md +++ /dev/null @@ -1,82 +0,0 @@ -# Kata Containers snap package - -> **WARNING:** -> -> The Snap package method is **unmaintained** and only provides an old -> version of Kata Containers: -> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) -> provides Kata Containers -> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) -> but the latest stable Kata Containers release at the time of writing is -> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). -> -> We recommend strongly that you switch to an alternative Kata Containers installation method. -> -> See: https://github.com/kata-containers/kata-containers/issues/6769 -> for further details. - -## Install Kata Containers - -Kata Containers can be installed in any Linux distribution that supports -[snapd](https://docs.snapcraft.io/installing-snapd). - -Run the following command to install **Kata Containers**: - -> **WARNING:** -> -> The Snap package method is **unmaintained** and only provides an old -> version of Kata Containers: -> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) -> provides Kata Containers -> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) -> but the latest stable Kata Containers release at the time of writing is -> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). -> -> We recommend strongly that you switch to an alternative Kata Containers installation method. -> -> See: https://github.com/kata-containers/kata-containers/issues/6769 -> for further details. - -```sh -$ sudo snap install kata-containers --stable --classic -``` - -## Configure Kata Containers - -By default Kata Containers snap image is mounted at `/snap/kata-containers` as a -read-only file system, therefore default configuration file can not be edited. -Fortunately Kata Containers supports loading a configuration file from another -path than the default. - -```sh -$ sudo mkdir -p /etc/kata-containers -$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/ -$ $EDITOR /etc/kata-containers/configuration.toml -``` - -## Integration with shim v2 Container Engines - -The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the -`containerd-shim-kata-v2` binary to allow Kata Containers to be created. -Run the following command to create a symbolic link to the shim v2 binary. - -```sh -$ sudo ln -sf /snap/kata-containers/current/usr/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2 -``` - -Once the symbolic link has been created and the engine daemon configured, `io.containerd.kata.v2` -can be used as runtime. - -Read the following documents to know how to run Kata Containers 2.x with `containerd`. - -* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md) -* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md) - - -## Remove Kata Containers snap package - -Run the following command to remove the Kata Containers snap: - -```sh -$ sudo snap remove kata-containers -``` diff --git a/snap/local/README.md b/snap/local/README.md deleted file mode 100644 index 4b449ef17..000000000 --- a/snap/local/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Kata Containers snap image - -This directory contains the resources needed to build the Kata Containers -[snap][1] image. - -## Initial setup - -Kata Containers can be installed in any Linux distribution that supports -[snapd](https://docs.snapcraft.io/installing-snapd). For this example, we -assume Ubuntu as your base distro. -```sh -$ sudo apt-get --no-install-recommends install -y apt-utils ca-certificates snapd snapcraft -``` - -## Install snap - -You can install the Kata Containers snap from the [snapcraft store][8] or by running the following command: - -```sh -$ sudo snap install kata-containers --classic -``` - -## Build and install snap image - -Run the command below which will use the packaging Makefile to build the snap image: - -```sh -$ make -C tools/packaging snap -``` - -> **Warning:** -> -> By default, `snapcraft` will create a clean virtual machine -> environment to build the snap in using the `multipass` tool. -> -> However, `multipass` is silently disabled when `--destructive-mode` is -> used. -> -> Since building the Kata Containers package currently requires -> `--destructive-mode`, the snap will be built using the host -> environment. To avoid parts of the build auto-detecting additional -> features to enable (for example for QEMU), we recommend that you -> only run the snap build in a minimal host environment. - -To install the resulting snap image, snap must be put in [classic mode][3] and the -security confinement must be disabled (`--classic`). Also since the resulting snap -has not been signed the verification of signature must be omitted (`--dangerous`). - -```sh -$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap" -``` - -Replace `${version}` with the current version of Kata Containers and `${arch}` with -the system architecture. - -## Configure Kata Containers - -By default Kata Containers snap image is mounted at `/snap/kata-containers` as a -read-only file system, therefore default configuration file can not be edited. -Fortunately [`kata-runtime`][4] supports loading a configuration file from another -path than the default. - -```sh -$ sudo mkdir -p /etc/kata-containers -$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/ -$ $EDITOR /etc/kata-containers/configuration.toml -``` - -## Integration with docker and Kubernetes - -The path to the runtime provided by the Kata Containers snap image is -`/snap/kata-containers/current/usr/bin/kata-runtime`. You should use it to -run Kata Containers with [docker][9] and [Kubernetes][10]. - -## Remove snap - -You can remove the Kata Containers snap by running the following command: - -```sh -$ sudo snap remove kata-containers -``` - -## Limitations - -The [miniOS image][2] is not included in the snap image as it is not possible for -QEMU to open a guest RAM backing store on a read-only filesystem. Fortunately, -you can start Kata Containers with a Linux initial RAM disk (initrd) that is -included in the snap image. If you want to use the miniOS image instead of initrd, -then a new configuration file can be [created](#configure-kata-containers) -and [configured][7]. - -[1]: https://docs.snapcraft.io/snaps/intro -[2]: ../../docs/design/architecture/README.md#root-filesystem-image -[3]: https://docs.snapcraft.io/reference/confinement#classic -[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration -[5]: https://docs.docker.com/engine/reference/commandline/dockerd -[6]: ../../docs/install/docker/ubuntu-docker-install.md -[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image -[8]: https://snapcraft.io/kata-containers -[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker -[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes diff --git a/snap/local/snap-common.sh b/snap/local/snap-common.sh deleted file mode 100644 index 0a2a18e15..000000000 --- a/snap/local/snap-common.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2022 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 - -# Description: Idempotent script to be sourced by all parts in a -# snapcraft config file. - -set -o errexit -set -o nounset -set -o pipefail - -# XXX: Bash-specific code. zsh doesn't support this option and that *does* -# matter if this script is run sourced... since it'll be using zsh! ;) -[ -n "$BASH_VERSION" ] && set -o errtrace - -[ -n "${DEBUG:-}" ] && set -o xtrace - -die() -{ - echo >&2 "ERROR: $0: $*" -} - -[ -n "${SNAPCRAFT_STAGE:-}" ] ||\ - die "must be sourced from a snapcraft config file" - -snap_yq_version=3.4.1 - -snap_common_install_yq() -{ - export yq="${SNAPCRAFT_STAGE}/bin/yq" - - local yq_pkg - yq_pkg="github.com/mikefarah/yq" - - local yq_url - yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}" - curl -o "${yq}" -L "${yq_url}" - chmod +x "${yq}" -} - -# Function that should be called for each snap "part" in -# snapcraft.yaml. -snap_common_main() -{ - # Architecture - arch="$(uname -m)" - - case "${arch}" in - aarch64) - goarch="arm64" - qemu_arch="${arch}" - ;; - - ppc64le) - goarch="ppc64le" - qemu_arch="ppc64" - ;; - - s390x) - goarch="${arch}" - qemu_arch="${arch}" - ;; - - x86_64) - goarch="amd64" - qemu_arch="${arch}" - ;; - - *) die "unsupported architecture: ${arch}" ;; - esac - - dpkg_arch=$(dpkg --print-architecture) - - # golang - # - # We need the O/S name in golang format, but since we don't - # know if the godeps part has run, we don't know if golang is - # available yet, hence fall back to a standard system command. - goos="$(go env GOOS &>/dev/null || true)" - [ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]') - - export GOROOT="${SNAPCRAFT_STAGE}" - export GOPATH="${GOROOT}/gopath" - export GO111MODULE="auto" - - mkdir -p "${GOPATH}/bin" - export PATH="${GOPATH}/bin:${PATH}" - - # Proxy - export http_proxy="${http_proxy:-}" - export https_proxy="${https_proxy:-}" - - # Binaries - mkdir -p "${SNAPCRAFT_STAGE}/bin" - - export PATH="$PATH:${SNAPCRAFT_STAGE}/bin" - - # YAML query tool - export yq="${SNAPCRAFT_STAGE}/bin/yq" - - # Kata paths - export kata_dir=$(printf "%s/src/github.com/%s/%s" \ - "${GOPATH}" \ - "${SNAPCRAFT_PROJECT_NAME}" \ - "${SNAPCRAFT_PROJECT_NAME}") - - export versions_file="${kata_dir}/versions.yaml" - - [ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq -} - -snap_common_main diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml deleted file mode 100644 index 4bbb8e0f1..000000000 --- a/snap/snapcraft.yaml +++ /dev/null @@ -1,170 +0,0 @@ -name: kata-containers -website: https://github.com/kata-containers/kata-containers -summary: Build lightweight VMs that seamlessly plug into the containers ecosystem -description: | - Kata Containers is an open source project and community working to build a - standard implementation of lightweight Virtual Machines (VMs) that feel and - perform like containers, but provide the workload isolation and security - advantages of VMs -confinement: classic -adopt-info: metadata -base: core20 - -parts: - metadata: - plugin: nil - prime: - - -* - build-packages: - - git - - git-extras - override-pull: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - version="9999" - - if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then - version=$(echo ${GITHUB_REF:-} | cut -d/ -f3) - git checkout ${version} - fi - - snapcraftctl set-grade "stable" - snapcraftctl set-version "${version}" - - mkdir -p $(dirname ${kata_dir}) - ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir} - - docker: - after: [metadata] - plugin: nil - prime: - - -* - build-packages: - - ca-certificates - - containerd - - curl - - gnupg - - lsb-release - - runc - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\ - sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - distro_codename=$(lsb_release -cs) - echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get -y update - sudo apt-get -y install docker-ce docker-ce-cli containerd.io - - echo "Unmasking docker service" - sudo -E systemctl unmask docker.service || true - sudo -E systemctl unmask docker.socket || true - echo "Adding $USER into docker group" - sudo -E gpasswd -a $USER docker - echo "Starting docker" - # docker may fail to start using "fd://" in docker.service - sudo sed -i 's/fd:\/\//unix:\/\//g' /lib/systemd/system/docker.service - sudo systemctl daemon-reload - sudo -E systemctl start docker || true - - image: - after: [docker] - plugin: nil - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make rootfs-image-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-image.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - - sudo -E NO_TTY=true make rootfs-initrd-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-initrd.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - - runtime: - after: [docker] - plugin: nil - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make shim-v2-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-shim-v2.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - mkdir -p "${SNAPCRAFT_PART_INSTALL}/usr/bin" - ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/containerd-shim-kata-v2" "${SNAPCRAFT_PART_INSTALL}/usr/bin/containerd-shim-kata-v2" - ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-runtime" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-runtime" - ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-collect-data.sh" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-collect-data.sh" - - kernel: - after: [docker] - plugin: nil - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make kernel-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - qemu: - plugin: make - after: [docker] - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make qemu-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-qemu.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - virtiofsd: - plugin: nil - after: [docker] - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make virtiofsd-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-virtiofsd.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - cloud-hypervisor: - plugin: nil - after: [docker] - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make cloud-hypervisor-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - fi - -apps: - runtime: - command: usr/bin/kata-runtime - shim: - command: usr/bin/containerd-shim-kata-v2 - collect-data: - command: usr/bin/kata-collect-data.sh diff --git a/src/runtime/README.md b/src/runtime/README.md index c8aeec0ce..d2290e200 100644 --- a/src/runtime/README.md +++ b/src/runtime/README.md @@ -32,8 +32,6 @@ to work seamlessly with both Docker and Kubernetes respectively. ## Download and install -[![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers) - See the [installation guides](../../docs/install/README.md) available for various operating systems. diff --git a/tools/packaging/.gitignore b/tools/packaging/.gitignore index 82abc0533..2a056e4ba 100644 --- a/tools/packaging/.gitignore +++ b/tools/packaging/.gitignore @@ -1,7 +1,6 @@ *.img *.initrd *.sha256 -*.snap *.tar.gz *.tar.xz *.gz @@ -13,7 +12,6 @@ debian.series parts/ prime/ sha256sums.asc -snap/.snapcraft/ stage/ typescript kata-linux-* diff --git a/tools/packaging/Makefile b/tools/packaging/Makefile index f8444e8f1..4bd1e0334 100644 --- a/tools/packaging/Makefile +++ b/tools/packaging/Makefile @@ -8,13 +8,11 @@ MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) SED := sed YQ := $(MK_DIR)/yq -SNAPCRAFT_FILE := ../../snap/snapcraft.yaml VERSIONS_YAML_FILE := ../../versions.yaml VERSION_FILE := ../../VERSION export MK_DIR export YQ -export SNAPCRAFT_FILE export VERSION_FILE export VERSIONS_YAML_FILE @@ -25,10 +23,4 @@ test-static-build: $(YQ): @bash -c "source scripts/lib.sh; install_yq $${MK_DIR}" -snap: $(YQ) - @if [ "$$(cat $(VERSION_FILE))" != "$$($(YQ) r $(SNAPCRAFT_FILE) version)" ]; then \ - >&2 echo "Warning: $(SNAPCRAFT_FILE) version is different to upstream $(VERSION_FILE) file"; \ - fi - snapcraft -d --destructive-mode - -.PHONY: test-static-build snap +.PHONY: test-static-build diff --git a/tools/packaging/README.md b/tools/packaging/README.md index 8a31c0a80..b1c7ae594 100644 --- a/tools/packaging/README.md +++ b/tools/packaging/README.md @@ -12,10 +12,6 @@ Kata build artifacts are available within a container image, created by a [`kata-deploy`](kata-deploy), which make installation of Kata Containers in a running Kubernetes Cluster very straightforward. -## Build a snap package - -See [the snap documentation](../../snap/local). - ## Build static binaries See [the static build documentation](static-build). diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md index 5c8580a68..32aa7a9e2 100644 --- a/tools/packaging/kernel/README.md +++ b/tools/packaging/kernel/README.md @@ -9,8 +9,7 @@ automates the process to build a kernel for Kata Containers. The `build-kernel.sh` script requires an installed Golang version matching the [component build requirements](../../../docs/Developer-Guide.md#requirements-to-build-individual-components). It also requires [yq](https://github.com/mikefarah/yq) version 3.4.1 -> **Hint**: `snap install yq --channel=v3/stable` \ -> **or** `go install github.com/mikefarah/yq/v3@latest` +> **Hint**: `go install github.com/mikefarah/yq/v3@latest` The Linux kernel scripts further require a few packages (flex, bison, and libelf-dev) From fa6dff9f70892aa16ec176953219daecff3d728d Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Wed, 15 Feb 2023 22:02:09 +0800 Subject: [PATCH 50/76] feat(runtime-rs): support vcpu resizing on runtime side Support vcpu resizing on runtime side: 1. Calculate vcpu numbers in resource_manager using all the containers' linux_resources in the spec. 2. Call the hypervisor(vmm) to do the vcpu resize. 3. Call the agent to online vcpus. Fixes: #5030 Signed-off-by: Ji-Xinyou Signed-off-by: Yushuo --- src/libs/kata-types/src/cpu.rs | 2 +- src/libs/kata-types/src/utils/u32_set.rs | 2 +- src/runtime-rs/Cargo.toml | 1 + src/runtime-rs/crates/agent/src/kata/agent.rs | 3 +- src/runtime-rs/crates/agent/src/lib.rs | 1 + src/runtime-rs/crates/hypervisor/Cargo.toml | 2 +- .../crates/hypervisor/src/dragonball/inner.rs | 49 ++++- .../crates/hypervisor/src/dragonball/mod.rs | 6 + .../hypervisor/src/dragonball/vmm_instance.rs | 10 +- src/runtime-rs/crates/hypervisor/src/lib.rs | 1 + .../crates/hypervisor/src/qemu/inner.rs | 5 + .../crates/hypervisor/src/qemu/mod.rs | 5 + .../crates/resource/src/cpu_mem/cpu.rs | 188 ++++++++++++++++++ .../crates/resource/src/cpu_mem/mod.rs | 7 + src/runtime-rs/crates/resource/src/lib.rs | 8 + src/runtime-rs/crates/resource/src/manager.rs | 21 +- .../crates/resource/src/manager_inner.rs | 39 +++- .../runtimes/common/src/runtime_handler.rs | 1 + .../crates/runtimes/common/src/sandbox.rs | 4 +- src/runtime-rs/crates/runtimes/src/manager.rs | 9 +- .../src/container_manager/container.rs | 3 + .../src/container_manager/manager.rs | 7 +- .../runtimes/virt_container/src/sandbox.rs | 2 +- 23 files changed, 348 insertions(+), 28 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs create mode 100644 src/runtime-rs/crates/resource/src/cpu_mem/mod.rs diff --git a/src/libs/kata-types/src/cpu.rs b/src/libs/kata-types/src/cpu.rs index 0020de097..e47681f62 100644 --- a/src/libs/kata-types/src/cpu.rs +++ b/src/libs/kata-types/src/cpu.rs @@ -26,7 +26,7 @@ pub enum Error { } /// Assigned CPU resources for a Linux container. -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub struct LinuxContainerCpuResources { shares: u64, period: u64, diff --git a/src/libs/kata-types/src/utils/u32_set.rs b/src/libs/kata-types/src/utils/u32_set.rs index 3742e4d54..44c55a163 100644 --- a/src/libs/kata-types/src/utils/u32_set.rs +++ b/src/libs/kata-types/src/utils/u32_set.rs @@ -13,7 +13,7 @@ use crate::Error; /// /// The `U32Set` may be used to save CPUs parsed from a CPU list file or NUMA nodes parsed from /// a NUMA node list file. -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub struct U32Set(Vec); impl U32Set { diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index 185912145..bbc401f64 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -3,3 +3,4 @@ members = [ "crates/shim", "crates/shim-ctl", ] + diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index d06da15ea..13ba4085d 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -117,5 +117,6 @@ impl_agent!( get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None, set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None, get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None, - resize_volume | crate::ResizeVolumeRequest | crate::Empty | None + resize_volume | crate::ResizeVolumeRequest | crate::Empty | None, + online_cpu_mem | crate::OnlineCPUMemRequest | crate::Empty | None ); diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index ea3bab78f..43449ca59 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -54,6 +54,7 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { // sandbox async fn create_sandbox(&self, req: CreateSandboxRequest) -> Result; async fn destroy_sandbox(&self, req: Empty) -> Result; + async fn online_cpu_mem(&self, req: OnlineCPUMemRequest) -> Result; // network async fn add_arp_neighbors(&self, req: AddArpNeighborRequest) -> Result; diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index eb613aad3..6a61faea4 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -32,7 +32,7 @@ kata-types = { path = "../../../libs/kata-types" } logging = { path = "../../../libs/logging" } shim-interface = { path = "../../../libs/shim-interface" } -dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs","dbs-upcall"] } +dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "dbs-upcall"] } ch-config = { path = "ch-config", optional = true } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 45b77f09e..beb99e1f3 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -13,7 +13,7 @@ use crate::{ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use dragonball::{ - api::v1::{BlockDeviceConfigInfo, BootSourceConfig}, + api::v1::{BlockDeviceConfigInfo, BootSourceConfig, VcpuResizeInfo}, vm::VmConfigInfo, }; use kata_sys_util::mount; @@ -327,6 +327,53 @@ impl DragonballInner { } } + // check if resizing info is valid + // the error in this function is not ok to be tolerated, the container boot will fail + fn precheck_resize_vcpus(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + // old_vcpus > 0, safe for conversion + let current_vcpus = old_vcpus as u32; + + // a non-zero positive is required + if new_vcpus == 0 { + return Err(anyhow!("resize vcpu error: 0 vcpu resizing is invalid")); + } + + // cannot exceed maximum value + if new_vcpus > self.config.cpu_info.default_maxvcpus { + return Err(anyhow!("resize vcpu error: cannot greater than maxvcpus")); + } + + Ok((current_vcpus, new_vcpus)) + } + + // do the check before resizing, returns Result<(old, new)> + pub(crate) async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + if old_vcpus == new_vcpus { + info!( + sl!(), + "resize_vcpu: no need to resize vcpus because old_vcpus is equal to new_vcpus" + ); + return Ok((new_vcpus, new_vcpus)); + } + + let (old_vcpus, new_vcpus) = self.precheck_resize_vcpus(old_vcpus, new_vcpus)?; + info!( + sl!(), + "check_resize_vcpus passed, passing new_vcpus = {:?} to vmm", new_vcpus + ); + + let cpu_resize_info = VcpuResizeInfo { + vcpu_count: Some(new_vcpus as u8), + }; + self.vmm_instance + .resize_vcpu(&cpu_resize_info) + .context(format!( + "failed to do_resize_vcpus on new_vcpus={:?}", + new_vcpus + ))?; + Ok((old_vcpus, new_vcpus)) + } + pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) { self.config = config; } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index c6df95cc9..707294937 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -77,6 +77,12 @@ impl Hypervisor for Dragonball { inner.save_vm().await } + // returns Result<(old_vcpus, new_vcpus)> + async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + let inner = self.inner.read().await; + inner.resize_vcpu(old_vcpus, new_vcpus).await + } + async fn add_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.add_device(device).await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 4c40b40c5..ad3977eca 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -16,8 +16,8 @@ use crossbeam_channel::{unbounded, Receiver, Sender}; use dragonball::{ api::v1::{ BlockDeviceConfigInfo, BootSourceConfig, FsDeviceConfigInfo, FsMountConfigInfo, - InstanceInfo, InstanceState, VirtioNetDeviceConfigInfo, VmmAction, VmmActionError, VmmData, - VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, + InstanceInfo, InstanceState, VcpuResizeInfo, VirtioNetDeviceConfigInfo, VmmAction, + VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, }, vm::VmConfigInfo, Vmm, @@ -248,6 +248,12 @@ impl VmmInstance { Ok(()) } + pub fn resize_vcpu(&self, cfg: &VcpuResizeInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::ResizeVcpu(cfg.clone()))) + .with_context(|| format!("Failed to resize_vm(hotplug vcpu), cfg: {:?}", cfg))?; + Ok(()) + } + pub fn pause(&self) -> Result<()> { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 2001433e5..2386d0924 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -78,6 +78,7 @@ pub trait Hypervisor: Send + Sync { async fn pause_vm(&self) -> Result<()>; async fn save_vm(&self) -> Result<()>; async fn resume_vm(&self) -> Result<()>; + async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)>; // returns (old_vcpus, new_vcpus) // device manager async fn add_device(&self, device: DeviceType) -> Result<()>; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 456bf6373..4b1f7cae3 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -104,6 +104,11 @@ impl QemuInner { todo!() } + pub(crate) async fn resize_vcpu(&self, _old_vcpus: u32, _new_vcpus: u32) -> Result<(u32, u32)> { + info!(sl!(), "QemuInner::resize_vcpu()"); + todo!() + } + pub(crate) async fn get_pids(&self) -> Result> { info!(sl!(), "QemuInner::get_pids()"); todo!() diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 77217f153..eb657dc2d 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -118,6 +118,11 @@ impl Hypervisor for Qemu { inner.cleanup().await } + async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + let inner = self.inner.read().await; + inner.resize_vcpu(old_vcpus, new_vcpus).await + } + async fn get_pids(&self) -> Result> { let inner = self.inner.read().await; inner.get_pids().await diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs new file mode 100644 index 000000000..7892c014e --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs @@ -0,0 +1,188 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + cmp, + collections::{HashMap, HashSet}, + convert::TryFrom, + sync::Arc, +}; + +use agent::{Agent, OnlineCPUMemRequest}; +use anyhow::{Context, Ok, Result}; +use hypervisor::Hypervisor; +use kata_types::{config::TomlConfig, cpu::LinuxContainerCpuResources}; +use oci::LinuxCpu; +use tokio::sync::RwLock; + +use crate::ResourceUpdateOp; + +#[derive(Default, Debug, Clone)] +pub struct CpuResource { + /// Current number of vCPUs + pub(crate) current_vcpu: Arc>, + + /// Default number of vCPUs + pub(crate) default_vcpu: u32, + + /// CpuResource of each container + pub(crate) container_cpu_resources: Arc>>, +} + +impl CpuResource { + pub fn new(config: Arc) -> Result { + let hypervisor_name = config.runtime.hypervisor_name.clone(); + let hypervisor_config = config + .hypervisor + .get(&hypervisor_name) + .context(format!("failed to get hypervisor {}", hypervisor_name))?; + Ok(Self { + current_vcpu: Arc::new(RwLock::new(hypervisor_config.cpu_info.default_vcpus as u32)), + default_vcpu: hypervisor_config.cpu_info.default_vcpus as u32, + container_cpu_resources: Arc::new(RwLock::new(HashMap::new())), + }) + } + + pub(crate) async fn update_cpu_resources( + &self, + cid: &str, + linux_cpus: Option<&LinuxCpu>, + op: ResourceUpdateOp, + hypervisor: &dyn Hypervisor, + agent: &dyn Agent, + ) -> Result<()> { + self.update_container_cpu_resources(cid, linux_cpus, op) + .await + .context("update container cpu resources")?; + let vcpu_required = self + .calc_cpu_resources() + .await + .context("calculate vcpus required")?; + + if vcpu_required == self.current_vcpu().await { + return Ok(()); + } + + let curr_vcpus = self + .do_update_cpu_resources(vcpu_required, op, hypervisor, agent) + .await?; + self.update_current_vcpu(curr_vcpus).await; + Ok(()) + } + + async fn current_vcpu(&self) -> u32 { + let current_vcpu = self.current_vcpu.read().await; + *current_vcpu + } + + async fn update_current_vcpu(&self, new_vcpus: u32) { + let mut current_vcpu = self.current_vcpu.write().await; + *current_vcpu = new_vcpus; + } + + // update container_cpu_resources field + async fn update_container_cpu_resources( + &self, + cid: &str, + linux_cpus: Option<&LinuxCpu>, + op: ResourceUpdateOp, + ) -> Result<()> { + if let Some(cpu) = linux_cpus { + let container_resource = LinuxContainerCpuResources::try_from(cpu)?; + let mut resources = self.container_cpu_resources.write().await; + match op { + ResourceUpdateOp::Add => { + resources.insert(cid.to_owned(), container_resource); + } + ResourceUpdateOp::Update => { + let resource = resources.insert(cid.to_owned(), container_resource.clone()); + if let Some(old_container_resource) = resource { + // the priority of cpu-quota is higher than cpuset when determine the number of vcpus. + // we should better ignore the resource update when update cpu only by cpuset if cpu-quota + // has been set previously. + if old_container_resource.quota() > 0 && container_resource.quota() < 0 { + resources.insert(cid.to_owned(), old_container_resource); + } + } + } + ResourceUpdateOp::Del => { + resources.remove(cid); + } + } + } + + Ok(()) + } + + // calculates the total required vcpus by adding each container's requirements within the pod + async fn calc_cpu_resources(&self) -> Result { + let mut total_vcpu = 0; + let mut cpuset_vcpu: HashSet = HashSet::new(); + + let resources = self.container_cpu_resources.read().await; + for (_, cpu_resource) in resources.iter() { + let vcpu = cpu_resource.get_vcpus().unwrap_or(0) as u32; + cpuset_vcpu.extend(cpu_resource.cpuset().iter()); + total_vcpu += vcpu; + } + + // contrained only by cpuset + if total_vcpu == 0 && !cpuset_vcpu.is_empty() { + info!(sl!(), "(from cpuset)get vcpus # {:?}", cpuset_vcpu); + return Ok(cpuset_vcpu.len() as u32); + } + + info!( + sl!(), + "(from cfs_quota&cfs_period)get vcpus count {}", total_vcpu + ); + Ok(total_vcpu) + } + + // do hotplug and hot-unplug the vcpu + async fn do_update_cpu_resources( + &self, + new_vcpus: u32, + op: ResourceUpdateOp, + hypervisor: &dyn Hypervisor, + agent: &dyn Agent, + ) -> Result { + let old_vcpus = self.current_vcpu().await; + + // when adding vcpus, ignore old_vcpus > new_vcpus + // when deleting vcpus, ignore old_vcpus < new_vcpus + if (op == ResourceUpdateOp::Add && old_vcpus > new_vcpus) + || (op == ResourceUpdateOp::Del && old_vcpus < new_vcpus) + { + return Ok(old_vcpus); + } + + // do not reduce computing power + // the number of vcpus would not be lower than the default size + let new_vcpus = cmp::max(new_vcpus, self.default_vcpu); + + let (old, new) = hypervisor + .resize_vcpu(old_vcpus, new_vcpus) + .await + .context("resize vcpus")?; + + if old < new { + let add = new - old; + info!(sl!(), "request to onlineCpuMem with {:?} cpus", add); + + agent + .online_cpu_mem(OnlineCPUMemRequest { + wait: false, + nb_cpus: add, + cpu_only: true, + }) + .await + .context("online vcpus")?; + } + + Ok(new) + } +} diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs new file mode 100644 index 000000000..22656753d --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub mod cpu; diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs index b118e2e38..e008c357e 100644 --- a/src/runtime-rs/crates/resource/src/lib.rs +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -22,6 +22,7 @@ pub mod rootfs; pub mod share_fs; pub mod volume; pub use manager::ResourceManager; +pub mod cpu_mem; use kata_types::config::hypervisor::SharedFsInfo; @@ -30,3 +31,10 @@ pub enum ResourceConfig { Network(NetworkConfig), ShareFs(SharedFsInfo), } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResourceUpdateOp { + Add, + Del, + Update, +} diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index a022f722f..78c3d8e22 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -4,9 +4,8 @@ // SPDX-License-Identifier: Apache-2.0 // -use crate::network::NetworkConfig; -use crate::resource_persist::ResourceState; -use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig}; +use std::sync::Arc; + use agent::types::Device; use agent::{Agent, Storage}; use anyhow::Result; @@ -17,9 +16,13 @@ use kata_types::config::TomlConfig; use kata_types::mount::Mount; use oci::{Linux, LinuxResources}; use persist::sandbox_persist::Persist; -use std::sync::Arc; use tokio::sync::RwLock; +use crate::network::NetworkConfig; +use crate::resource_persist::ResourceState; +use crate::ResourceUpdateOp; +use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig}; + pub struct ManagerArgs { pub sid: String, pub agent: Arc, @@ -119,6 +122,16 @@ impl ResourceManager { inner.update_cgroups(cid, linux_resources).await } + pub async fn update_linux_resource( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, + ) -> Result<()> { + let inner = self.inner.read().await; + inner.update_linux_resource(cid, linux_resources, op).await + } + pub async fn cleanup(&self) -> Result<()> { let inner = self.inner.read().await; inner.cleanup().await diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 0c77ec142..3eced8340 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -4,13 +4,11 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::{sync::Arc, thread, vec}; +use std::{sync::Arc, thread}; -use crate::{network::NetworkConfig, resource_persist::ResourceState}; use agent::{types::Device, Agent, Storage}; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; - use hypervisor::{ device::{ device_manager::{do_handle_device, DeviceManager}, @@ -20,18 +18,20 @@ use hypervisor::{ }; use kata_types::config::TomlConfig; use kata_types::mount::Mount; -use oci::{Linux, LinuxResources}; +use oci::{Linux, LinuxCpu, LinuxResources}; use persist::sandbox_persist::Persist; use tokio::{runtime, sync::RwLock}; use crate::{ cgroups::{CgroupArgs, CgroupsResource}, + cpu_mem::cpu::CpuResource, manager::ManagerArgs, - network::{self, Network}, + network::{self, Network, NetworkConfig}, + resource_persist::ResourceState, rootfs::{RootFsResource, Rootfs}, share_fs::{self, sandbox_bind_mounts::SandboxBindMounts, ShareFs}, volume::{Volume, VolumeResource}, - ResourceConfig, + ResourceConfig, ResourceUpdateOp, }; pub(crate) struct ResourceManagerInner { @@ -46,6 +46,7 @@ pub(crate) struct ResourceManagerInner { pub rootfs_resource: RootFsResource, pub volume_resource: VolumeResource, pub cgroups_resource: CgroupsResource, + pub cpu_resource: CpuResource, } impl ResourceManagerInner { @@ -55,12 +56,12 @@ impl ResourceManagerInner { hypervisor: Arc, toml_config: Arc, ) -> Result { - let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; - // create device manager let dev_manager = DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?; + let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; + let cpu_resource = CpuResource::new(toml_config.clone())?; Ok(Self { sid: sid.to_string(), toml_config, @@ -72,6 +73,7 @@ impl ResourceManagerInner { rootfs_resource: RootFsResource::new(), volume_resource: VolumeResource::new(), cgroups_resource, + cpu_resource, }) } @@ -354,6 +356,26 @@ impl ResourceManagerInner { self.rootfs_resource.dump().await; self.volume_resource.dump().await; } + + pub async fn update_linux_resource( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, + ) -> Result<()> { + let linux_cpus = || -> Option<&LinuxCpu> { linux_resources.as_ref()?.cpu.as_ref() }(); + + self.cpu_resource + .update_cpu_resources( + cid, + linux_cpus, + op, + self.hypervisor.as_ref(), + self.agent.as_ref(), + ) + .await?; + Ok(()) + } } #[async_trait] @@ -400,6 +422,7 @@ impl Persist for ResourceManagerInner { ) .await?, toml_config: Arc::new(TomlConfig::default()), + cpu_resource: CpuResource::default(), }) } } diff --git a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs index c12df38b1..80e4149c3 100644 --- a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs +++ b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: Apache-2.0 // + use std::sync::Arc; use anyhow::Result; diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index efe06fa43..1a79f23d6 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -26,12 +26,10 @@ pub trait Sandbox: Send + Sync { async fn cleanup(&self) -> Result<()>; async fn shutdown(&self) -> Result<()>; - // agent function - async fn agent_sock(&self) -> Result; - // utils async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result>; async fn get_iptables(&self, is_ipv6: bool) -> Result>; async fn direct_volume_stats(&self, volume_path: &str) -> Result; async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()>; + async fn agent_sock(&self) -> Result; } diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 1fa7ce4a0..683be3647 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -6,7 +6,6 @@ use std::{path::PathBuf, str::from_utf8, sync::Arc}; -use crate::{shim_mgmt::server::MgmtServer, static_resource::StaticResourceManager}; use anyhow::{anyhow, Context, Result}; use common::{ message::Message, @@ -18,12 +17,11 @@ use kata_sys_util::spec::load_oci_spec; use kata_types::{ annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig, }; -use netns_rs::NetNs; -use resource::network::generate_netns_name; - #[cfg(feature = "linux")] use linux_container::LinuxContainer; +use netns_rs::NetNs; use persist::sandbox_persist::Persist; +use resource::network::generate_netns_name; use shim_interface::shim_mgmt::ERR_NO_SHIM_SERVER; use tokio::fs; use tokio::sync::{mpsc::Sender, RwLock}; @@ -36,6 +34,9 @@ use virt_container::{ #[cfg(feature = "wasm")] use wasm_container::WasmContainer; +use crate::shim_mgmt::server::MgmtServer; +use crate::static_resource::StaticResourceManager; + struct RuntimeHandlerManagerInner { id: String, msg_sender: Sender, diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 6dfea9fd0..165cda63e 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -42,6 +42,7 @@ pub struct Container { agent: Arc, resource_manager: Arc, logger: slog::Logger, + pub linux_resources: Option, } impl Container { @@ -51,6 +52,7 @@ impl Container { spec: oci::Spec, agent: Arc, resource_manager: Arc, + linux_resources: Option, ) -> Result { let container_id = ContainerID::new(&config.container_id).context("new container id")?; let logger = sl!().new(o!("container_id" => config.container_id.clone())); @@ -78,6 +80,7 @@ impl Container { agent, resource_manager, logger, + linux_resources, }) } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index f85e7901a..1b00713dd 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -6,6 +6,7 @@ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; + use std::{collections::HashMap, sync::Arc}; use agent::Agent; @@ -59,12 +60,17 @@ impl VirtContainerManager { #[async_trait] impl ContainerManager for VirtContainerManager { async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result { + let linux_resources = match spec.linux.clone() { + Some(linux) => linux.resources, + _ => None, + }; let container = Container::new( self.pid, config.clone(), spec.clone(), self.agent.clone(), self.resource_manager.clone(), + linux_resources, ) .context("new container")?; @@ -96,7 +102,6 @@ impl ContainerManager for VirtContainerManager { let mut containers = self.containers.write().await; container.create(spec).await.context("create")?; containers.insert(container.container_id.to_string(), container); - Ok(PID { pid: self.pid }) } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 8202d854d..3cbdba493 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -361,7 +361,7 @@ impl Sandbox for VirtSandbox { .await .context("resource clean up")?; - // TODO: cleanup other snadbox resource + // TODO: cleanup other sandbox resource Ok(()) } From a39e1e6cd17615ee68b3c16d886150328ac685f0 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Sat, 6 May 2023 17:14:36 +0800 Subject: [PATCH 51/76] feat(runtime-rs): merge the update_cgroups in update_linux_resources Updating vCPU resources and memory resources of the sandbox and updating cgroups on the host will always happening together, and they are all updated based on the linux resources declarations of all the containers. So we merge update_cgroups into the update_linux_resources, so we can better manage the resources allocated to one pod in the host. Fixes: #5030 Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- .../crates/resource/src/cgroups/mod.rs | 58 +++++++++++++------ src/runtime-rs/crates/resource/src/lib.rs | 2 +- src/runtime-rs/crates/resource/src/manager.rs | 9 --- .../crates/resource/src/manager_inner.rs | 37 ++++++------ .../src/container_manager/container.rs | 11 +++- 5 files changed, 68 insertions(+), 49 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs index b7f515d7f..d32fa479d 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/mod.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs @@ -26,6 +26,8 @@ use oci::LinuxResources; use persist::sandbox_persist::Persist; use tokio::sync::RwLock; +use crate::ResourceUpdateOp; + const OS_ERROR_NO_SUCH_PROCESS: i32 = 3; pub struct CgroupArgs { @@ -149,29 +151,51 @@ impl CgroupsResource { &self, cid: &str, linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, h: &dyn Hypervisor, ) -> Result<()> { - let resource = self.calc_resource(linux_resources); - let changed = self.update_resources(cid, resource).await; + let new_resources = self.calc_resource(linux_resources); + let old_resources = self.update_resources(cid, new_resources.clone(), op).await; - if !changed { - return Ok(()); - } - - self.do_update_cgroups(h).await - } - - async fn update_resources(&self, cid: &str, new_resource: Resources) -> bool { - let mut resources = self.resources.write().await; - let old_resource = resources.insert(cid.to_owned(), new_resource.clone()); - - if let Some(old_resource) = old_resource { - if old_resource == new_resource { - return false; + if let Some(old_resource) = old_resources.clone() { + if old_resource == new_resources { + return Ok(()); } } - true + match self.do_update_cgroups(h).await { + Err(e) => { + // if update failed, we should roll back the records in resources + let mut resources = self.resources.write().await; + match op { + ResourceUpdateOp::Add => { + resources.remove(cid); + } + ResourceUpdateOp::Update | ResourceUpdateOp::Del => { + if let Some(old_resource) = old_resources { + resources.insert(cid.to_owned(), old_resource.clone()); + } + } + } + Err(e) + } + Ok(()) => Ok(()), + } + } + + async fn update_resources( + &self, + cid: &str, + new_resource: Resources, + op: ResourceUpdateOp, + ) -> Option { + let mut resources = self.resources.write().await; + match op { + ResourceUpdateOp::Add | ResourceUpdateOp::Update => { + resources.insert(cid.to_owned(), new_resource.clone()) + } + ResourceUpdateOp::Del => resources.remove(cid), + } } async fn do_update_cgroups(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs index e008c357e..4e4aae9e8 100644 --- a/src/runtime-rs/crates/resource/src/lib.rs +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -32,7 +32,7 @@ pub enum ResourceConfig { ShareFs(SharedFsInfo), } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum ResourceUpdateOp { Add, Del, diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index 78c3d8e22..6345dbff6 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -113,15 +113,6 @@ impl ResourceManager { inner.dump().await } - pub async fn update_cgroups( - &self, - cid: &str, - linux_resources: Option<&LinuxResources>, - ) -> Result<()> { - let inner = self.inner.read().await; - inner.update_cgroups(cid, linux_resources).await - } - pub async fn update_linux_resource( &self, cid: &str, diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 3eced8340..d565368eb 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -317,17 +317,7 @@ impl ResourceManagerInner { sb_bindmnt.cleanup_sandbox_bind_mounts() } } - - pub async fn update_cgroups( - &self, - cid: &str, - linux_resources: Option<&LinuxResources>, - ) -> Result<()> { - self.cgroups_resource - .update_cgroups(cid, linux_resources, self.hypervisor.as_ref()) - .await - } - + pub async fn cleanup(&self) -> Result<()> { // clean up cgroup self.cgroups_resource @@ -365,15 +355,24 @@ impl ResourceManagerInner { ) -> Result<()> { let linux_cpus = || -> Option<&LinuxCpu> { linux_resources.as_ref()?.cpu.as_ref() }(); - self.cpu_resource - .update_cpu_resources( - cid, - linux_cpus, - op, - self.hypervisor.as_ref(), - self.agent.as_ref(), - ) + // if static_sandbox_resource_mgmt, we will not have to update sandbox's cpu or mem resource + if !self.toml_config.runtime.static_sandbox_resource_mgmt { + self.cpu_resource + .update_cpu_resources( + cid, + linux_cpus, + op, + self.hypervisor.as_ref(), + self.agent.as_ref(), + ) + .await?; + } + + // we should firstly update the vcpus and mems, and then update the host cgroups + self.cgroups_resource + .update_cgroups(cid, linux_resources, op, self.hypervisor.as_ref()) .await?; + Ok(()) } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 165cda63e..cbde460a5 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -19,7 +19,7 @@ use common::{ use kata_sys_util::k8s::update_ephemeral_storage_type; use oci::{LinuxResources, Process as OCIProcess}; -use resource::ResourceManager; +use resource::{ResourceManager, ResourceUpdateOp}; use tokio::sync::RwLock; use super::{ @@ -155,11 +155,12 @@ impl Container { // update cgroups self.resource_manager - .update_cgroups( + .update_linux_resource( &config.container_id, spec.linux .as_ref() .and_then(|linux| linux.resources.as_ref()), + ResourceUpdateOp::Add, ) .await?; @@ -402,7 +403,11 @@ impl Container { pub async fn update(&self, resources: &LinuxResources) -> Result<()> { self.resource_manager - .update_cgroups(&self.config.container_id, Some(resources)) + .update_linux_resource( + &self.config.container_id, + Some(resources), + ResourceUpdateOp::Update, + ) .await?; let req = agent::UpdateContainerRequest { From a0385e138310b63b3a0f97b053a793aa963484d9 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Sat, 6 May 2023 17:43:36 +0800 Subject: [PATCH 52/76] feat(runtime-rs): update linux resource when stop_process Update the resource when delete container, which is in stop_process in runtime-rs. Fixes: #5030 Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- .../src/container_manager/container.rs | 32 ++++++++++++++----- .../src/container_manager/container_inner.rs | 10 +++++- .../src/container_manager/manager.rs | 5 --- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index cbde460a5..f4c42b0ba 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -42,7 +42,6 @@ pub struct Container { agent: Arc, resource_manager: Arc, logger: slog::Logger, - pub linux_resources: Option, } impl Container { @@ -52,7 +51,6 @@ impl Container { spec: oci::Spec, agent: Arc, resource_manager: Arc, - linux_resources: Option, ) -> Result { let container_id = ContainerID::new(&config.container_id).context("new container id")?; let logger = sl!().new(o!("container_id" => config.container_id.clone())); @@ -66,6 +64,10 @@ impl Container { config.stderr.clone(), config.terminal, ); + let linux_resources = spec + .linux + .as_ref() + .and_then(|linux| linux.resources.clone()); Ok(Self { pid, @@ -76,11 +78,11 @@ impl Container { agent.clone(), init_process, logger.clone(), + linux_resources, ))), agent, resource_manager, logger, - linux_resources, }) } @@ -153,13 +155,11 @@ impl Container { .handler_devices(&config.container_id, linux) .await?; - // update cgroups + // update vcpus, mems and host cgroups self.resource_manager .update_linux_resource( &config.container_id, - spec.linux - .as_ref() - .and_then(|linux| linux.resources.as_ref()), + inner.linux_resources.as_ref(), ResourceUpdateOp::Add, ) .await?; @@ -327,7 +327,20 @@ impl Container { inner .stop_process(container_process, true, &device_manager) .await - .context("stop process") + .context("stop process")?; + + // update vcpus, mems and host cgroups + if container_process.process_type == ProcessType::Container { + self.resource_manager + .update_linux_resource( + &self.config.container_id, + inner.linux_resources.as_ref(), + ResourceUpdateOp::Del, + ) + .await?; + } + + Ok(()) } pub async fn pause(&self) -> Result<()> { @@ -402,6 +415,9 @@ impl Container { } pub async fn update(&self, resources: &LinuxResources) -> Result<()> { + let mut inner = self.inner.write().await; + inner.linux_resources = Some(resources.clone()); + // update vcpus, mems and host cgroups self.resource_manager .update_linux_resource( &self.config.container_id, diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index 12d4810fb..bc478cbcd 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -14,6 +14,7 @@ use common::{ }; use hypervisor::device::device_manager::DeviceManager; use nix::sys::signal::Signal; +use oci::LinuxResources; use resource::{rootfs::Rootfs, volume::Volume}; use tokio::sync::RwLock; @@ -32,10 +33,16 @@ pub struct ContainerInner { pub(crate) exec_processes: HashMap, pub(crate) rootfs: Vec>, pub(crate) volumes: Vec>, + pub(crate) linux_resources: Option, } impl ContainerInner { - pub(crate) fn new(agent: Arc, init_process: Process, logger: slog::Logger) -> Self { + pub(crate) fn new( + agent: Arc, + init_process: Process, + logger: slog::Logger, + linux_resources: Option, + ) -> Self { Self { agent, logger, @@ -43,6 +50,7 @@ impl ContainerInner { exec_processes: HashMap::new(), rootfs: vec![], volumes: vec![], + linux_resources, } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index 1b00713dd..f5aa05e6c 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -60,17 +60,12 @@ impl VirtContainerManager { #[async_trait] impl ContainerManager for VirtContainerManager { async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result { - let linux_resources = match spec.linux.clone() { - Some(linux) => linux.resources, - _ => None, - }; let container = Container::new( self.pid, config.clone(), spec.clone(), self.agent.clone(), self.resource_manager.clone(), - linux_resources, ) .context("new container")?; From d66f7572dd196ee28289b6015d305e21fe091cd8 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Tue, 9 May 2023 10:27:11 +0800 Subject: [PATCH 53/76] feat(runtime-rs): clear cpuset in runtime side The declaration of the cpu number in the cpuset is greater than the actual number of vcpus, which will cause an error when updating the cgroup in the guest. This problem is difficult to solve, so we temporarily clean up the cpuset in the container spec before passing in the agent. Fixes: #5030 Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- src/runtime-rs/crates/agent/src/kata/trans.rs | 2 +- src/runtime-rs/crates/agent/src/types.rs | 2 +- src/runtime-rs/crates/resource/src/manager.rs | 2 +- .../crates/resource/src/manager_inner.rs | 26 +++++++++++++++++-- .../src/container_manager/container.rs | 11 +++++--- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 172095ceb..ca9c8f172 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -336,7 +336,7 @@ impl From for agent::UpdateContainerRequest { fn from(from: UpdateContainerRequest) -> Self { Self { container_id: from.container_id, - resources: from_option(Some(from.resources)), + resources: from_option(from.resources), ..Default::default() } } diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index da6e14430..1ba7efd5e 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -201,7 +201,7 @@ pub struct ListProcessesRequest { #[derive(PartialEq, Clone, Default)] pub struct UpdateContainerRequest { pub container_id: String, - pub resources: oci::LinuxResources, + pub resources: Option, pub mounts: Vec, } diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index 6345dbff6..d79de40cd 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -118,7 +118,7 @@ impl ResourceManager { cid: &str, linux_resources: Option<&LinuxResources>, op: ResourceUpdateOp, - ) -> Result<()> { + ) -> Result> { let inner = self.inner.read().await; inner.update_linux_resource(cid, linux_resources, op).await } diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index d565368eb..8c0e06d19 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -352,7 +352,7 @@ impl ResourceManagerInner { cid: &str, linux_resources: Option<&LinuxResources>, op: ResourceUpdateOp, - ) -> Result<()> { + ) -> Result> { let linux_cpus = || -> Option<&LinuxCpu> { linux_resources.as_ref()?.cpu.as_ref() }(); // if static_sandbox_resource_mgmt, we will not have to update sandbox's cpu or mem resource @@ -373,7 +373,29 @@ impl ResourceManagerInner { .update_cgroups(cid, linux_resources, op, self.hypervisor.as_ref()) .await?; - Ok(()) + // update the linux resources for agent + self.agent_linux_resources(linux_resources) + } + + fn agent_linux_resources( + &self, + linux_resources: Option<&LinuxResources>, + ) -> Result> { + let mut resources = match linux_resources { + Some(linux_resources) => linux_resources.clone(), + None => { + return Ok(None); + } + }; + + // clear the cpuset + // for example, if there are only 5 vcpus now, and the cpuset in LinuxResources is 0-2,6, guest os will report + // error when creating the container. so we choose to clear the cpuset here. + if let Some(cpu) = &mut resources.cpu { + cpu.cpus = String::new(); + } + + Ok(Some(resources)) } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index f4c42b0ba..fba45a784 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -156,13 +156,17 @@ impl Container { .await?; // update vcpus, mems and host cgroups - self.resource_manager + let resources = self + .resource_manager .update_linux_resource( &config.container_id, inner.linux_resources.as_ref(), ResourceUpdateOp::Add, ) .await?; + if let Some(linux) = &mut spec.linux { + linux.resources = resources; + } // create container let r = agent::CreateContainerRequest { @@ -418,7 +422,8 @@ impl Container { let mut inner = self.inner.write().await; inner.linux_resources = Some(resources.clone()); // update vcpus, mems and host cgroups - self.resource_manager + let agent_resources = self + .resource_manager .update_linux_resource( &self.config.container_id, Some(resources), @@ -428,7 +433,7 @@ impl Container { let req = agent::UpdateContainerRequest { container_id: self.container_id.container_id.clone(), - resources: resources.clone(), + resources: agent_resources, mounts: Vec::new(), }; self.agent From aaa96c749b80d1b3056beac32087981449909e79 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Tue, 9 May 2023 14:35:44 +0800 Subject: [PATCH 54/76] feat(runtime-rs): modify onlineCpuMemRequest Some vmms, such as dragonball, will actively help us perform online cpu operations when doing cpu hotplug. Under the old onlineCpuMem interface, it is difficult to adapt to this situation. So we modify the semantics of nb_cpus in onlineCpuMemRequest. In the original semantics, nb_cpus represents the number of newly added CPUs that need to be online. The modified semantics become that the number of online CPUs in the guest needs to be guaranteed. Fixes: #5030 Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- src/agent/src/linux_abi.rs | 3 +- src/agent/src/sandbox.rs | 44 ++++++++++++++----- src/libs/protocols/protos/agent.proto | 3 +- .../crates/resource/src/cpu_mem/cpu.rs | 2 +- src/runtime/virtcontainers/agent.go | 3 +- .../pkg/agent/protocols/grpc/agent.pb.go | 3 +- src/runtime/virtcontainers/sandbox.go | 5 +-- src/runtime/virtcontainers/vm.go | 2 +- 8 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index 042acd0ae..de131faf0 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -81,7 +81,8 @@ cfg_if! { // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00 pub const ACPI_DEV_PATH: &str = "/devices/LNXSYSTM"; -pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu"; +pub const SYSFS_CPU_PATH: &str = "/sys/devices/system/cpu"; +pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu/online"; pub const SYSFS_MEMORY_BLOCK_SIZE_PATH: &str = "/sys/devices/system/memory/block_size_bytes"; pub const SYSFS_MEMORY_HOTPLUG_PROBE_PATH: &str = "/sys/devices/system/memory/probe"; diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 34275cc41..175304ed1 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -12,6 +12,7 @@ use crate::pci; use crate::uevent::{Uevent, UeventMatcher}; use crate::watcher::BindWatcher; use anyhow::{anyhow, Context, Result}; +use kata_types::cpu::CpuSet; use libc::pid_t; use oci::{Hook, Hooks}; use protocols::agent::OnlineCPUMemRequest; @@ -25,6 +26,7 @@ use std::collections::HashMap; use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use std::{thread, time}; use tokio::sync::mpsc::{channel, Receiver, Sender}; @@ -263,12 +265,12 @@ impl Sandbox { pub fn online_cpu_memory(&self, req: &OnlineCPUMemRequest) -> Result<()> { if req.nb_cpus > 0 { // online cpus - online_cpus(&self.logger, req.nb_cpus as i32)?; + online_cpus(&self.logger, req.nb_cpus as i32).context("online cpus")?; } if !req.cpu_only { // online memory - online_memory(&self.logger)?; + online_memory(&self.logger).context("online memory")?; } if req.nb_cpus == 0 { @@ -432,23 +434,33 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res // max wait for all CPUs to online will use 50 * 100 = 5 seconds. const ONLINE_CPUMEM_WATI_MILLIS: u64 = 50; -const ONLINE_CPUMEM_MAX_RETRIES: u32 = 100; +const ONLINE_CPUMEM_MAX_RETRIES: i32 = 100; #[instrument] fn online_cpus(logger: &Logger, num: i32) -> Result { - let mut onlined_count: i32 = 0; + let mut onlined_cpu_count = onlined_cpus().context("onlined cpu count")?; + // for some vmms, like dragonball, they will online cpus for us + // so check first whether agent need to do the online operation + if onlined_cpu_count >= num { + return Ok(num); + } for i in 0..ONLINE_CPUMEM_MAX_RETRIES { - let r = online_resources( + // online num resources + online_resources( logger, - SYSFS_CPU_ONLINE_PATH, + SYSFS_CPU_PATH, r"cpu[0-9]+", - num - onlined_count, - ); + num - onlined_cpu_count, + ) + .context("online cpu resource")?; - onlined_count += r?; - if onlined_count == num { - info!(logger, "online {} CPU(s) after {} retries", num, i); + onlined_cpu_count = onlined_cpus().context("onlined cpu count")?; + if onlined_cpu_count >= num { + info!( + logger, + "Currently {} onlined CPU(s) after {} retries", onlined_cpu_count, i + ); return Ok(num); } thread::sleep(time::Duration::from_millis(ONLINE_CPUMEM_WATI_MILLIS)); @@ -463,10 +475,18 @@ fn online_cpus(logger: &Logger, num: i32) -> Result { #[instrument] fn online_memory(logger: &Logger) -> Result<()> { - online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)?; + online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1) + .context("online memory resource")?; Ok(()) } +fn onlined_cpus() -> Result { + let content = + fs::read_to_string(SYSFS_CPU_ONLINE_PATH).context("read sysfs cpu online file")?; + let online_cpu_set = CpuSet::from_str(content.trim())?; + Ok(online_cpu_set.len() as i32) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index 3ad755256..9ed7c0a04 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -366,7 +366,8 @@ message OnlineCPUMemRequest { // resources are connected asynchronously and the agent returns immediately. bool wait = 1; - // NbCpus specifies the number of CPUs that were added and the agent has to online. + // NbCpus specifies the number of CPUs that should be onlined in the guest. + // Special value 0 means agent will skip this check. uint32 nb_cpus = 2; // CpuOnly specifies whether only online CPU or not. diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs index 7892c014e..661e1a5e3 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs @@ -176,7 +176,7 @@ impl CpuResource { agent .online_cpu_mem(OnlineCPUMemRequest { wait: false, - nb_cpus: add, + nb_cpus: new, cpu_only: true, }) .await diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index ddf11d9ce..8f0aca26e 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -10,6 +10,7 @@ import ( "time" "context" + persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" @@ -119,7 +120,7 @@ type agent interface { // onlineCPUMem will online CPUs and Memory inside the Sandbox. // This function should be called after hot adding vCPUs or Memory. - // cpus specifies the number of CPUs that were added and the agent should online + // cpus specifies the number of CPUs that should be onlined in the guest, and special value 0 means agent will skip this check. // cpuOnly specifies that we should online cpu or online memory or both onlineCPUMem(ctx context.Context, cpus uint32, cpuOnly bool) error diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index a8dd81104..3a1b7c607 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1924,7 +1924,8 @@ type OnlineCPUMemRequest struct { // If true the agent returns once all resources have been connected, otherwise all // resources are connected asynchronously and the agent returns immediately. Wait bool `protobuf:"varint,1,opt,name=wait,proto3" json:"wait,omitempty"` - // NbCpus specifies the number of CPUs that were added and the agent has to online. + // NbCpus specifies the number of CPUs that should be onlined in the guest. + // Special value 0 means agent will skip this check. NbCpus uint32 `protobuf:"varint,2,opt,name=nb_cpus,json=nbCpus,proto3" json:"nb_cpus,omitempty"` // CpuOnly specifies whether only online CPU or not. CpuOnly bool `protobuf:"varint,3,opt,name=cpu_only,json=cpuOnly,proto3" json:"cpu_only,omitempty"` diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index b0697fd84..9bd5f402e 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -2117,9 +2117,8 @@ func (s *Sandbox) updateResources(ctx context.Context) error { s.Logger().Debugf("Request to hypervisor to update oldCPUs/newCPUs: %d/%d", oldCPUs, newCPUs) // If the CPUs were increased, ask agent to online them if oldCPUs < newCPUs { - vcpusAdded := newCPUs - oldCPUs - s.Logger().Debugf("Request to onlineCPUMem with %d CPUs", vcpusAdded) - if err := s.agent.onlineCPUMem(ctx, vcpusAdded, true); err != nil { + s.Logger().Debugf("Request to onlineCPUMem with %d CPUs", newCPUs) + if err := s.agent.onlineCPUMem(ctx, newCPUs, true); err != nil { return err } } diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go index b5dec9912..72afa9581 100644 --- a/src/runtime/virtcontainers/vm.go +++ b/src/runtime/virtcontainers/vm.go @@ -293,7 +293,7 @@ func (v *VM) AddMemory(ctx context.Context, numMB uint32) error { // OnlineCPUMemory puts the hotplugged CPU and memory online. func (v *VM) OnlineCPUMemory(ctx context.Context) error { v.logger().Infof("online CPU %d and memory", v.cpuDelta) - err := v.agent.onlineCPUMem(ctx, v.cpuDelta, false) + err := v.agent.onlineCPUMem(ctx, v.cpu, false) if err == nil { v.cpuDelta = 0 } From 67972ec48a40434fd7c3883c4538cc1d328ede40 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Thu, 11 May 2023 11:34:05 +0800 Subject: [PATCH 55/76] feat(runtime-rs): calculate initial size In this commit, we refactored the logic of static resource management. We defined the sandbox size calculated from PodSandbox's annotation and SingleContainer's spec as initial size, which will always be the sandbox size when booting the VM. The configuration static_sandbox_resource_mgmt controls whether we will modify the sandbox size in the following container operation. Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- src/runtime-rs/crates/resource/Cargo.toml | 1 + .../src/cpu_mem/initial_size.rs} | 69 ++++++++++--------- .../crates/resource/src/cpu_mem/mod.rs | 1 + .../crates/resource/src/manager_inner.rs | 2 +- src/runtime-rs/crates/runtimes/src/lib.rs | 1 - src/runtime-rs/crates/runtimes/src/manager.rs | 16 ++--- 6 files changed, 46 insertions(+), 44 deletions(-) rename src/runtime-rs/crates/{runtimes/src/static_resource.rs => resource/src/cpu_mem/initial_size.rs} (82%) diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 9847ce61b..22ffda48b 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -41,4 +41,5 @@ logging = { path = "../../../libs/logging" } oci = { path = "../../../libs/oci" } actix-rt = "2.7.0" persist = { path = "../persist"} + [features] diff --git a/src/runtime-rs/crates/runtimes/src/static_resource.rs b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs similarity index 82% rename from src/runtime-rs/crates/runtimes/src/static_resource.rs rename to src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs index 453ce85b3..53eccc52b 100644 --- a/src/runtime-rs/crates/runtimes/src/static_resource.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs @@ -13,17 +13,16 @@ use kata_types::{ cpu::LinuxContainerCpuResources, k8s::container_type, }; -// static resource that StaticResourceManager needs, this is the spec for the +// initial resource that InitialSizeManager needs, this is the spec for the // sandbox/container's workload #[derive(Clone, Copy, Debug)] -struct StaticResource { +struct InitialSize { vcpu: u32, mem_mb: u32, } -// generate static resource(vcpu and memory in MiB) from spec's information -// used for static resource management -impl TryFrom<&oci::Spec> for StaticResource { +// generate initial resource(vcpu and memory in MiB) from spec's information +impl TryFrom<&oci::Spec> for InitialSize { type Error = anyhow::Error; fn try_from(spec: &oci::Spec) -> Result { let mut vcpu: u32 = 0; @@ -65,31 +64,32 @@ impl TryFrom<&oci::Spec> for StaticResource { } info!( sl!(), - "static resource mgmt result: vcpu={}, mem_mb={}", vcpu, mem_mb + "(from PodSandbox's annotation / SingleContainer's spec) initial size: vcpu={}, mem_mb={}", vcpu, mem_mb ); Ok(Self { vcpu, mem_mb }) } } -// StaticResourceManager is responsible for static resource management +// InitialSizeManager is responsible for initial vcpu/mem management // -// static resource management sizing information is optionally provided, either by +// inital vcpu/mem management sizing information is optionally provided, either by // upper layer runtime (containerd / crio) or by the container spec itself (when it // is a standalone single container such as the one started with *docker run*) // // the sizing information uses three values, cpu quota, cpu period and memory limit, -// and with above values it calculates the # vcpus and memory for the workload and -// add them to default value of the config +// and with above values it calculates the # vcpus and memory for the workload +// +// if the workload # of vcpus and memory is invalid for vmms, we still use default +// value in toml_config #[derive(Clone, Copy, Debug)] -pub struct StaticResourceManager { - resource: StaticResource, +pub struct InitialSizeManager { + resource: InitialSize, } -impl StaticResourceManager { +impl InitialSizeManager { pub fn new(spec: &oci::Spec) -> Result { Ok(Self { - resource: StaticResource::try_from(spec) - .context("failed to construct static resource")?, + resource: InitialSize::try_from(spec).context("failed to construct static resource")?, }) } @@ -100,8 +100,13 @@ impl StaticResourceManager { .hypervisor .get_mut(hypervisor_name) .context("failed to get hypervisor config")?; - hv.cpu_info.default_vcpus += self.resource.vcpu as i32; - hv.memory_info.default_memory += self.resource.mem_mb; + + if self.resource.vcpu > 0 { + hv.cpu_info.default_vcpus = self.resource.vcpu as i32 + } + if self.resource.mem_mb > 0 { + hv.memory_info.default_memory = self.resource.mem_mb; + } Ok(()) } } @@ -151,7 +156,7 @@ mod tests { struct TestData<'a> { desc: &'a str, input: InputData, - result: StaticResource, + result: InitialSize, } fn get_test_data() -> Vec> { @@ -163,7 +168,7 @@ mod tests { quota: None, memory: None, }, - result: StaticResource { vcpu: 0, mem_mb: 0 }, + result: InitialSize { vcpu: 0, mem_mb: 0 }, }, TestData { desc: "normal resource limit", @@ -173,7 +178,7 @@ mod tests { quota: Some(220_000), memory: Some(1024 * 1024 * 512), }, - result: StaticResource { + result: InitialSize { vcpu: 3, mem_mb: 512, }, @@ -183,7 +188,7 @@ mod tests { } #[test] - fn test_static_resource_mgmt_sandbox() { + fn test_initial_size_sandbox() { let tests = get_test_data(); // run tests @@ -210,22 +215,22 @@ mod tests { ..Default::default() }; - let static_resource = StaticResource::try_from(&spec); + let initial_size = InitialSize::try_from(&spec); assert!( - static_resource.is_ok(), + initial_size.is_ok(), "test[{}]: {:?} should be ok", i, d.desc ); - let static_resource = static_resource.unwrap(); + let initial_size = initial_size.unwrap(); assert_eq!( - static_resource.vcpu, d.result.vcpu, + initial_size.vcpu, d.result.vcpu, "test[{}]: {:?} vcpu should be {}", i, d.desc, d.result.vcpu, ); assert_eq!( - static_resource.mem_mb, d.result.mem_mb, + initial_size.mem_mb, d.result.mem_mb, "test[{}]: {:?} memory should be {}", i, d.desc, d.result.mem_mb, ); @@ -233,7 +238,7 @@ mod tests { } #[test] - fn test_static_resource_mgmt_container() { + fn test_initial_size_container() { let tests = get_test_data(); // run tests @@ -261,22 +266,22 @@ mod tests { ..Default::default() }; - let static_resource = StaticResource::try_from(&spec); + let initial_size = InitialSize::try_from(&spec); assert!( - static_resource.is_ok(), + initial_size.is_ok(), "test[{}]: {:?} should be ok", i, d.desc ); - let static_resource = static_resource.unwrap(); + let initial_size = initial_size.unwrap(); assert_eq!( - static_resource.vcpu, d.result.vcpu, + initial_size.vcpu, d.result.vcpu, "test[{}]: {:?} vcpu should be {}", i, d.desc, d.result.vcpu, ); assert_eq!( - static_resource.mem_mb, d.result.mem_mb, + initial_size.mem_mb, d.result.mem_mb, "test[{}]: {:?} memory should be {}", i, d.desc, d.result.mem_mb, ); diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs index 22656753d..f2984cd1c 100644 --- a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs @@ -5,3 +5,4 @@ // pub mod cpu; +pub mod initial_size; diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 8c0e06d19..b60925c11 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -317,7 +317,7 @@ impl ResourceManagerInner { sb_bindmnt.cleanup_sandbox_bind_mounts() } } - + pub async fn cleanup(&self) -> Result<()> { // clean up cgroup self.cgroups_resource diff --git a/src/runtime-rs/crates/runtimes/src/lib.rs b/src/runtime-rs/crates/runtimes/src/lib.rs index dffa69697..867c8ef9e 100644 --- a/src/runtime-rs/crates/runtimes/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/src/lib.rs @@ -13,4 +13,3 @@ pub mod manager; pub use manager::RuntimeHandlerManager; pub use shim_interface; mod shim_mgmt; -mod static_resource; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 683be3647..d05e1961f 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -21,7 +21,7 @@ use kata_types::{ use linux_container::LinuxContainer; use netns_rs::NetNs; use persist::sandbox_persist::Persist; -use resource::network::generate_netns_name; +use resource::{cpu_mem::initial_size::InitialSizeManager, network::generate_netns_name}; use shim_interface::shim_mgmt::ERR_NO_SHIM_SERVER; use tokio::fs; use tokio::sync::{mpsc::Sender, RwLock}; @@ -35,7 +35,6 @@ use virt_container::{ use wasm_container::WasmContainer; use crate::shim_mgmt::server::MgmtServer; -use crate::static_resource::StaticResourceManager; struct RuntimeHandlerManagerInner { id: String, @@ -423,14 +422,11 @@ fn load_config(spec: &oci::Spec, option: &Option>) -> Result // 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."), // then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is // a single container. - if toml_config.runtime.static_sandbox_resource_mgmt { - info!(sl!(), "static resource management enabled"); - let static_resource_manager = StaticResourceManager::new(spec) - .context("failed to construct static resource manager")?; - static_resource_manager - .setup_config(&mut toml_config) - .context("failed to setup static resource mgmt config")?; - } + let initial_size_manager = + InitialSizeManager::new(spec).context("failed to construct static resource manager")?; + initial_size_manager + .setup_config(&mut toml_config) + .context("failed to setup static resource mgmt config")?; info!(sl!(), "get config content {:?}", &toml_config); Ok(toml_config) From 7b1e67819cf3117aeede59a5b8886bb41a4c2de1 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Tue, 9 May 2023 19:08:01 +0800 Subject: [PATCH 56/76] fix(clippy): fix clippy error Fixes: #5030 Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs | 4 ++++ src/runtime-rs/crates/hypervisor/src/ch/mod.rs | 5 +++++ src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs | 2 +- src/runtime-rs/crates/resource/src/cgroups/mod.rs | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 9cf5bcd8c..a54a6a43e 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -494,6 +494,10 @@ impl CloudHypervisorInner { Ok(()) } + pub(crate) async fn resize_vcpu(&self, old_vcpu: u32, new_vcpu: u32) -> Result<(u32, u32)> { + Ok((old_vcpu, new_vcpu)) + } + pub(crate) async fn get_pids(&self) -> Result> { Ok(Vec::::new()) } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index a4b8b05ff..dd95413fc 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -114,6 +114,11 @@ impl Hypervisor for CloudHypervisor { inner.cleanup().await } + async fn resize_vcpu(&self, old_vcpu: u32, new_vcpu: u32) -> Result<(u32, u32)> { + let inner = self.inner.read().await; + inner.resize_vcpu(old_vcpu, new_vcpu).await + } + async fn get_pids(&self) -> Result> { let inner = self.inner.read().await; inner.get_pids().await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index beb99e1f3..09afc4c1a 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -331,7 +331,7 @@ impl DragonballInner { // the error in this function is not ok to be tolerated, the container boot will fail fn precheck_resize_vcpus(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { // old_vcpus > 0, safe for conversion - let current_vcpus = old_vcpus as u32; + let current_vcpus = old_vcpus; // a non-zero positive is required if new_vcpus == 0 { diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs index d32fa479d..831e30c0f 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/mod.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs @@ -173,7 +173,7 @@ impl CgroupsResource { } ResourceUpdateOp::Update | ResourceUpdateOp::Del => { if let Some(old_resource) = old_resources { - resources.insert(cid.to_owned(), old_resource.clone()); + resources.insert(cid.to_owned(), old_resource); } } } From ae2cfa8263fcff5a472710a2b80b6740c963e726 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Fri, 9 Jun 2023 18:08:47 +0800 Subject: [PATCH 57/76] doc: add vcpu handlint doc for runtime-rs Kubernetes and Containerd will help calculate the Sandbox Size and pass it to Kata Containers through annotations. In order to accommodate this favorable change and be compatible with the past, we have implemented the handling of the number of vCPUs in runtime-rs. This is This is slightly different from the original runtime-go design. This doc introduce how we handle vCPU size in runtime-rs. Fixes: #5030 Signed-off-by: Yushuo Signed-off-by: Ji-Xinyou --- docs/Limitations.md | 3 +- docs/design/README.md | 3 +- docs/design/VSocks.md | 2 +- ...andling.md => vcpu-handling-runtime-go.md} | 0 docs/design/vcpu-handling-runtime-rs.md | 51 +++++++++++++++++++ src/runtime-rs/Cargo.lock | 15 +++--- 6 files changed, 64 insertions(+), 10 deletions(-) rename docs/design/{vcpu-handling.md => vcpu-handling-runtime-go.md} (100%) create mode 100644 docs/design/vcpu-handling-runtime-rs.md diff --git a/docs/Limitations.md b/docs/Limitations.md index d122e5ca5..74a6acf2d 100644 --- a/docs/Limitations.md +++ b/docs/Limitations.md @@ -147,7 +147,8 @@ these commands is potentially challenging. See issue https://github.com/clearcontainers/runtime/issues/341 and [the constraints challenge](#the-constraints-challenge) for more information. For CPUs resource management see -[CPU constraints](design/vcpu-handling.md). +[CPU constraints(in runtime-go)](design/vcpu-handling-runtime-go.md). +[CPU constraints(in runtime-rs)](design/vcpu-handling-runtime-rs.md). # Architectural limitations diff --git a/docs/design/README.md b/docs/design/README.md index 0c732defd..2fe93b5f6 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -6,7 +6,8 @@ Kata Containers design documents: - [API Design of Kata Containers](kata-api-design.md) - [Design requirements for Kata Containers](kata-design-requirements.md) - [VSocks](VSocks.md) -- [VCPU handling](vcpu-handling.md) +- [VCPU handling(in runtime-go)](vcpu-handling-runtime-go.md) +- [VCPU handling(in runtime-rs)](vcpu-handling-runtime-rs.md) - [VCPU threads pinning](vcpu-threads-pinning.md) - [Host cgroups](host-cgroups.md) - [Agent systemd cgroup](agent-systemd-cgroup.md) diff --git a/docs/design/VSocks.md b/docs/design/VSocks.md index 0271645c2..9375f30f5 100644 --- a/docs/design/VSocks.md +++ b/docs/design/VSocks.md @@ -78,4 +78,4 @@ with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, i the containers are removed automatically. [1]: https://wiki.qemu.org/Features/VirtioVsock -[2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods +[2]: ./vcpu-handling-runtime-go.md#virtual-cpus-and-kubernetes-pods diff --git a/docs/design/vcpu-handling.md b/docs/design/vcpu-handling-runtime-go.md similarity index 100% rename from docs/design/vcpu-handling.md rename to docs/design/vcpu-handling-runtime-go.md diff --git a/docs/design/vcpu-handling-runtime-rs.md b/docs/design/vcpu-handling-runtime-rs.md new file mode 100644 index 000000000..44989ce4e --- /dev/null +++ b/docs/design/vcpu-handling-runtime-rs.md @@ -0,0 +1,51 @@ +# Virtual machine vCPU sizing in Kata Containers 3.0 + +> Preview: +> [Kubernetes(since 1.23)][1] and [Containerd(since 1.6.0-beta4)][2] will help calculate `Sandbox Size` info and pass it to Kata Containers through annotations. +> In order to adapt to this beneficial change and be compatible with the past, we have implemented the new vCPUs handling way in `runtime-rs`, which is slightly different from the original `runtime-go`'s design. + +## When do we need to handle vCPUs size? +vCPUs sizing should be determined by the container workloads. So throughout the life cycle of Kata Containers, there are several points in time when we need to think about how many vCPUs should be at the time. Mainly including the time points of `CreateVM`, `CreateContainer`, `UpdateContainer`, and `DeleteContainer`. +* `CreateVM`: When creating a sandbox, we need to know how many vCPUs to start the VM with. +* `CreateContainer`: When creating a new container in the VM, we may need to hot-plug the vCPUs according to the requirements in container's spec. +* `UpdateContainer`: When receiving the `UpdateContainer` request, we may need to update the vCPU resources according to the new requirements of the container. +* `DeleteContainer`: When a container is removed from the VM, we may need to hot-unplug the vCPUs to reclaim the vCPU resources introduced by the container. + +## On what basis do we calculate the number of vCPUs? +When Kata calculate the number of vCPUs, We have three data sources, the `default_vcpus` and `default_maxvcpus` specified in the configuration file (named `TomlConfig` later in the doc), the `io.kubernetes.cri.sandbox-cpu-quota` and `io.kubernetes.cri.sandbox-cpu-period` annotations passed by the upper layer runtime, and the corresponding CPU resource part in the container's spec for the container when `CreateContainer`/`UpdateContainer`/`DeleteContainer` is requested. + +Our understanding and priority of these resources are as follows, which will affect how we calculate the number of vCPUs later. + +* From `TomlConfig`: + * `default_vcpus`: default number of vCPUs when starting a VM. + * `default_maxvcpus`: maximum number of vCPUs. +* From `Annotation`: + * `InitialSize`: we call the size of the resource passed from the annotations as `InitialSize`. Kubernetes will calculate the sandbox size according to the Pod's statement, which is the `InitialSize` here. This size should be the size we want to prioritize. +* From `Container Spec`: + * The amount of CPU resources that the Container wants to use will be declared through the spec. Including the aforementioned annotations, we mainly consider `cpu quota` and `cpuset` when calculating the number of vCPUs. + * `cpu quota`: `cpu quota` is the most common way to declare the amount of CPU resources. The number of vCPUs introduced by `cpu quota` declared in a container's spec is: `vCPUs = ceiling( quota / period )`. + * `cpuset`: `cpuset` is often used to bind the CPUs that tasks can run on. The number of vCPUs may introduced by `cpuset` declared in a container's spec is the number of CPUs specified in the set that do not overlap with other containers. + + +## How to calculate and adjust the vCPUs size: +There are two types of vCPUs that we need to consider, one is the number of vCPUs when starting the VM (named `Boot Size` in the doc). The second is the number of vCPUs when `CreateContainer`/`UpdateContainer`/`DeleteContainer` request is received (`Real-time Size` in the doc). + +### `Boot Size` +The main considerations are `InitialSize` and `default_vcpus`. There are the following principles: +`InitialSize` has priority over `default_vcpus` declared in `TomlConfig`. +1. When there is such an annotation statement, the originally `default_vcpus` will be modified to the number of vCPUs in the `InitialSize` as the `Boot Size`. (Because not all runtimes support this annotation for the time being, we still keep the `default_cpus` in `TomlConfig`.) +2. When the specs of all containers are aggregated for sandbox size calculation, the method is consistent with the calculation method of `InitialSize` here. + +### `Real-time Size` +When we receive an OCI request, it may be for a single container. But what we have to consider is the number of vCPUs for the entire VM. So we will maintain a list. Every time there is a demand for adjustment, the entire list will be traversed to calculate a value for the number of vCPUs. In addition, there are the following principles: +1. Do not cut computing power and try to keep the number of vCPUs specified by `InitialSize`. + * So the number of vCPUs after will not be less than the `Boot Size`. +2. `cpu quota` takes precedence over `cpuset` and the setting history are took into account. + * We think quota describes the CPU time slice that a cgroup can use, and `cpuset` describes the actual CPU number that a cgroup can use. Quota can better describe the size of the CPU time slice that a cgroup actually wants to use. The `cpuset` only describes which CPUs the cgroup can use, but the cgroup can use the specified CPU but consumes a smaller time slice, so the quota takes precedence over the `cpuset`. + * On the one hand, when both `cpu quota` and `cpuset` are specified, we will calculate the number of vCPUs based on `cpu quota` and ignore `cpuset`. On the other hand, if `cpu quota` was used to control the number of vCPUs in the past, and only `cpuset` was updated during `UpdateContainer`, we will not adjust the number of vCPUs at this time. +3. `StaticSandboxResourceMgmt` controls hotplug. + * Some VMMs and kernels of some architectures do not support hotplugging. We can accommodate this situation through `StaticSandboxResourceMgmt`. When `StaticSandboxResourceMgmt = true` is set, we don't make any further attempts to update the number of vCPUs after booting. + + +[1]: https://github.com/kubernetes/kubernetes/pull/104886 +[2]: https://github.com/containerd/containerd/pull/6155 diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 0b5cd7ca9..358776fab 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -655,11 +655,12 @@ dependencies = [ [[package]] name = "dbs-address-space" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc" +checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322" dependencies = [ "arc-swap", + "lazy_static", "libc", "nix 0.23.2", "thiserror", @@ -746,9 +747,9 @@ dependencies = [ [[package]] name = "dbs-upcall" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f" +checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb" dependencies = [ "anyhow", "dbs-utils", @@ -776,9 +777,9 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668" +checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8" dependencies = [ "byteorder", "caps", @@ -792,7 +793,7 @@ dependencies = [ "kvm-ioctls", "libc", "log", - "nix 0.23.2", + "nix 0.24.3", "nydus-api", "nydus-blobfs", "nydus-rafs", From 0e2379909b0a543b9431f27f47e17a479b2dedff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Mon, 12 Jun 2023 11:24:45 -0700 Subject: [PATCH 58/76] gha: Fix `stage` definition in matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This defines `stage` as a list instead of a literal to fix the GHA CI. Fixes: #7086 Signed-off-by: Aurélien Bombo --- .github/workflows/build-kata-static-tarball-amd64.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 1ef136ff1..b5c7584fe 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -19,7 +19,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - stage: ${{ inputs.stage }} + stage: + - ${{ inputs.stage }} asset: - cloud-hypervisor - cloud-hypervisor-glibc From 21d22785399b676983d243650af10ad9d23c3f2e Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Mon, 12 Jun 2023 20:32:02 +0000 Subject: [PATCH 59/76] versions: Update firecracker version to 1.3.3 This PR updates the firecracker version to 1.3.3 which includes the following changes Fixed passing through cache information from host in CPUID leaf 0x80000006. A race condition that has been identified between the API thread and the VMM thread due to a misconfiguration of the api_event_fd. Fixes #7089 Signed-off-by: Gabriela Cervantes --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 3455221a8..ec9f8df7a 100644 --- a/versions.yaml +++ b/versions.yaml @@ -83,7 +83,7 @@ assets: uscan-url: >- https://github.com/firecracker-microvm/firecracker/tags .*/v?(\d\S+)\.tar\.gz - version: "v1.3.1" + version: "v1.3.3" qemu: description: "VMM that uses KVM" From 347385b4ee5861a1271ad2cc7081ec6a6f6c0449 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Tue, 13 Jun 2023 15:12:47 +0800 Subject: [PATCH 60/76] runtime-rs: Enhance flexibility of virtio-fs config support more and flexible options for inline virtiofs. Fixes: #7091 Signed-off-by: alex.lyn --- .../hypervisor/src/device/driver/virtio_fs.rs | 3 ++ .../hypervisor/src/dragonball/inner_device.rs | 35 +++++++++++++++---- .../resource/src/share_fs/share_virtio_fs.rs | 1 + 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs index d2d3cc762..0a97845e7 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs @@ -67,6 +67,9 @@ pub struct ShareFsDeviceConfig { /// queue_num: queue number pub queue_num: u64, + + /// options: virtiofs device's config options. + pub options: Vec, } #[derive(Debug, Clone)] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 94861536c..10e13b578 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -157,7 +157,11 @@ impl DragonballInner { .context("insert vsock") } - fn parse_inline_virtiofs_args(&self, fs_cfg: &mut FsDeviceConfigInfo) -> Result<()> { + fn parse_inline_virtiofs_args( + &self, + fs_cfg: &mut FsDeviceConfigInfo, + options: &mut Vec, + ) -> Result<()> { let mut debug = false; let mut opt_list = String::new(); @@ -169,8 +173,8 @@ impl DragonballInner { sl!(), "args: {:?}", &self.config.shared_fs.virtio_fs_extra_args ); - let args = &self.config.shared_fs.virtio_fs_extra_args; - let _ = go_flag::parse_args_with_warnings::(args, None, |flags| { + let mut args = self.config.shared_fs.virtio_fs_extra_args.clone(); + let _ = go_flag::parse_args_with_warnings::(&args, None, |flags| { flags.add_flag("d", &mut debug); flags.add_flag("thread-pool-size", &mut fs_cfg.thread_pool_size); flags.add_flag("drop-sys-resource", &mut fs_cfg.drop_sys_resource); @@ -178,6 +182,9 @@ impl DragonballInner { }) .with_context(|| format!("parse args: {:?}", args))?; + // more options parsed for inline virtio-fs' custom config + args.append(options); + if debug { warn!( sl!(), @@ -202,6 +209,7 @@ impl DragonballInner { "xattr" => fs_cfg.xattr = true, "no_xattr" => fs_cfg.xattr = false, "cache_symlinks" => {} // inline virtiofs always cache symlinks + "no_readdir" => fs_cfg.no_readdir = true, "trace" => warn!( sl!(), "Inline virtiofs \"-o trace\" option not supported yet, ignored." @@ -234,16 +242,25 @@ impl DragonballInner { xattr: true, ..Default::default() }; - self.do_add_fs_device(&config.fs_type, &mut fs_cfg) + + let mut options = config.options.clone(); + self.do_add_fs_device(&config.fs_type, &mut fs_cfg, &mut options) } - fn do_add_fs_device(&self, fs_type: &str, fs_cfg: &mut FsDeviceConfigInfo) -> Result<()> { + fn do_add_fs_device( + &self, + fs_type: &str, + fs_cfg: &mut FsDeviceConfigInfo, + options: &mut Vec, + ) -> Result<()> { match fs_type { VIRTIO_FS => { fs_cfg.mode = String::from("vhostuser"); } INLINE_VIRTIO_FS => { - self.parse_inline_virtiofs_args(fs_cfg)?; + // All parameters starting with --patch-fs do not need to be processed, these are the parameters required by patch fs + options.retain(|x| !x.starts_with("--patch-fs")); + self.parse_inline_virtiofs_args(fs_cfg, options)?; } _ => { return Err(anyhow!( @@ -311,8 +328,12 @@ mod tests { "--drop-sys-resource".to_string(), "-d".to_string(), ]; + + let mut options: Vec = Vec::new(); dragonball.config.shared_fs.virtio_fs_cache = "auto".to_string(); - dragonball.parse_inline_virtiofs_args(&mut fs_cfg).unwrap(); + dragonball + .parse_inline_virtiofs_args(&mut fs_cfg, &mut options) + .unwrap(); assert!(!fs_cfg.no_open); assert!(fs_cfg.xattr); diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs index 81ab8b7fd..9a5676bdf 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -56,6 +56,7 @@ pub(crate) async fn prepare_virtiofs( fs_type: fs_type.to_string(), queue_size: 0, queue_num: 0, + options: vec![], }, }; h.add_device(DeviceType::ShareFs(share_fs_device)) From 35e4938e8ceadb0cbc82d468765e693ced1d584c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 13 Jun 2023 10:56:49 -0700 Subject: [PATCH 61/76] tools: Fix no-op builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the builds of `cloud-hypervisor-glibc` and `rootfs-initrd-mariner` to properly create the `build/` directory. Fixes: #7098 Signed-off-by: Aurélien Bombo --- tools/packaging/kata-deploy/local-build/Makefile | 4 ++-- .../packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index c23f6b04e..fc7ed41d2 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -56,7 +56,7 @@ cloud-hypervisor-tarball: ${MAKE} $@-build cloud-hypervisor-glibc-tarball: - exit 0 + ${MAKE} $@-build firecracker-tarball: ${MAKE} $@-build @@ -110,7 +110,7 @@ rootfs-image-tdx-tarball: kernel-tdx-experimental-tarball ${MAKE} $@-build rootfs-initrd-mariner-tarball: - exit 0 + ${MAKE} $@-build rootfs-initrd-sev-tarball: kernel-sev-tarball ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index c65f4cd80..a552aed12 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -578,6 +578,8 @@ handle_build() { cloud-hypervisor) install_clh ;; + cloud-hypervisor-glibc) ;; + firecracker) install_firecracker ;; kernel) install_kernel ;; @@ -614,6 +616,8 @@ handle_build() { rootfs-initrd) install_initrd ;; + rootfs-initrd-mariner) ;; + rootfs-initrd-sev) install_initrd_sev ;; shim-v2) install_shimv2 ;; From dad731d5c10137b62c492f74c49dd8472b941950 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 13 Jun 2023 16:27:01 +0000 Subject: [PATCH 62/76] docs: Update Developer Guide This PR updates the developer guide at the connect to the debug console section. Fixes #7094 Signed-off-by: Gabriela Cervantes --- docs/Developer-Guide.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 7f0b3eaa7..2f60ab165 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -587,10 +587,15 @@ $ sudo kata-monitor #### Connect to debug console -Command `kata-runtime exec` is used to connect to the debug console. +You need to start a container for example: +```bash +$ sudo ctr run --runtime io.containerd.kata.v2 -d docker.io/library/ubuntu:latest testdebug +``` + +Then, you can use the command `kata-runtime exec ` to connect to the debug console. ``` -$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd +$ kata-runtime exec testdebug bash-4.2# id uid=0(root) gid=0(root) groups=0(root) bash-4.2# pwd From bc152b1141c9bb8e31d5eef5c9145de1b2fd2460 Mon Sep 17 00:00:00 2001 From: David Esparza Date: Tue, 13 Jun 2023 15:56:43 -0600 Subject: [PATCH 63/76] gha: ci-on-push: Run metrics tests This gh-workflow prints a simple msg, but is the base for future PRs that will gradually add the jobs corresponding to the kata metrics test. Fixes: #7100 Signed-off-by: David Esparza --- .github/workflows/ci-on-push.yaml | 4 +++ .../workflows/run-launchtimes-metrics.yaml | 15 ++++++++ tests/metrics/gha-run.sh | 35 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 .github/workflows/run-launchtimes-metrics.yaml create mode 100755 tests/metrics/gha-run.sh diff --git a/.github/workflows/ci-on-push.yaml b/.github/workflows/ci-on-push.yaml index 3951bba08..819a9b33c 100644 --- a/.github/workflows/ci-on-push.yaml +++ b/.github/workflows/ci-on-push.yaml @@ -52,3 +52,7 @@ jobs: registry: ghcr.io repo: ${{ github.repository_owner }}/kata-deploy-ci tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64 + + run-metrics-tests: + needs: build-kata-static-tarball-amd64 + uses: ./.github/workflows/run-launchtimes-metrics.yaml diff --git a/.github/workflows/run-launchtimes-metrics.yaml b/.github/workflows/run-launchtimes-metrics.yaml new file mode 100644 index 000000000..67045acf2 --- /dev/null +++ b/.github/workflows/run-launchtimes-metrics.yaml @@ -0,0 +1,15 @@ +name: CI | Run launch-times metrics +on: + workflow_call: + +jobs: + launch-times-tests: + runs-on: metrics + env: + GOPATH: ${{ github.workspace }} + steps: + - name: run launch times on qemu + run: bash tests/metrics/gha-run.sh run-test-launchtimes-qemu + + - name: run launch times on clh + run: bash tests/metrics/gha-run.sh run-test-launchtimes-clh diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh new file mode 100755 index 000000000..e3656b2cd --- /dev/null +++ b/tests/metrics/gha-run.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +metrics_dir="$(dirname "$(readlink -f "$0")")" + +function run_test_launchtimes() { + hypervisor="${1}" + + echo "Running launchtimes tests: " + + if [ "${hypervisor}" = 'qemu' ]; then + echo "qemu" + elif [ "${hypervisor}" = 'clh' ]; then + echo "clh" + fi +} + +function main() { + action="${1:-}" + case "${action}" in + run-test-launchtimes-qemu) run_test_launchtimes "qemu" ;; + run-test-launchtimes-clh) run_test_launchtimes "clh" ;; + *) >&2 echo "Invalid argument"; exit 2 ;; + esac +} + +main "$@" From b16e0de7342d4a205fbec41e648e4f5e6b86784c Mon Sep 17 00:00:00 2001 From: David Esparza Date: Thu, 15 Jun 2023 13:11:33 -0600 Subject: [PATCH 64/76] gha: Add base branch on SHA on pull requst The run-launchtimes-metrics workflow needs to get the commit ID for the last commit to the head branch of the PR. Fixes: #7116 Signed-off-by: David Esparza --- .github/workflows/run-launchtimes-metrics.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run-launchtimes-metrics.yaml b/.github/workflows/run-launchtimes-metrics.yaml index 67045acf2..788f3488f 100644 --- a/.github/workflows/run-launchtimes-metrics.yaml +++ b/.github/workflows/run-launchtimes-metrics.yaml @@ -8,6 +8,10 @@ jobs: env: GOPATH: ${{ github.workspace }} steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: run launch times on qemu run: bash tests/metrics/gha-run.sh run-test-launchtimes-qemu From c3043a6c608930db901733f81ffe39d9ed6cc28c Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 15 Jun 2023 16:19:04 +0000 Subject: [PATCH 65/76] tests: Add tests lib common script This PR adds the test lib common script that is going to be used for kata containers metrics. Fixes #7113 Signed-off-by: Gabriela Cervantes --- tests/metrics/lib/common.bash | 353 ++++++++++++++++++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100755 tests/metrics/lib/common.bash diff --git a/tests/metrics/lib/common.bash b/tests/metrics/lib/common.bash new file mode 100755 index 000000000..30c5c1321 --- /dev/null +++ b/tests/metrics/lib/common.bash @@ -0,0 +1,353 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +THIS_FILE=$(readlink -f ${BASH_SOURCE[0]}) +LIB_DIR=${THIS_FILE%/*} +RESULT_DIR="${LIB_DIR}/../results" + +source ${LIB_DIR}/../../lib/common.bash +source ${LIB_DIR}/json.bash +source /etc/os-release || source /usr/lib/os-release + +# Set variables to reasonable defaults if unset or empty +CTR_EXE="${CTR_EXE:-ctr}" +DOCKER_EXE="${DOCKER_EXE:-docker}" +CTR_RUNTIME="${CTR_RUNTIME:-io.containerd.kata.v2}" +RUNTIME="${RUNTIME:-containerd-shim-kata-v2}" +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +TEST_REPO="${TEST_REPO:-github.com/kata-containers/tests}" +JSON_HOST="${JSON_HOST:-}" + +KSM_BASE="/sys/kernel/mm/ksm" +KSM_ENABLE_FILE="${KSM_BASE}/run" +KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan" +KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs" +KSM_PAGES_SHARED="${KSM_BASE}/pages_shared" + +http_proxy="${http_proxy:-}" +https_proxy="${https_proxy:-}" + +# The settings we use for an 'aggresive' KSM setup +# Scan 1000 pages every 50ms - 20,000 pages/s +KSM_AGGRESIVE_PAGES=1000 +KSM_AGGRESIVE_SLEEP=50 + +declare -A registries +registries[ubuntu]=\ +"docker.io/library +public.ecr.aws/lts +mirror.gcr.io/library +quay.io/libpod" + +# This function checks existence of commands. +# They can be received standalone or as an array, e.g. +# +# cmds=(“cmd1” “cmd2”) +# check_cmds "${cmds[@]}" +check_cmds() +{ + local cmd req_cmds=( "$@" ) + for cmd in "${req_cmds[@]}"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + die "command $cmd not available" + fi + echo "command: $cmd: yes" + done +} + +# This function performs a pull on the image names +# passed in (notionally as 'about to be used'), to ensure +# - that we have the most upto date images +# - that any pull/refresh time (for a first pull) does not +# happen during the test itself. +# +# The image list can be received standalone or as an array, e.g. +# +# images=(“img1” “img2”) +# check_imgs "${images[@]}" +check_images() +{ + local img req_images=( "$@" ) + for img in "${req_images[@]}"; do + echo "ctr pull'ing: $img" + if ! sudo "${CTR_EXE}" image pull "$img"; then + die "Failed to pull image $img" + fi + echo "ctr pull'd: $img" + done +} + +generate_build_dockerfile() { + local dockerfile="$1" + local image="$2" + local map_key="$3" + local text_to_replace="$4" + local regs=(${registries["${map_key}"]}) + for r in ${regs[@]}; do + sed 's|'${text_to_replace}'|'${r}'|g' \ + "${dockerfile}.in" > "${dockerfile}" + if sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then + return 0 + fi + done + return 1 +} + +# This function performs a build on the image names +# passed in, to ensure that we have the latest changes from +# the dockerfiles +build_dockerfile_image() +{ + local image="$1" + local dockerfile_path="$2" + local dockerfile_dir=${2%/*} + + if [ -f "$dockerfile_path" ]; then + echo "docker building $image" + if ! sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then + die "Failed to docker build image $image" + fi + return 0 + fi + + generate_build_dockerfile "${dockerfile_path}" "${image}" "ubuntu" "@UBUNTU_REGISTRY@" \ + || die "Failed to docker build image $image" +} + +# This function removes the ctr image, builds a new one using a dockerfile +# and imports the image from docker to ctr +check_ctr_images() +{ + local ctr_image="$1" + local dockerfile_path="$2" + local docker_image="$(echo ${ctr_image} | cut -d/ -f3 | cut -d: -f1)" + + if [ -z "$ctr_image" ] || [ -z "$dockerfile_path" ]; then + die "Missing image or dockerfile path variable" + fi + + sudo "${CTR_EXE}" i rm "${ctr_image}" + build_dockerfile_image "${docker_image}" "${dockerfile_path}" + sudo "${DOCKER_EXE}" save -o "${docker_image}.tar" "${docker_image}" + sudo "${CTR_EXE}" i import "${docker_image}.tar" + rm -rf "${docker_image}".tar +} + +# A one time (per uber test cycle) init that tries to get the +# system to a 'known state' as much as possible +metrics_onetime_init() +{ + # The onetime init must be called once, and only once + if [ ! -z "$onetime_init_done" ]; then + die "onetime_init() called more than once" + fi + + # Restart services + restart_containerd_service + + # We want this to be seen in sub shells as well... + # otherwise init_env() cannot check us + export onetime_init_done=1 +} + +# Print a banner to the logs noting clearly which test +# we are about to run +test_banner() +{ + echo -e "\n===== starting test [$1] =====" +} + +# Initialization/verification environment. This function makes +# minimal steps for metrics/tests execution. +init_env() +{ + test_banner "${TEST_NAME}" + + cmd=("docker" "ctr") + + # check dependencies + check_cmds "${cmd[@]}" + + # Remove all stopped containers + clean_env + clean_env_ctr + + # This clean up is more aggressive, this is in order to + # decrease the factors that could affect the metrics results. + kill_processes_before_start +} + +# This function checks if there are containers or +# shim/proxy/hypervisor processes up, if found, they are +# killed to start test with clean environment. +kill_processes_before_start() { + DOCKER_PROCS=$(sudo "${DOCKER_EXE}" ps -q) + [[ -n "${DOCKER_PROCS}" ]] && clean_env + + CTR_PROCS=$(sudo "${CTR_EXE}" t list -q) + [[ -n "${CTR_PROCS}" ]] && clean_env_ctr + + check_processes +} + +# Generate a random name - generally used when creating containers, but can +# be used for any other appropriate purpose +random_name() { + mktemp -u kata-XXXXXX +} + +show_system_ctr_state() { + echo "Showing system state:" + echo " --Check containers--" + sudo "${CTR_EXE}" c list + echo " --Check tasks--" + sudo "${CTR_EXE}" task list + + local processes="containerd-shim-kata-v2" + + for p in ${processes}; do + echo " --pgrep ${p}--" + pgrep -a ${p} + done +} + +common_init(){ + if [ "$CTR_RUNTIME" == "io.containerd.kata.v2" ] || [ "$RUNTIME" == "containerd-shim-kata-v2" ]; then + extract_kata_env + else + # We know we have nothing to do for runc or shimv2 + if [ "$CTR_RUNTIME" != "io.containerd.runc.v2" ] || [ "$RUNTIME" != "runc" ]; then + warn "Unrecognised runtime" + fi + fi +} + + +# Save the current KSM settings so we can restore them later +save_ksm_settings(){ + echo "saving KSM settings" + ksm_stored_run=$(cat ${KSM_ENABLE_FILE}) + ksm_stored_pages=$(cat ${KSM_ENABLE_FILE}) + ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE}) +} + +set_ksm_aggressive(){ + echo "setting KSM to aggressive mode" + # Flip the run off/on to ensure a restart/rescan + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}" + sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}" + sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}" + + if [ "${KATA_HYPERVISOR}" == "qemu" ]; then + # Disable virtio-fs and save whether it was enabled previously + set_virtio_out=$(sudo -E PATH="$PATH" "${LIB_DIR}/../../.ci/set_kata_config.sh" shared_fs virtio-9p) + echo "${set_virtio_out}" + grep -q "already" <<< "${set_virtio_out}" || was_virtio_fs=true; + fi +} + +restore_virtio_fs(){ + # Re-enable virtio-fs if it was enabled previously + [ -n "${was_virtio_fs}" ] && sudo -E PATH="$PATH" "${LIB_DIR}/../../.ci/set_kata_config.sh" shared_fs virtio-fs || \ + info "Not restoring virtio-fs since it wasn't enabled previously" +} + +restore_ksm_settings(){ + echo "restoring KSM settings" + # First turn off the run to ensure if we are then re-enabling + # that any changes take effect + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}" + sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}" + sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}" + [ "${KATA_HYPERVISOR}" == "qemu" ] && restore_virtio_fs +} + +disable_ksm(){ + echo "disabling KSM" + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + [ "${KATA_HYPERVISOR}" == "qemu" ] && restore_virtio_fs +} + +# See if KSM is enabled. +# If so, amend the test name to reflect that +check_for_ksm(){ + if [ ! -f ${KSM_ENABLE_FILE} ]; then + return + fi + + ksm_on=$(< ${KSM_ENABLE_FILE}) + + if [ $ksm_on == "1" ]; then + TEST_NAME="${TEST_NAME} ksm" + fi +} + +# Wait for KSM to settle down, or timeout waiting +# The basic algorithm is to look at the pages_shared value +# at the end of every 'full scan', and if the value +# has changed very little, then we are done (because we presume +# a full scan has managed to do few new merges) +# +# arg1 - timeout in seconds +wait_ksm_settle(){ + [[ "$RUNTIME" == "runc" ]] || [[ "$CTR_RUNTIME" == "io.containerd.runc.v2" ]] && return + local t pcnt + local oldscan=-1 newscan + local oldpages=-1 newpages + + oldscan=$(cat /sys/kernel/mm/ksm/full_scans) + + # Wait some time for KSM to kick in to avoid early dismissal + for ((t=0; t<5; t++)); do + pages=$(cat "${KSM_PAGES_SHARED}") + [[ "$pages" -ne 0 ]] && echo "Discovered KSM activity" && break + sleep 1 + done + + # Go around the loop until either we see a small % change + # between two full_scans, or we timeout + for ((t=0; t<$1; t++)); do + + newscan=$(cat /sys/kernel/mm/ksm/full_scans) + newpages=$(cat "${KSM_PAGES_SHARED}") + [[ "$newpages" -eq 0 ]] && echo "No need to wait for KSM to settle" && return + + if (( newscan != oldscan )); then + echo -e "\nnew full_scan ($oldscan to $newscan)" + + # Do we have a previous scan to compare with + echo "check pages $oldpages to $newpages" + + if (( oldpages != -1 )); then + # avoid divide by zero problems + if (( $oldpages > 0 )); then + pcnt=$(( 100 - ((newpages * 100) / oldpages) )) + # abs() + pcnt=$(( $pcnt * -1 )) + + echo "$oldpages to $newpages is ${pcnt}%" + + if (( $pcnt <= 5 )); then + echo "KSM stabilised at ${t}s" + return + fi + else + echo "$oldpages KSM pages... waiting" + fi + fi + oldscan=$newscan + oldpages=$newpages + else + echo -n "." + fi + sleep 1 + done + echo "Timed out after ${1}s waiting for KSM to settle" +} + +common_init From 2a15ad97888c8e784537b45cf61434ee5a8b8c47 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 13 Jun 2023 18:14:38 +0200 Subject: [PATCH 66/76] virtiofsd: Stop using deprecated `-f` option The rust implementation of virtiofsd always runs foreground and spits a deprecation warning when `-f` is passed. Signed-off-by: Greg Kurz --- src/runtime/virtcontainers/virtiofsd.go | 2 -- src/runtime/virtcontainers/virtiofsd_test.go | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index 6df62adf9..bb5aaa319 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -194,8 +194,6 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { "-o", "source=" + v.sourcePath, // fd number of vhost-user socket fmt.Sprintf("--fd=%v", FdSocketNumber), - // foreground operation - "-f", } if len(v.extraArgs) != 0 { diff --git a/src/runtime/virtcontainers/virtiofsd_test.go b/src/runtime/virtcontainers/virtiofsd_test.go index a4252a2ba..2fd096b38 100644 --- a/src/runtime/virtcontainers/virtiofsd_test.go +++ b/src/runtime/virtcontainers/virtiofsd_test.go @@ -79,12 +79,12 @@ func TestVirtiofsdArgs(t *testing.T) { cache: "none", } - expected := "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=123 -f" + expected := "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=123" args, err := v.args(123) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) - expected = "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=456 -f" + expected = "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=456" args, err = v.args(456) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) From 8e00dc694416d1bd962d130de3f1fa42737233f6 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Wed, 14 Jun 2023 14:56:30 +0200 Subject: [PATCH 67/76] virtiofsd: Drop `-o no_posix_lock` The C implementation of virtiofsd had some kind of limited support for remote POSIX locks that was causing some workflows to fail with kata. Commit 432f9bea6e8b2 hard coded `-o no_posix_lock` in order to enforce guest local POSIX locks and avoid the issues. We've switched to the rust implementation of virtiofsd since then, but it emits a warning about `-o` being deprecated. According to https://gitlab.com/virtio-fs/virtiofsd/-/issues/53 : The C implementation of the daemon has limited support for remote POSIX locks, restricted exclusively to non-blocking operations. We tried to implement the same level of functionality in #2, but we finally decided against it because, in practice most applications will fail if non-blocking operations aren't supported. Implementing support for non-blocking isn't trivial and will probably require extending the kernel interface before we can even start working on the daemon side. There is thus no justification to pass `-o no_posix_lock` anymore. Signed-off-by: Greg Kurz --- src/runtime/virtcontainers/virtiofsd.go | 3 --- src/runtime/virtcontainers/virtiofsd_test.go | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index bb5aaa319..d45c08dca 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -187,9 +187,6 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { "--syslog", // cache mode for virtiofsd "-o", "cache=" + v.cache, - // disable posix locking in daemon: bunch of basic posix locks properties are broken - // apt-get update is broken if enabled - "-o", "no_posix_lock", // shared directory tree "-o", "source=" + v.sourcePath, // fd number of vhost-user socket diff --git a/src/runtime/virtcontainers/virtiofsd_test.go b/src/runtime/virtcontainers/virtiofsd_test.go index 2fd096b38..fbcdbec27 100644 --- a/src/runtime/virtcontainers/virtiofsd_test.go +++ b/src/runtime/virtcontainers/virtiofsd_test.go @@ -79,12 +79,12 @@ func TestVirtiofsdArgs(t *testing.T) { cache: "none", } - expected := "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=123" + expected := "--syslog -o cache=none -o source=/run/kata-shared/foo --fd=123" args, err := v.args(123) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) - expected = "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=456" + expected = "--syslog -o cache=none -o source=/run/kata-shared/foo --fd=456" args, err = v.args(456) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) From a43ea24dfc2611be3da5b358e079402db54fd7db Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Wed, 14 Jun 2023 12:33:45 +0200 Subject: [PATCH 68/76] virtiofsd: Convert legacy `-o` sub-options to their `--` replacement The `-o` option is the legacy way to configure virtiofsd, inherited from the C implementation. The rust implementation honours it for compatibility but it logs deprecation warnings. Let's use the replacement options in the go shim code. Also drop references to `-o` from the configuration TOML file. Fixes #7111 Signed-off-by: Greg Kurz --- src/runtime/Makefile | 2 +- src/runtime/config/configuration-clh.toml.in | 4 ++-- src/runtime/config/configuration-qemu.toml.in | 4 ++-- src/runtime/virtcontainers/virtiofsd.go | 4 ++-- src/runtime/virtcontainers/virtiofsd_test.go | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 80c423612..f34d21249 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -232,7 +232,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024 # # see `virtiofsd -h` for possible options. # Make sure you quote args. -DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"-o\", \"announce_submounts\"] +DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"--announce-submounts\"] DEFENABLEIOTHREADS := false DEFENABLEVHOSTUSERSTORE := false DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index d6653bce9..0ac6e1434 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -150,9 +150,9 @@ virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ # Extra args for virtiofsd daemon # # Format example: -# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# ["--arg1=xxx", "--arg2=yyy"] # Examples: -# Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"] +# Set virtiofsd log level to debug : ["--log-level=debug"] # see `virtiofsd -h` for possible options. virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index f58f87141..37ee3a752 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -197,9 +197,9 @@ virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ # Extra args for virtiofsd daemon # # Format example: -# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# ["--arg1=xxx", "--arg2=yyy"] # Examples: -# Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"] +# Set virtiofsd log level to debug : ["--log-level=debug"] # # see `virtiofsd -h` for possible options. virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index d45c08dca..3e02756eb 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -186,9 +186,9 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { // Send logs to syslog "--syslog", // cache mode for virtiofsd - "-o", "cache=" + v.cache, + "--cache=" + v.cache, // shared directory tree - "-o", "source=" + v.sourcePath, + "--shared-dir=" + v.sourcePath, // fd number of vhost-user socket fmt.Sprintf("--fd=%v", FdSocketNumber), } diff --git a/src/runtime/virtcontainers/virtiofsd_test.go b/src/runtime/virtcontainers/virtiofsd_test.go index fbcdbec27..c7d1e1e78 100644 --- a/src/runtime/virtcontainers/virtiofsd_test.go +++ b/src/runtime/virtcontainers/virtiofsd_test.go @@ -79,12 +79,12 @@ func TestVirtiofsdArgs(t *testing.T) { cache: "none", } - expected := "--syslog -o cache=none -o source=/run/kata-shared/foo --fd=123" + expected := "--syslog --cache=none --shared-dir=/run/kata-shared/foo --fd=123" args, err := v.args(123) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) - expected = "--syslog -o cache=none -o source=/run/kata-shared/foo --fd=456" + expected = "--syslog --cache=none --shared-dir=/run/kata-shared/foo --fd=456" args, err = v.args(456) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) From 1c7fcc6cbb748d34a34a07f18dcedc18b595dd6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 25 Oct 2022 15:07:11 +0200 Subject: [PATCH 69/76] packaging: Use existing image to build the initramfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's first try to pull a pre-existing image, instead of building our own, to be used as a builder for the initramds. This will save us some CI time. Fixes: #7084 Signed-off-by: Fabiano Fidêncio (cherry picked from commit ebf6c8383983e78a071baf0aed565f38453226ba) --- tools/packaging/static-build/initramfs/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index 489ddd5e1..787d6e85b 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -15,7 +15,6 @@ readonly default_install_dir="$(cd "${script_dir}/../../kernel" && pwd)" source "${script_dir}/../../scripts/lib.sh" -container_image="kata-initramfs-builder" kata_version="${kata_version:-}" cryptsetup_repo="${cryptsetup_repo:-}" cryptsetup_version="${cryptsetup_version:-}" @@ -33,7 +32,9 @@ package_output_dir="${package_output_dir:-}" [ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" [ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" -sudo docker build \ +container_image="${BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})" + +sudo docker pull ${container_image} || sudo docker build \ -t "${container_image}" "${script_dir}" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ From aa2380fdd627b061ef498b203ee9cea1b906497d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 25 Oct 2022 15:09:11 +0200 Subject: [PATCH 70/76] packaging: Add infra to push the initramfs builder image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add the needed infra for only building and pushing the initramfs builder image to the Kata Containers' quay.io registry. Fixes: #7084 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 111ad87828e479dd986ac1a158637d195c0283bd) --- tools/packaging/static-build/initramfs/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index 787d6e85b..fce2a5fce 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -34,8 +34,10 @@ package_output_dir="${package_output_dir:-}" container_image="${BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})" -sudo docker pull ${container_image} || sudo docker build \ - -t "${container_image}" "${script_dir}" +sudo docker pull ${container_image} || (sudo docker build \ + -t "${container_image}" "${script_dir}" && \ + # No-op unless PUSH_TO_REGISTRY is exported as "yes" + push_to_registry "${container_image}") sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ From 6a3710055b7abd81921240976bd6f085dfacce69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 25 Oct 2022 20:35:45 +0200 Subject: [PATCH 71/76] initramfs: Build dependencies as part of the Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will help to not have to build those on every CI run, and rather take advantage of the cached image. Fixes: #7084 Signed-off-by: Fabiano Fidêncio (cherry picked from commit c720869eefd7ff0555d7092db87f327a174f138d) --- .../static-build/initramfs/Dockerfile | 36 ++++++++++++++++- .../static-build/initramfs/build-initramfs.sh | 40 ------------------- .../packaging/static-build/initramfs/build.sh | 8 ++-- 3 files changed, 39 insertions(+), 45 deletions(-) diff --git a/tools/packaging/static-build/initramfs/Dockerfile b/tools/packaging/static-build/initramfs/Dockerfile index 3ebe0f347..9ba6968c2 100644 --- a/tools/packaging/static-build/initramfs/Dockerfile +++ b/tools/packaging/static-build/initramfs/Dockerfile @@ -4,6 +4,13 @@ from ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive +ARG cryptsetup_repo=${cryptsetup_repo} +ARG cryptsetup_version=${cryptsetup_version} +ARG lvm2_repo=${lvm2_repo} +ARG lvm2_version=${lvm2_version} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + ENV TZ=UTC RUN apt-get update &&\ apt-get --no-install-recommends install -y software-properties-common &&\ @@ -35,4 +42,31 @@ RUN apt-get update &&\ libseccomp-dev \ libseccomp2 \ zlib1g-dev &&\ - apt-get clean && rm -rf /var/lib/apt/lists/ + apt-get clean && rm -rf /var/lib/apt/lists/ && \ + build_root=$(mktemp -d) && \ + pushd ${build_root} && \ + echo "Build ${lvm2_repo} version: ${lvm2_version}" && \ + git clone --depth 1 --branch "${lvm2_version}" "${lvm2_repo}" lvm2 && \ + pushd lvm2 && \ + ./configure --enable-static_link --disable-selinux && \ + make && make install && \ + cp ./libdm/libdevmapper.pc /usr/lib/pkgconfig/devmapper.pc && \ + popd && \ + echo "Build ${cryptsetup_repo} version: ${cryptsetup_version}" && \ + git clone --depth 1 --branch "${cryptsetup_version}" "${cryptsetup_repo}" cryptsetup && \ + pushd cryptsetup && \ + ./autogen.sh && \ + ./configure --enable-static --enable-static-cryptsetup --disable-udev --disable-external-tokens --disable-ssh-token && \ + make && make install && \ + strip /usr/sbin/veritysetup.static && \ + popd && \ + echo "Build gen_init_cpio tool" && \ + git clone --depth 1 --filter=blob:none --sparse https://github.com/torvalds/linux.git && \ + pushd linux && \ + git sparse-checkout add usr && cd usr && make gen_init_cpio && \ + install gen_init_cpio /usr/sbin/ && \ + popd && \ + popd && \ + rm -rf ${build_root} + +COPY init.sh /usr/sbin/init.sh diff --git a/tools/packaging/static-build/initramfs/build-initramfs.sh b/tools/packaging/static-build/initramfs/build-initramfs.sh index a011e9822..4fa370d12 100755 --- a/tools/packaging/static-build/initramfs/build-initramfs.sh +++ b/tools/packaging/static-build/initramfs/build-initramfs.sh @@ -12,44 +12,4 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../../scripts/lib.sh" install_dir="${1:-.}" -cryptsetup_repo="${cryptsetup_repo:-}" -cryptsetup_version="${cryptsetup_version:-}" -lvm2_repo="${lvm2_repo:-}" -lvm2_version="${lvm2_version:-}" - -[ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo" -[ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version" -[ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" -[ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" - -build_root=$(mktemp -d) -pushd ${build_root} - -info "Build ${lvm2_repo} version: ${lvm2_version}" -git clone --depth 1 --branch "${lvm2_version}" "${lvm2_repo}" lvm2 -pushd lvm2 -./configure --enable-static_link --disable-selinux -make && make install -cp ./libdm/libdevmapper.pc /usr/lib/pkgconfig/devmapper.pc -popd #lvm2 - -info "Build ${cryptsetup_repo} version: ${cryptsetup_version}" -git clone --depth 1 --branch "${cryptsetup_version}" "${cryptsetup_repo}" cryptsetup -pushd cryptsetup -./autogen.sh -./configure --enable-static --enable-static-cryptsetup --disable-udev --disable-external-tokens --disable-ssh-token -make && make install -strip /usr/sbin/veritysetup.static -popd #cryptsetup - -info "Build gen_init_cpio tool" -git clone --depth 1 --filter=blob:none --sparse https://github.com/torvalds/linux.git -pushd linux -git sparse-checkout add usr && cd usr && make gen_init_cpio -install gen_init_cpio /usr/sbin/ -popd #linux - -popd #${build_root} - -install "${script_dir}/init.sh" /usr/sbin/ gen_init_cpio "${script_dir}/initramfs.list" | gzip -9 -n > "${install_dir}"/initramfs.cpio.gz diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index fce2a5fce..34b8c1bd1 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -35,15 +35,15 @@ package_output_dir="${package_output_dir:-}" container_image="${BUILDER_REGISTRY}:initramfs-cryptsetup-${cryptsetup_version}-lvm2-${lvm2_version}-$(get_last_modification ${repo_root_dir} ${script_dir})" sudo docker pull ${container_image} || (sudo docker build \ + --build-arg cryptsetup_repo="${cryptsetup_repo}" \ + --build-arg cryptsetup_version="${cryptsetup_version}" \ + --build-arg lvm2_repo="${lvm2_repo}" \ + --build-arg lvm2_version="${lvm2_version}" \ -t "${container_image}" "${script_dir}" && \ # No-op unless PUSH_TO_REGISTRY is exported as "yes" push_to_registry "${container_image}") sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ - --env cryptsetup_repo="${cryptsetup_repo}" \ - --env cryptsetup_version="${cryptsetup_version}" \ - --env lvm2_repo="${lvm2_repo}" \ - --env lvm2_version="${lvm2_version}" \ "${container_image}" \ bash -c "${initramfs_builder} ${default_install_dir}" From 6b084893010a9e8d576bfaadb1606dd454759497 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Fri, 16 Jun 2023 21:13:23 +0000 Subject: [PATCH 72/76] gha: Fix format for run launchtimes metrics yaml This PR fixes the format for the run launchtimes metrics yaml which is causing to the workflow to fail. Fixes #7130 Signed-off-by: Gabriela Cervantes --- .github/workflows/run-launchtimes-metrics.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-launchtimes-metrics.yaml b/.github/workflows/run-launchtimes-metrics.yaml index 788f3488f..bfbb9f2eb 100644 --- a/.github/workflows/run-launchtimes-metrics.yaml +++ b/.github/workflows/run-launchtimes-metrics.yaml @@ -12,8 +12,8 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - - name: run launch times on qemu - run: bash tests/metrics/gha-run.sh run-test-launchtimes-qemu + - name: run launch times on qemu + run: bash tests/metrics/gha-run.sh run-test-launchtimes-qemu - - name: run launch times on clh - run: bash tests/metrics/gha-run.sh run-test-launchtimes-clh + - name: run launch times on clh + run: bash tests/metrics/gha-run.sh run-test-launchtimes-clh From 7fc10b975fda4a606e9239d5e35f95c47d6a4c0d Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 19 Jun 2023 11:37:35 +0100 Subject: [PATCH 73/76] agent: re-vendor Re-vendor after bad merge Signed-off-by: stevenhorsman --- src/agent/Cargo.lock | 61 +++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 0ce9dfd6c..f902cd041 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -2647,6 +2647,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "objc" version = "0.2.7" @@ -2827,7 +2836,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project", - "rand", + "rand 0.8.5", "serde", "thiserror", "tokio", @@ -3525,6 +3534,19 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -3876,6 +3898,13 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] + [[package]] name = "salsa20" version = "0.10.2" @@ -3894,13 +3923,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "safe-path" -version = "0.1.0" -dependencies = [ - "libc", -] - [[package]] name = "scan_fmt" version = "0.2.6" @@ -4390,7 +4412,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.11", + "time 0.3.20", ] [[package]] @@ -4585,17 +4607,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "termcolor" version = "1.2.0" @@ -4666,6 +4677,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", @@ -5149,6 +5162,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -5567,7 +5586,7 @@ dependencies = [ "nix 0.23.2", "once_cell", "ordered-stream", - "rand", + "rand 0.8.5", "serde", "serde_repr", "sha1", From 6350f49baf174a1951ef9ff16b0a47c6e2b756d5 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 19 Jun 2023 11:58:43 +0100 Subject: [PATCH 74/76] agent-ctl: Re-vendor Re-vendor after bad merge Signed-off-by: stevenhorsman --- src/tools/agent-ctl/Cargo.lock | 252 ++++++++++----------------------- 1 file changed, 76 insertions(+), 176 deletions(-) diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index 456506c71..cf54d4473 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -226,7 +226,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" dependencies = [ - "errno 0.2.8", "libc", "thiserror", ] @@ -363,6 +362,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.7" @@ -374,6 +383,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "either" version = "1.8.1" @@ -412,17 +432,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "errno" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -577,7 +586,7 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -712,17 +721,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "itertools" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "itertools" version = "0.10.5" @@ -860,6 +858,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -884,7 +883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -931,16 +930,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num-integer" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.15" @@ -960,6 +949,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "oci" version = "0.1.0" @@ -1012,7 +1010,7 @@ dependencies = [ "libc", "redox_syscall 0.2.16", "smallvec", - "windows-sys 0.36.1", + "windows-sys 0.45.0", ] [[package]] @@ -1309,7 +1307,7 @@ dependencies = [ name = "regex-syntax" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "rlimit" @@ -1327,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ "bitflags", - "errno 0.3.1", + "errno", "io-lifetimes", "libc", "linux-raw-sys", @@ -1369,6 +1367,12 @@ dependencies = [ "zbus", ] +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + [[package]] name = "ryu" version = "1.0.13" @@ -1519,6 +1523,19 @@ dependencies = [ "slog", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.21", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -1598,6 +1615,17 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1655,6 +1683,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", @@ -1700,7 +1730,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -1843,6 +1873,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1942,11 +1978,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", + "windows-targets 0.48.0", ] [[package]] @@ -2009,66 +2041,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -2081,30 +2053,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -2129,42 +2077,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -2210,18 +2122,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" - [[package]] name = "xattr" version = "0.2.3" From a7fce537c7e948b607dd148bbbc743150b88149d Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 19 Jun 2023 17:16:37 +0100 Subject: [PATCH 75/76] kata-deploy: Add MEASURED_ROOTFS check - Check MEASURED_ROOTFS is set before adding the cc_rootfs_verity config Signed-off-by: stevenhorsman --- .../local-build/kata-deploy-binaries.sh | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 8cfda0ef1..73ce73d05 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -389,21 +389,25 @@ install_cc_shimv2() { export RUST_VERSION export REMOVE_VMM_CONFIGS="acrn fc" - extra_opts="DEFSERVICEOFFLOAD=true" - if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then - root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt") - root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" - extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" - fi + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + extra_opts="DEFSERVICEOFFLOAD=true" + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt") + root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" + fi - if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" ]; then - root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt") - root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" - extra_opts+=" ROOTMEASURECONFIGTDX=\"${root_measure_config}\"" + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt") + root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIGTDX=\"${root_measure_config}\"" + fi + + info "extra_opts: ${extra_opts}" + DESTDIR="${destdir}" PREFIX="${cc_prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" + else + DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${shimv2_builder}" fi - - info "extra_opts: ${extra_opts}" - DESTDIR="${destdir}" PREFIX="${cc_prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" } # Install static CC virtiofsd asset From 5a4a89c108f375d18af116772684fd6195d819b3 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Tue, 20 Jun 2023 09:17:39 +0100 Subject: [PATCH 76/76] runtime: Remove duplicated variables Remove duplicated variables that were in `CCv0` and merged in from main Signed-off-by: stevenhorsman --- src/runtime/Makefile | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 971dadfd6..17e36ed09 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -155,10 +155,6 @@ FIRMWARETDVFVOLUMEPATH := $(PREFIXDEPS)/share/tdvf/OVMF_VARS.fd FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd -ROOTMEASURECONFIG ?= "" -KERNELPARAMS += $(ROOTMEASURECONFIG) -KERNELTDXPARAMS += $(ROOTMEASURECONFIG) - # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml @@ -391,31 +387,6 @@ ifneq (,$(QEMUCMD)) CONFIGS += $(CONFIG_REMOTE) - - CONFIG_FILE_QEMU_GPU = configuration-qemu-gpu.toml - CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU) - CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in - - CONFIG_PATH_QEMU_SEV = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SEV)) - CONFIG_PATHS += $(CONFIG_PATH_QEMU_SEV) - - SYSCONFIG_QEMU_SEV = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SEV)) - SYSCONFIG_PATHS_SEV += $(SYSCONFIG_QEMU_SEV) - - CONFIGS += $(CONFIG_QEMU_SEV) - - CONFIG_FILE_QEMU_SNP = configuration-qemu-snp.toml - CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP) - CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in - - CONFIG_PATH_QEMU_SNP = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SNP)) - CONFIG_PATHS += $(CONFIG_PATH_QEMU_SNP) - - SYSCONFIG_QEMU_SNP = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SNP)) - SYSCONFIG_PATHS_SNP += $(SYSCONFIG_QEMU_SNP) - - CONFIGS += $(CONFIG_QEMU_SNP) - CONFIG_FILE_QEMU_NVIDIA_GPU = configuration-qemu-nvidia-gpu.toml CONFIG_QEMU_NVIDIA_GPU = config/$(CONFIG_FILE_QEMU_NVIDIA_GPU) CONFIG_QEMU_NVIDIA_GPU_IN = $(CONFIG_QEMU_NVIDIA_GPU).in