diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index e7ff49e1a..c2270ebd0 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -1,19 +1,11 @@ on: - pull_request: # this will trigger the workflow on PRs that changes release version - types: - - opened - - edited - - reopened - - synchronize - paths: - - VERSION - workflow_dispatch: # this allows to trigger the workflow manually on non-main branches + workflow_dispatch: # this is used to trigger the workflow on non-main branches inputs: pr: description: 'PR number from the selected branch to test' type: string required: true - issue_comment: # this allows to trigger the workflow from main by commenting "/test_kata_deploy" + issue_comment: types: [created, edited] name: test-kata-deploy @@ -27,14 +19,13 @@ jobs: && github.event.action == 'created' && startsWith(github.event.comment.body, '/test_kata_deploy') || github.event_name == 'workflow_dispatch' - || github.event_name == 'pull_request' steps: - name: Check membership on comment or dispatch uses: kata-containers/is-organization-member@1.0.1 id: is_organization_member with: organization: kata-containers - username: ${{ github.event.comment.user.login || github.event.sender.login }} # first one applies only on issue_comment + username: ${{ github.event.comment.user.login || github.event.sender.login }} token: ${{ secrets.GITHUB_TOKEN }} - name: Fail if not member run: | @@ -70,10 +61,8 @@ jobs: run: | if [ ${{ github.event_name }} == 'issue_comment' ]; then ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - elif [ ${{ github.event_name }} == 'workflow_dispatch' ]; then + else # workflow_dispatch ref="refs/pull/${{ github.event.inputs.pr }}/merge" - elif [ ${{ github.event_name }} == 'pull_request' ]; then - ref="refs/pull/${{ github.event.number }}/merge" fi echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" @@ -112,10 +101,8 @@ jobs: run: | if [ ${{ github.event_name }} == 'issue_comment' ]; then ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - elif [ ${{ github.event_name }} == 'workflow_dispatch' ]; then + else # workflow_dispatch ref="refs/pull/${{ github.event.inputs.pr }}/merge" - elif [ ${{ github.event_name }} == 'pull_request' ]; then - ref="refs/pull/${{ github.event.number }}/merge" fi echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" @@ -145,10 +132,8 @@ jobs: run: | if [ ${{ github.event_name }} == 'issue_comment' ]; then ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - elif [ ${{ github.event_name }} == 'workflow_dispatch' ]; then + else # workflow_dispatch ref="refs/pull/${{ github.event.inputs.pr }}/merge" - elif [ ${{ github.event_name }} == 'pull_request' ]; then - ref="refs/pull/${{ github.event.number }}/merge" fi echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" diff --git a/VERSION b/VERSION index e3311eab4..13997e279 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-rc0 +3.1.0-alpha0 diff --git a/docs/Release-Process.md b/docs/Release-Process.md index c9991d4b0..7dcfb84a3 100644 --- a/docs/Release-Process.md +++ b/docs/Release-Process.md @@ -48,7 +48,7 @@ ### Merge all bump version Pull requests - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request. - - `test-kata-deploy` workflow should be triggered automatically, validate it passes under the `Actions` tab on the repository GitHub page (you're also able to run it manually from there). + - Trigger the `test-kata-deploy` workflow which is under the `Actions` tab on the repository GitHub page (make sure to select the correct branch and validate it passes). - Check any failures and fix if needed. - Work with the Kata approvers to verify that the CI works and the pull requests are merged. diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index e1089995e..ab54b01fb 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -3130,7 +3130,7 @@ dependencies = [ "lazy_static", "libc", "libseccomp", - "nix 0.23.1", + "nix 0.24.2", "oci", "path-absolutize", "protobuf", diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index b8cdb9029..324f540e1 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -12,7 +12,7 @@ serde_derive = "1.0.91" oci = { path = "../../libs/oci" } protocols = { path ="../../libs/protocols" } caps = "0.5.0" -nix = "0.23.0" +nix = "0.24.2" scopeguard = "1.0.0" capctl = "0.2.0" lazy_static = "1.3.0" diff --git a/src/agent/rustjail/src/console.rs b/src/agent/rustjail/src/console.rs index 52e33f392..3ac351357 100644 --- a/src/agent/rustjail/src/console.rs +++ b/src/agent/rustjail/src/console.rs @@ -6,8 +6,9 @@ use anyhow::{anyhow, Result}; use nix::errno::Errno; use nix::pty; -use nix::sys::{socket, uio}; +use nix::sys::socket; use nix::unistd::{self, dup2}; +use std::io::IoSlice; use std::os::unix::io::{AsRawFd, RawFd}; use std::path::Path; @@ -23,10 +24,7 @@ pub fn setup_console_socket(csocket_path: &str) -> Result> { None, )?; - match socket::connect( - socket_fd, - &socket::SockAddr::Unix(socket::UnixAddr::new(Path::new(csocket_path))?), - ) { + match socket::connect(socket_fd, &socket::UnixAddr::new(Path::new(csocket_path))?) { Ok(()) => Ok(Some(socket_fd)), Err(errno) => Err(anyhow!("failed to open console fd: {}", errno)), } @@ -36,11 +34,11 @@ pub fn setup_master_console(socket_fd: RawFd) -> Result<()> { let pseudo = pty::openpty(None, None)?; let pty_name: &[u8] = b"/dev/ptmx"; - let iov = [uio::IoVec::from_slice(pty_name)]; + let iov = [IoSlice::new(pty_name)]; let fds = [pseudo.master]; let cmsg = socket::ControlMessage::ScmRights(&fds); - socket::sendmsg(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?; + socket::sendmsg::<()>(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?; unistd::setsid()?; let ret = unsafe { libc::ioctl(pseudo.slave, libc::TIOCSCTTY) }; diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 8a43179c1..1db16343e 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -779,16 +779,20 @@ pub async fn add_storages( } }; - // Todo need to rollback the mounted storage if err met. - - if res.is_err() { - error!( - logger, - "add_storages failed, storage: {:?}, error: {:?} ", storage, res - ); - } - - let mount_point = res?; + let mount_point = match res { + Err(e) => { + error!( + logger, + "add_storages failed, storage: {:?}, error: {:?} ", storage, e + ); + let mut sb = sandbox.lock().await; + sb.unset_sandbox_storage(&storage.mount_point) + .map_err(|e| warn!(logger, "fail to unset sandbox storage {:?}", e)) + .ok(); + return Err(e); + } + Ok(m) => m, + }; if !mount_point.is_empty() { mount_list.push(mount_point); diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index cab703b43..03bb1e0e1 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -462,8 +462,22 @@ impl AgentService { if p.init && sig == libc::SIGTERM && !is_signal_handled(&proc_status_file, sig as u32) { sig = libc::SIGKILL; } - p.signal(sig)?; - } + + match p.signal(sig) { + Err(Errno::ESRCH) => { + info!( + sl!(), + "signal encounter ESRCH, continue"; + "container-id" => cid.clone(), + "exec-id" => eid.clone(), + "pid" => p.pid, + "signal" => sig, + ); + } + Err(err) => return Err(anyhow!(err)), + Ok(()) => (), + } + }; if eid.is_empty() { // eid is empty, signal all the remaining processes in the container cgroup diff --git a/src/libs/kata-sys-util/src/fs.rs b/src/libs/kata-sys-util/src/fs.rs index 1d85fa61c..32bfafee9 100644 --- a/src/libs/kata-sys-util/src/fs.rs +++ b/src/libs/kata-sys-util/src/fs.rs @@ -8,7 +8,7 @@ use std::ffi::OsString; use std::fs::{self, File}; use std::io::{Error, Result}; use std::os::unix::io::AsRawFd; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use crate::{eother, sl}; @@ -29,11 +29,6 @@ const FUSE_SUPER_MAGIC: FsType = 0x65735546; // from linux.git/include/uapi/linux/magic.h const OVERLAYFS_SUPER_MAGIC: FsType = 0x794c7630; -/// Get bundle path (current working directory). -pub fn get_bundle_path() -> Result { - std::env::current_dir() -} - /// Get the basename of the canonicalized path pub fn get_base_name>(src: P) -> Result { let s = src.as_ref().canonicalize()?; diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index d749b73df..ae7acb135 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -301,35 +301,39 @@ impl CpuInfo { pub fn adjust_config(&mut self) -> Result<()> { let features: Vec<&str> = self.cpu_features.split(',').map(|v| v.trim()).collect(); self.cpu_features = features.join(","); + + let cpus = num_cpus::get() as u32; + + // adjust default_maxvcpus + if self.default_maxvcpus == 0 || self.default_maxvcpus > cpus { + self.default_maxvcpus = cpus; + } + + // adjust default_vcpus + if self.default_vcpus < 0 || self.default_vcpus as u32 > cpus { + self.default_vcpus = cpus as i32; + } else if self.default_vcpus == 0 { + self.default_vcpus = default::DEFAULT_GUEST_VCPUS as i32; + } + + if self.default_vcpus > self.default_maxvcpus as i32 { + self.default_vcpus = self.default_maxvcpus as i32; + } + Ok(()) } /// Validate the configuration information. pub fn validate(&self) -> Result<()> { + if self.default_vcpus > self.default_maxvcpus as i32 { + return Err(eother!( + "The default_vcpus({}) is greater than default_maxvcpus({})", + self.default_vcpus, + self.default_maxvcpus + )); + } Ok(()) } - - /// Get default number of guest vCPUs. - pub fn get_default_vcpus(&self) -> u32 { - let cpus = num_cpus::get() as u32; - if self.default_vcpus < 0 || self.default_vcpus as u32 > cpus { - cpus - } else if self.default_vcpus == 0 { - default::DEFAULT_GUEST_VCPUS - } else { - self.default_vcpus as u32 - } - } - - /// Get default maximal number of guest vCPUs. - pub fn get_default_max_vcpus(&self) -> u32 { - let cpus = num_cpus::get() as u32; - if self.default_maxvcpus == 0 || self.default_maxvcpus > cpus { - cpus - } else { - self.default_maxvcpus - } - } } /// Configuration information for debug @@ -1106,4 +1110,80 @@ mod tests { String::from("boo=far a b=c foo bar baz=faz") ); } + + #[test] + fn test_cpu_info_adjust_config() { + // get CPU cores of the test node + let node_cpus = num_cpus::get() as u32; + let default_vcpus = default::DEFAULT_GUEST_VCPUS as i32; + + struct TestData<'a> { + desc: &'a str, + input: &'a mut CpuInfo, + output: CpuInfo, + } + + let tests = &mut [ + TestData { + desc: "all with default values", + input: &mut CpuInfo { + cpu_features: "".to_string(), + default_vcpus: 0, + default_maxvcpus: 0, + }, + output: CpuInfo { + cpu_features: "".to_string(), + default_vcpus: default_vcpus as i32, + default_maxvcpus: node_cpus, + }, + }, + TestData { + desc: "all with big values", + input: &mut CpuInfo { + cpu_features: "a,b,c".to_string(), + default_vcpus: 9999999, + default_maxvcpus: 9999999, + }, + output: CpuInfo { + cpu_features: "a,b,c".to_string(), + default_vcpus: node_cpus as i32, + default_maxvcpus: node_cpus, + }, + }, + TestData { + desc: "default_vcpus lager than default_maxvcpus", + input: &mut CpuInfo { + cpu_features: "a, b ,c".to_string(), + default_vcpus: -1, + default_maxvcpus: 1, + }, + output: CpuInfo { + cpu_features: "a,b,c".to_string(), + default_vcpus: 1, + default_maxvcpus: 1, + }, + }, + ]; + + for (_, tc) in tests.iter_mut().enumerate() { + // we can ensure that unwrap will not panic + tc.input.adjust_config().unwrap(); + + assert_eq!( + tc.input.cpu_features, tc.output.cpu_features, + "test[{}] cpu_features", + tc.desc + ); + assert_eq!( + tc.input.default_vcpus, tc.output.default_vcpus, + "test[{}] default_vcpus", + tc.desc + ); + assert_eq!( + tc.input.default_maxvcpus, tc.output.default_maxvcpus, + "test[{}] default_maxvcpus", + tc.desc + ); + } + } } diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs index f1a5bc5fe..c4e398665 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -13,7 +13,6 @@ use kata_sys_util::mount; use super::utils; pub(crate) const MOUNT_GUEST_TAG: &str = "kataShared"; -pub(crate) const PASSTHROUGH_FS_DIR: &str = "passthrough"; pub(crate) const FS_TYPE_VIRTIO_FS: &str = "virtiofs"; pub(crate) const KATA_VIRTIO_FS_DEV_TYPE: &str = "virtio-fs"; diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs index e3967b8ce..2607444cc 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs @@ -16,9 +16,8 @@ use kata_types::config::hypervisor::SharedFsInfo; use super::{ share_virtio_fs::{ prepare_virtiofs, FS_TYPE_VIRTIO_FS, KATA_VIRTIO_FS_DEV_TYPE, MOUNT_GUEST_TAG, - PASSTHROUGH_FS_DIR, }, - utils, ShareFs, *, + utils, ShareFs, PASSTHROUGH_FS_DIR, *, }; lazy_static! { diff --git a/src/runtime/Makefile b/src/runtime/Makefile index fc4b3cdc2..6ae112c27 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -404,7 +404,8 @@ ifneq (,$(ACRNCMD)) DEFMAXVCPUS_ACRN := 1 DEFBLOCKSTORAGEDRIVER_ACRN := virtio-blk DEFNETWORKMODEL_ACRN := macvtap - KERNEL_NAME_ACRN = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) + KERNELTYPE_ACRN = compressed + KERNEL_NAME_ACRN = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_ACRN)) KERNELPATH_ACRN = $(KERNELDIR)/$(KERNEL_NAME_ACRN) endif diff --git a/src/runtime/cmd/kata-runtime/kata-check_amd64.go b/src/runtime/cmd/kata-runtime/kata-check_amd64.go index 77d6b6ce7..c1d0a14f0 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_amd64.go +++ b/src/runtime/cmd/kata-runtime/kata-check_amd64.go @@ -30,7 +30,6 @@ const ( cpuFlagLM = "lm" cpuFlagSVM = "svm" cpuFlagSSE4_1 = "sse4_1" - kernelModvhm = "vhm_dev" kernelModvhost = "vhost" kernelModvhostnet = "vhost_net" kernelModvhostvsock = "vhost_vsock" @@ -46,7 +45,7 @@ const ( cpuTypeUnknown = -1 ) -const acrnDevice = "/dev/acrn_vhm" +const acrnDevice = "/dev/acrn_hsm" // ioctl_ACRN_CREATE_VM is the IOCTL to create VM in ACRN. // Current Linux mainstream kernel doesn't have support for ACRN. @@ -54,18 +53,31 @@ const acrnDevice = "/dev/acrn_vhm" // Until the support is available, directly use the value instead // of macros. //https://github.com/kata-containers/runtime/issues/1784 -const ioctl_ACRN_CREATE_VM = 0x43000010 //nolint -const ioctl_ACRN_DESTROY_VM = 0x43000011 //nolint +const ioctl_ACRN_CREATE_VM = 0xC030A210 //nolint +const ioctl_ACRN_PAUSE_VM = 0xA213 //nolint +const ioctl_ACRN_DESTROY_VM = 0xA211 //nolint -type acrn_create_vm struct { //nolint - vmid uint16 //nolint - reserved0 uint16 //nolint - vcpu_num uint16 //nolint - reserved1 uint16 //nolint - uuid [16]uint8 - vm_flag uint64 //nolint - req_buf uint64 //nolint - reserved2 [16]uint8 //nolint +type acrn_vm_creation struct { //nolint + vmid uint16 //nolint + reserved0 uint16 //nolint + vcpu_num uint16 //nolint + reserved1 uint16 //nolint + name [16]uint8 + vm_flag uint64 //nolint + ioreq_buf uint64 //nolint + cpu_affinity uint64 //nolint +} + +var io_request_page [4096]byte + +type acrn_io_request struct { // nolint + io_type uint32 // nolint + completion_polling uint32 // nolint + reserved0 [14]uint32 // nolint + data [8]uint64 // nolint + reserved1 uint32 // nolint + kernel_handled uint32 // nolint + processed uint32 // nolint } // cpuType save the CPU type @@ -150,10 +162,6 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { archGenuineIntel: "Intel Architecture CPU", } archRequiredKernelModules = map[string]kernelModule{ - kernelModvhm: { - desc: "Intel ACRN", - required: false, - }, kernelModvhost: { desc: msgKernelVirtio, required: false, @@ -162,6 +170,10 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { desc: msgKernelVirtioNet, required: false, }, + kernelModvhostvsock: { + desc: msgKernelVirtioVhostVsock, + required: false, + }, } case "mock": archRequiredCPUFlags = map[string]string{ @@ -247,19 +259,10 @@ func acrnIsUsable() error { defer syscall.Close(f) kataLog.WithField("device", acrnDevice).Info("device available") - acrnInst := vc.Acrn{} - uuidStr, err := acrnInst.GetNextAvailableUUID() - if err != nil { - return err - } - - uuid, err := acrnInst.GetACRNUUIDBytes(uuidStr) - if err != nil { - return fmt.Errorf("Converting UUID str to bytes failed, Err:%s", err) - } - - var createVM acrn_create_vm - createVM.uuid = uuid + var createVM acrn_vm_creation + copy(createVM.name[:], "KataACRNVM") + ioreq_buf := (*acrn_io_request)(unsafe.Pointer(&io_request_page)) + createVM.ioreq_buf = uint64(uintptr(unsafe.Pointer(ioreq_buf))) ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(f), @@ -269,10 +272,23 @@ func acrnIsUsable() error { if errno == syscall.EBUSY { kataLog.WithField("reason", "another hypervisor running").Error("cannot create VM") } + kataLog.WithFields(logrus.Fields{ + "ret": ret, + "errno": errno, + "VM_name": createVM.name, + }).Info("Create VM Error") + return errno + } + + ret, _, errno = syscall.Syscall(syscall.SYS_IOCTL, + uintptr(f), + uintptr(ioctl_ACRN_PAUSE_VM), + 0) + if ret != 0 || errno != 0 { kataLog.WithFields(logrus.Fields{ "ret": ret, "errno": errno, - }).Info("Create VM Error") + }).Info("PAUSE VM Error") return errno } diff --git a/src/runtime/pkg/device/drivers/block.go b/src/runtime/pkg/device/drivers/block.go index d2e9644fd..7107053f2 100644 --- a/src/runtime/pkg/device/drivers/block.go +++ b/src/runtime/pkg/device/drivers/block.go @@ -60,6 +60,12 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec return err } + hypervisorType := devReceiver.GetHypervisorType() + if hypervisorType == "acrn" { + deviceLogger().Debug("Special casing for ACRN to increment BlockIndex") + index = index + 1 + } + drive := &config.BlockDrive{ File: device.DeviceInfo.HostPath, Format: "raw", diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index b11cafb45..33f86b837 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -14,18 +14,18 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" - "unsafe" "github.com/pkg/errors" + "github.com/prometheus/procfs" "github.com/sirupsen/logrus" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" - "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" @@ -39,56 +39,9 @@ var acrnTracingTags = map[string]string{ "type": "acrn", } -// Since ACRN is using the store in a quite abnormal way, let's first draw it back from store to here - -/* -// UUIDPathSuffix is the suffix used for uuid storage -const ( - UUIDPathSuffix = "uuid" - uuidFile = "uuid.json" -) -*/ - -// ACRN currently supports only known UUIDs for security -// reasons (FuSa). When launching VM, only these pre-defined -// UUID should be used else VM launch will fail. The main -// of purpose UUID is is not used for image identification -// but generating vSeed (virtual seed which takes UUID -// as one of the parameter) which is used during VM boot. - -// acrnUUIDsToIdx lists Idx corresponding to the UUID -var acrnUUIDsToIdx = map[string]uint8{ - "a7ada506-1ab0-4b6b-a0da-e513ca9b8c2f": 0, - "dbeae168-26e4-4084-9227-622193e56325": 1, - "18ed60cd-e9ea-4bf4-8f87-8523fc8347a3": 2, - "3f90b6f8-449a-4e72-b99c-063a889fc422": 3, - "1ae8587b-e599-4b59-8260-6d14ac166a55": 4, - "75f3b94b-49ed-48fc-b019-577ef45adf2b": 5, - "ca62cf3c-8359-47e8-a3f7-de2d682dfb02": 6, - "e3189497-c3f6-4b97-9e2c-18ac0ab9064d": 7, -} - -// acrnIdxToUUIDs lists UUIDs corresponding to the Idx -var acrnIdxToUUIDs = map[uint8]string{ - 0: "a7ada506-1ab0-4b6b-a0da-e513ca9b8c2f", - 1: "dbeae168-26e4-4084-9227-622193e56325", - 2: "18ed60cd-e9ea-4bf4-8f87-8523fc8347a3", - 3: "3f90b6f8-449a-4e72-b99c-063a889fc422", - 4: "1ae8587b-e599-4b59-8260-6d14ac166a55", - 5: "75f3b94b-49ed-48fc-b019-577ef45adf2b", - 6: "ca62cf3c-8359-47e8-a3f7-de2d682dfb02", - 7: "e3189497-c3f6-4b97-9e2c-18ac0ab9064d", -} - -// AcrnInfo keeps track of UUID availability -type AcrnInfo struct { - UUIDAvailability [8]uint8 -} - // AcrnState keeps track of VM UUID, PID. type AcrnState struct { - UUID string - PID int + PID int } // Acrn is an Hypervisor interface implementation for the Linux acrn hypervisor. @@ -98,41 +51,16 @@ type Acrn struct { arch acrnArch store persistapi.PersistDriver id string - state AcrnState acrnConfig Config config HypervisorConfig - info AcrnInfo + state AcrnState } -type acrnPlatformInfo struct { - cpuNum uint16 //nolint - reserved0 [126]uint8 //nolint - maxVCPUsPerVM uint16 //nolint - maxKataContainers uint8 - reserved1 [125]uint8 //nolint -} - -const acrnDevice = "/dev/acrn_vhm" - -// ioctl_ACRN_CREATE_VM is the IOCTL to create VM in ACRN. -// Current Linux mainstream kernel doesn't have support for ACRN. -// Due to this several macros are not defined in Linux headers. -// Until the support is available, directly use the value instead -// of macros. -//https://github.com/kata-containers/runtime/issues/1784 -const ioctl_ACRN_GET_PLATFORM_INFO = 0x43000003 //nolint - const ( acrnConsoleSocket = "console.sock" acrnStopSandboxTimeoutSecs = 15 ) -//UUIDBusy marks a particular UUID as busy -const UUIDBusy = 1 - -//UUIDFree marks a particular UUID as free -const UUIDFree = 0 - // agnostic list of kernel parameters var acrnDefaultKernelParameters = []Param{ {"panic", "1"}, @@ -170,6 +98,49 @@ func (a *Acrn) HypervisorConfig() HypervisorConfig { return a.config } +// Get cpu apicid to identify vCPU that will be assigned for a VM by reading `proc/cpuinfo` +func (a *Acrn) getNextApicid() (string, error) { + fs, err := procfs.NewFS("/proc") + if err != nil { + return "", err + } + + cpuinfo, err := fs.CPUInfo() + if err != nil { + return "", err + } + + prevIdx := -1 + fileName := filepath.Join(a.config.VMStorePath, "cpu_affinity_idx") + _, err = os.Stat(fileName) + if err == nil { + data, err := os.ReadFile(fileName) + if err != nil { + a.Logger().Error("Loading cpu affinity index from file failed!") + return "", err + } + + prevIdx, err = strconv.Atoi(string(data)) + if err != nil { + a.Logger().Error("CreateVM: Convert from []byte to integer failed!") + return "", err + } + + if prevIdx >= (len(cpuinfo) - 1) { + prevIdx = -1 + } + } + + currentIdx := prevIdx + 1 + err = os.WriteFile(fileName, []byte(strconv.Itoa(currentIdx)), defaultFilePerms) + if err != nil { + a.Logger().Error("Storing cpu affinity index from file failed!") + return "", err + } + + return cpuinfo[currentIdx].APICID, nil +} + // get the acrn binary path func (a *Acrn) acrnPath() (string, error) { p, err := a.config.HypervisorAssetPath() @@ -228,15 +199,7 @@ func (a *Acrn) appendImage(devices []Device, imagePath string) ([]Device, error) return nil, fmt.Errorf("Image path is empty: %s", imagePath) } - // Get sandbox and increment the globalIndex. - // This is to make sure the VM rootfs occupies - // the first Index which is /dev/vda. var err error - - if _, err = a.sandbox.GetAndSetSandboxBlockIndex(); err != nil { - return nil, err - } - devices, err = a.arch.appendImage(devices, imagePath) if err != nil { return nil, err @@ -295,36 +258,6 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso a.id = id a.arch = newAcrnArch(a.config) - var create bool - var uuid string - - if a.state.UUID == "" { - create = true - } - - if create { - a.Logger().Debug("Setting UUID") - - var err error - - if uuid, err = a.GetNextAvailableUUID(); err != nil { - return err - } - a.state.UUID = uuid - Idx := acrnUUIDsToIdx[uuid] - a.info.UUIDAvailability[Idx] = UUIDBusy - - // The path might already exist, but in case of VM templating, - // we have to create it since the sandbox has not created it yet. - if err = os.MkdirAll(filepath.Join(a.config.RunStorePath, id), DirMode); err != nil { - return err - } - - if err = a.storeInfo(); err != nil { - return err - } - } - return nil } @@ -386,10 +319,6 @@ func (a *Acrn) CreateVM(ctx context.Context, id string, network Network, hypervi Params: a.kernelParameters(), } - if a.state.UUID == "" { - return fmt.Errorf("ACRN UUID should not be empty") - } - devices, err := a.buildDevices(ctx, imagePath) if err != nil { return err @@ -405,15 +334,25 @@ func (a *Acrn) CreateVM(ctx context.Context, id string, network Network, hypervi return err } + vmName := fmt.Sprintf("sbx-%s", a.id) + if len(vmName) > 15 { + return fmt.Errorf("VM Name len is %d but ACRN supports max VM name len of 15.", len(vmName)) + } + + apicID, err := a.getNextApicid() + if err != nil { + return err + } + acrnConfig := Config{ - UUID: a.state.UUID, ACPIVirt: true, Path: acrnPath, CtlPath: acrnctlPath, Memory: memory, Devices: devices, Kernel: kernel, - Name: fmt.Sprintf("sandbox-%s", a.id), + Name: vmName, + ApicID: apicID, } a.acrnConfig = acrnConfig @@ -453,6 +392,7 @@ func (a *Acrn) StartVM(ctx context.Context, timeoutSecs int) error { var strErr string var PID int + a.Logger().Error("StartVM: LaunchAcrn() function called") PID, strErr, err = LaunchAcrn(a.acrnConfig, virtLog.WithField("subsystem", "acrn-dm")) if err != nil { return fmt.Errorf("%s", strErr) @@ -496,19 +436,23 @@ func (a *Acrn) StopVM(ctx context.Context, waitOnly bool) (err error) { } }() - // Mark the UUID as free - uuid := a.state.UUID - Idx := acrnUUIDsToIdx[uuid] - - if err = a.loadInfo(); err != nil { - a.Logger().Info("Failed to Load UUID availabiity info") + fileName := filepath.Join(a.config.VMStorePath, "cpu_affinity_idx") + data, err := os.ReadFile(fileName) + if err != nil { + a.Logger().Error("Loading cpu affinity index from file failed!") return err } - a.info.UUIDAvailability[Idx] = UUIDFree + currentIdx, err := strconv.Atoi(string(data)) + if err != nil { + a.Logger().Error("Converting from []byte to integer failed!") + return err + } - if err = a.storeInfo(); err != nil { - a.Logger().Info("Failed to store UUID availabiity info") + currentIdx = currentIdx - 1 + err = os.WriteFile(fileName, []byte(strconv.Itoa(currentIdx)), defaultFilePerms) + if err != nil { + a.Logger().Error("Storing cpu affinity index from file failed!") return err } @@ -617,8 +561,7 @@ func (a *Acrn) AddDevice(ctx context.Context, devInfo interface{}, devType Devic case types.Socket: a.acrnConfig.Devices = a.arch.appendSocket(a.acrnConfig.Devices, v) case types.VSock: - // Not supported. return success - err = nil + a.acrnConfig.Devices = a.arch.appendVSock(a.acrnConfig.Devices, v) case Endpoint: a.acrnConfig.Devices = a.arch.appendNetwork(a.acrnConfig.Devices, v) case config.BlockDrive: @@ -644,7 +587,9 @@ func (a *Acrn) GetVMConsole(ctx context.Context, id string) (string, string, err defer span.End() consoleURL, err := utils.BuildSocketPath(a.config.VMStorePath, id, acrnConsoleSocket) + if err != nil { + a.Logger().Error("GetVMConsole returned error") return consoleProtoUnix, "", err } @@ -714,13 +659,11 @@ func (a *Acrn) toGrpc(ctx context.Context) ([]byte, error) { func (a *Acrn) Save() (s hv.HypervisorState) { s.Pid = a.state.PID s.Type = string(AcrnHypervisor) - s.UUID = a.state.UUID return } func (a *Acrn) Load(s hv.HypervisorState) { a.state.PID = s.Pid - a.state.UUID = s.UUID } func (a *Acrn) Check() error { @@ -732,97 +675,14 @@ func (a *Acrn) Check() error { } func (a *Acrn) GenerateSocket(id string) (interface{}, error) { - return generateVMSocket(id, a.config.VMStorePath) -} - -// GetACRNUUIDBytes returns UUID bytes that is used for VM creation -func (a *Acrn) GetACRNUUIDBytes(uid string) (uuid.UUID, error) { - return uuid.Parse(uid) -} - -// GetNextAvailableUUID returns next available UUID VM creation -// If no valid UUIDs are available it returns err. -func (a *Acrn) GetNextAvailableUUID() (string, error) { - var MaxVMSupported uint8 - var Idx uint8 - var uuidStr string - var err error - - if err = a.loadInfo(); err != nil { - a.Logger().Infof("Load UUID store failed") - } - - if MaxVMSupported, err = a.GetMaxSupportedACRNVM(); err != nil { - return "", fmt.Errorf("IOCTL GetMaxSupportedACRNVM failed") - } - - for Idx = 0; Idx < MaxVMSupported; Idx++ { - if a.info.UUIDAvailability[Idx] == UUIDFree { - uuidStr = acrnIdxToUUIDs[Idx] - break - } - } - - if uuidStr == "" { - return "", fmt.Errorf("Invalid UUID: Max VMs reached") - } - - return uuidStr, nil -} - -// GetMaxSupportedACRNVM checks the max number of VMs that can be -// launched from kata-runtime. -func (a *Acrn) GetMaxSupportedACRNVM() (uint8, error) { - flags := syscall.O_RDWR | syscall.O_CLOEXEC - - f, err := syscall.Open(acrnDevice, flags, 0) + socket, err := generateVMSocket(id, a.config.VMStorePath) if err != nil { - return 0, err + return "", err } - defer syscall.Close(f) + vsock, _ := socket.(types.VSock) + vsock.VhostFd.Close() - var platformInfo acrnPlatformInfo - - ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL, - uintptr(f), - uintptr(ioctl_ACRN_GET_PLATFORM_INFO), - uintptr(unsafe.Pointer(&platformInfo))) - if ret != 0 || errno != 0 { - return 0, errno - } - - return platformInfo.maxKataContainers, nil -} - -func (a *Acrn) storeInfo() error { - /* - relPath := filepath.Join(UUIDPathSuffix, uuidFile) - - jsonOut, err := json.Marshal(a.info) - if err != nil { - return fmt.Errorf("Could not marshal data: %s", err) - } - - if err := a.store.GlobalWrite(relPath, jsonOut); err != nil { - return fmt.Errorf("failed to write uuid to file: %v", err) - }*/ - - return nil -} - -func (a *Acrn) loadInfo() error { - /* - relPath := filepath.Join(UUIDPathSuffix, uuidFile) - data, err := a.store.GlobalRead(relPath) - if err != nil { - return fmt.Errorf("failed to read uuid from file: %v", err) - } - - if err := json.Unmarshal(data, &a.info); err != nil { - return fmt.Errorf("failed to unmarshal uuid info: %v", err) - }*/ - - return nil + return socket, err } func (a *Acrn) IsRateLimiterBuiltin() bool { diff --git a/src/runtime/virtcontainers/acrn_arch_base.go b/src/runtime/virtcontainers/acrn_arch_base.go index f84019ee6..7379274b0 100644 --- a/src/runtime/virtcontainers/acrn_arch_base.go +++ b/src/runtime/virtcontainers/acrn_arch_base.go @@ -55,6 +55,9 @@ type acrnArch interface { // appendSocket appends a socket to devices appendSocket(devices []Device, socket types.Socket) []Device + // appendVSock appends a vsock PCI to devices + appendVSock(devices []Device, vsock types.VSock) []Device + // appendNetwork appends a endpoint device to devices appendNetwork(devices []Device, endpoint Endpoint) []Device @@ -141,13 +144,6 @@ var acrnKernelParams = []Param{ {"console", "hvc0"}, {"log_buf_len", "16M"}, {"consoleblank", "0"}, - {"iommu", "off"}, - {"i915.avail_planes_per_pipe", "0x070F00"}, - {"i915.enable_hangcheck", "0"}, - {"i915.nuclear_pageflip", "1"}, - {"i915.enable_guc_loading", "0"}, - {"i915.enable_guc_submission", "0"}, - {"i915.enable_guc", "0"}, } // Device is the acrn device interface. @@ -203,6 +199,12 @@ type ConsoleDevice struct { PortType BEPortType } +// VSOCKDevice represents a AF_VSOCK socket. +type VSOCKDevice struct { + //Guest CID assigned by Host. + ContextID uint64 +} + // NetDeviceType is a acrn networking device type. type NetDeviceType string @@ -293,8 +295,8 @@ type Config struct { // Name is the acrn guest name Name string - // UUID is the acrn process UUID. - UUID string + // APICID to identify vCPU that will be assigned for this VM. + ApicID string // Kernel is the guest kernel configuration. Kernel Kernel @@ -431,6 +433,33 @@ func (netdev NetDevice) AcrnNetdevParam() []string { return deviceParams } +const ( + // MinimalGuestCID is the smallest valid context ID for a guest. + MinimalGuestCID uint64 = 3 + + // MaxGuestCID is the largest valid context ID for a guest. + MaxGuestCID uint64 = 1<<32 - 1 +) + +// Valid returns true if the VSOCKDevice structure is valid and complete. +func (vsock VSOCKDevice) Valid() bool { + if vsock.ContextID < MinimalGuestCID || vsock.ContextID > MaxGuestCID { + return false + } + + return true +} + +// AcrnParams returns the acrn parameters built out of this vsock device. +func (vsock VSOCKDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,vhost-vsock,cid=%d", slot, uint32(vsock.ContextID))) + + return acrnParams +} + // Valid returns true if the NetDevice structure is valid and complete. func (netdev NetDevice) Valid() bool { if netdev.IFName == "" { @@ -546,13 +575,6 @@ func (config *Config) appendDevices() { } } -func (config *Config) appendUUID() { - if config.UUID != "" { - config.acrnParams = append(config.acrnParams, "-U") - config.acrnParams = append(config.acrnParams, config.UUID) - } -} - func (config *Config) appendACPI() { if config.ACPIVirt { config.acrnParams = append(config.acrnParams, "-A") @@ -566,10 +588,20 @@ func (config *Config) appendMemory() { } } +func (config *Config) appendCPUAffinity() { + if config.ApicID == "" { + return + } + + config.acrnParams = append(config.acrnParams, "--cpu_affinity") + config.acrnParams = append(config.acrnParams, config.ApicID) +} + func (config *Config) appendKernel() { if config.Kernel.Path == "" { return } + config.acrnParams = append(config.acrnParams, "-k") config.acrnParams = append(config.acrnParams, config.Kernel.Path) @@ -587,10 +619,10 @@ func (config *Config) appendKernel() { // This function writes its log output via logger parameter. func LaunchAcrn(config Config, logger *logrus.Entry) (int, string, error) { baselogger = logger - config.appendUUID() config.appendACPI() config.appendMemory() config.appendDevices() + config.appendCPUAffinity() config.appendKernel() config.appendName() @@ -696,6 +728,15 @@ func (a *acrnArchBase) appendSocket(devices []Device, socket types.Socket) []Dev return devices } +func (a *acrnArchBase) appendVSock(devices []Device, vsock types.VSock) []Device { + vmsock := VSOCKDevice{ + ContextID: vsock.ContextID, + } + + devices = append(devices, vmsock) + return devices +} + func networkModelToAcrnType(model NetInterworkingModel) NetDeviceType { switch model { case NetXConnectMacVtapModel: diff --git a/src/runtime/virtcontainers/acrn_test.go b/src/runtime/virtcontainers/acrn_test.go index f0e8c7ce2..25c79f960 100644 --- a/src/runtime/virtcontainers/acrn_test.go +++ b/src/runtime/virtcontainers/acrn_test.go @@ -243,9 +243,7 @@ func TestAcrnCreateVM(t *testing.T) { a.sandbox = sandbox - //set PID to 1 to ignore hypercall to get UUID and set a random UUID a.state.PID = 1 - a.state.UUID = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" network, err := NewNetwork() assert.NoError(err) err = a.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig) diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index e7296bf4e..47a22478b 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -436,7 +436,11 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) rootfsStorage.Source = blockDrive.DevNo case f.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: rootfsStorage.Driver = kataBlkDevType - rootfsStorage.Source = blockDrive.PCIPath.String() + if f.sandbox.config.HypervisorType == AcrnHypervisor { + rootfsStorage.Source = blockDrive.VirtPath + } else { + rootfsStorage.Source = blockDrive.PCIPath.String() + } case f.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: rootfsStorage.Driver = kataSCSIDevType rootfsStorage.Source = blockDrive.SCSIAddr diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 1ada73da6..1f8ad7b73 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1981,6 +1981,7 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { } if err := q.qmpMonitorCh.qmp.ExecuteCPUDeviceAdd(q.qmpMonitorCh.ctx, driver, cpuID, socketID, dieID, coreID, threadID, romFile); err != nil { + q.Logger().WithField("hotplug", "cpu").Warnf("qmp hotplug cpu, cpuID=%s socketID=%s, error: %v", cpuID, socketID, err) // don't fail, let's try with other CPU continue } diff --git a/src/runtime/virtcontainers/types/capabilities.go b/src/runtime/virtcontainers/types/capabilities.go index 28772f1b7..c035444b6 100644 --- a/src/runtime/virtcontainers/types/capabilities.go +++ b/src/runtime/virtcontainers/types/capabilities.go @@ -53,7 +53,7 @@ func (caps *Capabilities) IsFsSharingSupported() bool { return caps.flags&fsSharingSupported != 0 } -// SetFsSharingUnsupported sets the host filesystem sharing capability to true. +// SetFsSharingSupport sets the host filesystem sharing capability to true. func (caps *Capabilities) SetFsSharingSupport() { caps.flags |= fsSharingSupported } diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index e91ed1bf8..58da4d2d5 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -31,6 +31,12 @@ if [ ${docker_gid} == ${gid} ]; then docker_gid="" fi +remove_dot_docker_dir=false +if [ ! -d "$HOME/.docker" ]; then + mkdir $HOME/.docker + remove_dot_docker_dir=true +fi + docker build -q -t build-kata-deploy \ --build-arg IMG_USER="${USER}" \ --build-arg UID=${uid} \ @@ -39,6 +45,7 @@ docker build -q -t build-kata-deploy \ "${script_dir}/dockerbuild/" docker run \ + -v $HOME/.docker:/root/.docker \ -v /var/run/docker.sock:/var/run/docker.sock \ --user ${uid}:${gid} \ --env CI="${CI:-}" \ @@ -52,3 +59,7 @@ docker run \ --rm \ -w ${script_dir} \ build-kata-deploy "${kata_deploy_create}" $@ + +if [ $remove_dot_docker_dir == true ]; then + rm -rf "$HOME/.docker" +fi