runtime-rs: Add basic CH implementation

Add a basic runtime-rs `Hypervisor` trait implementation for Cloud
Hypervisor (CH).

> **Notes:**
>
> - This only supports a default Kata configuration for CH currently.
>
> - Since this feature is still under development, `cargo` features have
>   been added to enable the feature optionally. The default is to not enable
>   currently since the code is not ready for general use.
>
>   To enable the feature for testing and development, enable the
>   `cloud-hypervisor` feature in the `virt_container` crate and enable the
>   `cloud-hypervisor` feature for its `hypervisor` dependency.

Fixes: #5242.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -32,4 +32,14 @@ shim-interface = { path = "../../../libs/shim-interface" }
 
 dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs","dbs-upcall"] }
 
+ch-config = { path = "ch-config", optional = true }
+
+futures = "0.3.25"
+safe-path = "0.1.0"
+
 [features]
+default = []
+
+# Feature is not yet complete, so not enabled by default.
+# See https://github.com/kata-containers/kata-containers/issues/6264.
+cloud-hypervisor = ["ch-config"]

src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml

@@ -0,0 +1,22 @@
+# Copyright (c) 2022-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "ch-config"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.68"
+serde = { version = "1.0.145", features = ["rc", "derive"] }
+serde_json = "1.0.91"
+tokio = { version = "1.25.0", features = ["sync", "rt"] }
+
+# Cloud Hypervisor public HTTP API functions
+# Note that the version specified is not necessarily the version of CH
+# being used. This version is used to pin the CH config structure
+# which is relatively static.
+api_client = { git = "https://github.com/cloud-hypervisor/cloud-hypervisor", crate = "api_client", tag = "v27.0" }

src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs

@@ -0,0 +1,274 @@
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::net_util::MAC_ADDR_LEN;
+use crate::{
+    ConsoleConfig, ConsoleOutputMode, CpuTopology, CpusConfig, DeviceConfig, FsConfig, MacAddr,
+    MemoryConfig, NetConfig, PayloadConfig, PmemConfig, RngConfig, VmConfig, VsockConfig,
+};
+use anyhow::{anyhow, Context, Result};
+use api_client::simple_api_full_command_and_response;
+
+use std::fmt::Display;
+use std::net::Ipv4Addr;
+use std::os::unix::net::UnixStream;
+use std::path::PathBuf;
+use tokio::task;
+
+pub async fn cloud_hypervisor_vmm_ping(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(&mut socket, "GET", "vmm.ping", None)
+            .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vmm_shutdown(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response =
+            simple_api_full_command_and_response(&mut socket, "PUT", "vmm.shutdown", None)
+                .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vm_create(
+    sandbox_path: String,
+    vsock_socket_path: String,
+    mut socket: UnixStream,
+    shared_fs_devices: Option<Vec<FsConfig>>,
+    pmem_devices: Option<Vec<PmemConfig>>,
+) -> Result<Option<String>> {
+    let cfg = cloud_hypervisor_vm_create_cfg(
+        sandbox_path,
+        vsock_socket_path,
+        shared_fs_devices,
+        pmem_devices,
+    )
+    .await?;
+
+    let serialised = serde_json::to_string_pretty(&cfg)?;
+
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let data = Some(serialised.as_str());
+
+        let response = simple_api_full_command_and_response(&mut socket, "PUT", "vm.create", data)
+            .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vm_start(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(&mut socket, "PUT", "vm.boot", None)
+            .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+#[allow(dead_code)]
+pub async fn cloud_hypervisor_vm_stop(mut socket: UnixStream) -> Result<Option<String>> {
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response =
+            simple_api_full_command_and_response(&mut socket, "PUT", "vm.shutdown", None)
+                .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+#[allow(dead_code)]
+pub async fn cloud_hypervisor_vm_device_add(mut socket: UnixStream) -> Result<Option<String>> {
+    let device_config = DeviceConfig::default();
+
+    task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(
+            &mut socket,
+            "PUT",
+            "vm.add-device",
+            Some(&serde_json::to_string(&device_config)?),
+        )
+        .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?
+}
+
+pub async fn cloud_hypervisor_vm_fs_add(
+    mut socket: UnixStream,
+    fs_config: FsConfig,
+) -> Result<Option<String>> {
+    let result = task::spawn_blocking(move || -> Result<Option<String>> {
+        let response = simple_api_full_command_and_response(
+            &mut socket,
+            "PUT",
+            "vm.add-fs",
+            Some(&serde_json::to_string(&fs_config)?),
+        )
+        .map_err(|e| anyhow!(e))?;
+
+        Ok(response)
+    })
+    .await?;
+
+    result
+}
+
+pub async fn cloud_hypervisor_vm_create_cfg(
+    // FIXME:
+    _sandbox_path: String,
+    vsock_socket_path: String,
+    shared_fs_devices: Option<Vec<FsConfig>>,
+    pmem_devices: Option<Vec<PmemConfig>>,
+) -> Result<VmConfig> {
+    let topology = CpuTopology {
+        threads_per_core: 1,
+        cores_per_die: 12,
+        dies_per_package: 1,
+        packages: 1,
+    };
+
+    let cpus = CpusConfig {
+        boot_vcpus: 1,
+        max_vcpus: 12,
+        max_phys_bits: 46,
+        topology: Some(topology),
+        ..Default::default()
+    };
+
+    let rng = RngConfig {
+        src: PathBuf::from("/dev/urandom"),
+        ..Default::default()
+    };
+
+    let kernel_args = vec![
+        "root=/dev/pmem0p1",
+        "rootflags=dax,data=ordered,errors=remount-ro",
+        "ro",
+        "rootfstype=ext4",
+        "panic=1",
+        "no_timer_check",
+        "noreplace-smp",
+        "console=ttyS0,115200n8",
+        "systemd.log_target=console",
+        "systemd.unit=kata-containers",
+        "systemd.mask=systemd-networkd.service",
+        "systemd.mask=systemd-networkd.socket",
+        "agent.log=debug",
+    ];
+
+    let cmdline = kernel_args.join(" ");
+
+    let kernel = PathBuf::from("/opt/kata/share/kata-containers/vmlinux.container");
+
+    // Note that PmemConfig replaces the PayloadConfig.initrd.
+    let payload = PayloadConfig {
+        kernel: Some(kernel),
+        cmdline: Some(cmdline),
+        ..Default::default()
+    };
+
+    let serial = ConsoleConfig {
+        mode: ConsoleOutputMode::Tty,
+        ..Default::default()
+    };
+
+    let ip = Ipv4Addr::new(192, 168, 10, 10);
+    let mask = Ipv4Addr::new(255, 255, 255, 0);
+
+    let mac_str = "12:34:56:78:90:01";
+
+    let mac = parse_mac(mac_str)?;
+
+    let network = NetConfig {
+        ip,
+        mask,
+        mac,
+        ..Default::default()
+    };
+
+    let memory = MemoryConfig {
+        size: (1024 * 1024 * 2048),
+
+        // Required
+        shared: true,
+
+        prefault: false,
+        hugepages: false,
+        mergeable: false,
+
+        // FIXME:
+        hotplug_size: Some(16475226112),
+
+        ..Default::default()
+    };
+
+    let fs = shared_fs_devices;
+    let pmem = pmem_devices;
+
+    let vsock = VsockConfig {
+        cid: 3,
+        socket: PathBuf::from(vsock_socket_path),
+        ..Default::default()
+    };
+
+    let cfg = VmConfig {
+        cpus,
+        memory,
+        fs,
+        serial,
+        pmem,
+        payload: Some(payload),
+        vsock: Some(vsock),
+        rng,
+        net: Some(vec![network]),
+        ..Default::default()
+    };
+
+    Ok(cfg)
+}
+
+fn parse_mac<S>(s: &S) -> Result<MacAddr>
+where
+    S: AsRef<str> + ?Sized + Display,
+{
+    let v: Vec<&str> = s.as_ref().split(':').collect();
+    let mut bytes = [0u8; MAC_ADDR_LEN];
+
+    if v.len() != MAC_ADDR_LEN {
+        return Err(anyhow!(
+            "invalid MAC {} (length {}, expected {})",
+            s,
+            v.len(),
+            MAC_ADDR_LEN
+        ));
+    }
+
+    for i in 0..MAC_ADDR_LEN {
+        if v[i].len() != 2 {
+            return Err(anyhow!(
+                "invalid MAC {} (segment {} length {}, expected {})",
+                s,
+                i,
+                v.len(),
+                2
+            ));
+        }
+
+        bytes[i] =
+            u8::from_str_radix(v[i], 16).context(format!("failed to parse MAC address: {}", s))?;
+    }
+
+    Ok(MacAddr { bytes })
+}

src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs

@@ -0,0 +1,481 @@
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+use std::net::Ipv4Addr;
+use std::path::PathBuf;
+
+pub mod ch_api;
+pub mod net_util;
+mod virtio_devices;
+
+use crate::virtio_devices::RateLimiterConfig;
+pub use net_util::MacAddr;
+
+pub const MAX_NUM_PCI_SEGMENTS: u16 = 16;
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct BalloonConfig {
+    pub size: u64,
+    /// Option to deflate the balloon in case the guest is out of memory.
+    #[serde(default)]
+    pub deflate_on_oom: bool,
+    /// Option to enable free page reporting from the guest.
+    #[serde(default)]
+    pub free_page_reporting: bool,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
+pub struct CmdlineConfig {
+    pub args: String,
+}
+
+impl CmdlineConfig {
+    fn is_empty(&self) -> bool {
+        self.args.is_empty()
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct ConsoleConfig {
+    //#[serde(default = "default_consoleconfig_file")]
+    pub file: Option<PathBuf>,
+    pub mode: ConsoleOutputMode,
+    #[serde(default)]
+    pub iommu: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub enum ConsoleOutputMode {
+    #[default]
+    Off,
+    Pty,
+    Tty,
+    File,
+    Null,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct CpuAffinity {
+    pub vcpu: u8,
+    pub host_cpus: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct CpusConfig {
+    pub boot_vcpus: u8,
+    pub max_vcpus: u8,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub topology: Option<CpuTopology>,
+    #[serde(default)]
+    pub kvm_hyperv: bool,
+    #[serde(skip_serializing_if = "u8_is_zero")]
+    pub max_phys_bits: u8,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub affinity: Option<Vec<CpuAffinity>>,
+    #[serde(default)]
+    pub features: CpuFeatures,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
+pub struct CpuFeatures {
+    #[cfg(target_arch = "x86_64")]
+    #[serde(default)]
+    pub amx: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct CpuTopology {
+    pub threads_per_core: u8,
+    pub cores_per_die: u8,
+    pub dies_per_package: u8,
+    pub packages: u8,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct DeviceConfig {
+    pub path: PathBuf,
+    #[serde(default)]
+    pub iommu: bool,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct DiskConfig {
+    pub path: Option<PathBuf>,
+    #[serde(default)]
+    pub readonly: bool,
+    #[serde(default)]
+    pub direct: bool,
+    #[serde(default)]
+    pub iommu: bool,
+    //#[serde(default = "default_diskconfig_num_queues")]
+    pub num_queues: usize,
+    //#[serde(default = "default_diskconfig_queue_size")]
+    pub queue_size: u16,
+    #[serde(default)]
+    pub vhost_user: bool,
+    pub vhost_socket: Option<String>,
+    #[serde(default)]
+    pub rate_limiter_config: Option<RateLimiterConfig>,
+    #[serde(default)]
+    pub id: Option<String>,
+    // For testing use only. Not exposed in API.
+    #[serde(default)]
+    pub disable_io_uring: bool,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct FsConfig {
+    pub tag: String,
+    pub socket: PathBuf,
+    //#[serde(default = "default_fsconfig_num_queues")]
+    pub num_queues: usize,
+    //#[serde(default = "default_fsconfig_queue_size")]
+    pub queue_size: u16,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub enum HotplugMethod {
+    #[default]
+    Acpi,
+    VirtioMem,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct InitramfsConfig {
+    pub path: PathBuf,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct KernelConfig {
+    pub path: PathBuf,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct MemoryConfig {
+    pub size: u64,
+    #[serde(default)]
+    pub mergeable: bool,
+    #[serde(default)]
+    pub hotplug_method: HotplugMethod,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hotplug_size: Option<u64>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hotplugged_size: Option<u64>,
+    #[serde(default)]
+    pub shared: bool,
+    #[serde(default)]
+    pub hugepages: bool,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hugepage_size: Option<u64>,
+    #[serde(default)]
+    pub prefault: bool,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub zones: Option<Vec<MemoryZoneConfig>>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct MemoryZoneConfig {
+    pub id: String,
+    pub size: u64,
+    #[serde(default)]
+    pub file: Option<PathBuf>,
+    #[serde(default)]
+    pub shared: bool,
+    #[serde(default)]
+    pub hugepages: bool,
+    #[serde(default)]
+    pub hugepage_size: Option<u64>,
+    #[serde(default)]
+    pub host_numa_node: Option<u32>,
+    #[serde(default)]
+    pub hotplug_size: Option<u64>,
+    #[serde(default)]
+    pub hotplugged_size: Option<u64>,
+    #[serde(default)]
+    pub prefault: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+pub struct NetConfig {
+    //#[serde(default = "default_netconfig_tap")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tap: Option<String>,
+    //#[serde(default = "default_netconfig_ip")]
+    pub ip: Ipv4Addr,
+    //#[serde(default = "default_netconfig_mask")]
+    pub mask: Ipv4Addr,
+    //#[serde(default = "default_netconfig_mac")]
+    pub mac: MacAddr,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub host_mac: Option<MacAddr>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mtu: Option<u16>,
+    #[serde(default)]
+    pub iommu: bool,
+    //#[serde(default = "default_netconfig_num_queues")]
+    #[serde(skip_serializing_if = "usize_is_zero")]
+    pub num_queues: usize,
+    //#[serde(default = "default_netconfig_queue_size")]
+    #[serde(skip_serializing_if = "u16_is_zero")]
+    pub queue_size: u16,
+    #[serde(default)]
+    pub vhost_user: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vhost_socket: Option<String>,
+    #[serde(default)]
+    pub vhost_mode: VhostMode,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<String>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fds: Option<Vec<i32>>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rate_limiter_config: Option<RateLimiterConfig>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "u16_is_zero")]
+    pub pci_segment: u16,
+}
+
+impl Default for NetConfig {
+    fn default() -> Self {
+        NetConfig {
+            tap: None,
+            ip: Ipv4Addr::new(0, 0, 0, 0),
+            mask: Ipv4Addr::new(0, 0, 0, 0),
+            mac: MacAddr::default(),
+            host_mac: None,
+            mtu: None,
+            iommu: false,
+            num_queues: 0,
+            queue_size: 0,
+            vhost_user: false,
+            vhost_socket: None,
+            vhost_mode: VhostMode::default(),
+            id: None,
+            fds: None,
+            rate_limiter_config: None,
+            pci_segment: 0,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct NumaConfig {
+    #[serde(default)]
+    pub guest_numa_id: u32,
+    #[serde(default)]
+    pub cpus: Option<Vec<u8>>,
+    #[serde(default)]
+    pub distances: Option<Vec<NumaDistance>>,
+    #[serde(default)]
+    pub memory_zones: Option<Vec<String>>,
+    #[cfg(target_arch = "x86_64")]
+    #[serde(default)]
+    pub sgx_epc_sections: Option<Vec<String>>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct NumaDistance {
+    #[serde(default)]
+    pub destination: u32,
+    #[serde(default)]
+    pub distance: u8,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct PayloadConfig {
+    #[serde(default)]
+    pub firmware: Option<PathBuf>,
+    #[serde(default)]
+    pub kernel: Option<PathBuf>,
+    #[serde(default)]
+    pub cmdline: Option<String>,
+    #[serde(default)]
+    pub initramfs: Option<PathBuf>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct PlatformConfig {
+    //#[serde(default = "default_platformconfig_num_pci_segments")]
+    pub num_pci_segments: u16,
+    #[serde(default)]
+    pub iommu_segments: Option<Vec<u16>>,
+    #[serde(default)]
+    pub serial_number: Option<String>,
+    #[serde(default)]
+    pub uuid: Option<String>,
+    #[serde(default)]
+    pub oem_strings: Option<Vec<String>>,
+    #[cfg(feature = "tdx")]
+    #[serde(default)]
+    pub tdx: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct PmemConfig {
+    pub file: PathBuf,
+    #[serde(default)]
+    pub size: Option<u64>,
+    #[serde(default)]
+    pub iommu: bool,
+    #[serde(default)]
+    pub discard_writes: bool,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct RngConfig {
+    pub src: PathBuf,
+    #[serde(default)]
+    pub iommu: bool,
+}
+
+#[cfg(target_arch = "x86_64")]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct SgxEpcConfig {
+    pub id: String,
+    #[serde(default)]
+    pub size: u64,
+    #[serde(default)]
+    pub prefault: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct UserDeviceConfig {
+    pub socket: PathBuf,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct VdpaConfig {
+    pub path: PathBuf,
+    //#[serde(default = "default_vdpaconfig_num_queues")]
+    pub num_queues: usize,
+    #[serde(default)]
+    pub iommu: bool,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub enum VhostMode {
+    #[default]
+    Client,
+    Server,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct VmConfig {
+    #[serde(default)]
+    pub cpus: CpusConfig,
+    #[serde(default)]
+    pub memory: MemoryConfig,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub kernel: Option<KernelConfig>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub initramfs: Option<InitramfsConfig>,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "CmdlineConfig::is_empty")]
+    pub cmdline: CmdlineConfig,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub payload: Option<PayloadConfig>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub disks: Option<Vec<DiskConfig>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub net: Option<Vec<NetConfig>>,
+    #[serde(default)]
+    pub rng: RngConfig,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub balloon: Option<BalloonConfig>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fs: Option<Vec<FsConfig>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pmem: Option<Vec<PmemConfig>>,
+    //#[serde(default = "ConsoleConfig::default_serial")]
+    pub serial: ConsoleConfig,
+    //#[serde(default = "ConsoleConfig::default_console")]
+    pub console: ConsoleConfig,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub devices: Option<Vec<DeviceConfig>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub user_devices: Option<Vec<UserDeviceConfig>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vdpa: Option<Vec<VdpaConfig>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vsock: Option<VsockConfig>,
+    #[serde(default)]
+    pub iommu: bool,
+    #[cfg(target_arch = "x86_64")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sgx_epc: Option<Vec<SgxEpcConfig>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub numa: Option<Vec<NumaConfig>>,
+    #[serde(default)]
+    pub watchdog: bool,
+    #[cfg(feature = "guest_debug")]
+    pub gdb: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub platform: Option<PlatformConfig>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub struct VsockConfig {
+    pub cid: u64,
+    pub socket: PathBuf,
+    #[serde(default)]
+    pub iommu: bool,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
+//--------------------------------------------------------------------
+// For serde serialization
+
+#[allow(clippy::trivially_copy_pass_by_ref)]
+fn u8_is_zero(v: &u8) -> bool {
+    *v == 0
+}
+
+#[allow(clippy::trivially_copy_pass_by_ref)]
+fn usize_is_zero(v: &usize) -> bool {
+    *v == 0
+}
+
+#[allow(clippy::trivially_copy_pass_by_ref)]
+fn u16_is_zero(v: &u16) -> bool {
+    *v == 0
+}

src/runtime-rs/crates/hypervisor/ch-config/src/net_util.rs

@@ -0,0 +1,32 @@
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize, Serializer};
+use std::fmt;
+
+pub const MAC_ADDR_LEN: usize = 6;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Default)]
+pub struct MacAddr {
+    pub bytes: [u8; MAC_ADDR_LEN],
+}
+
+// Note: Implements ToString automatically.
+impl fmt::Display for MacAddr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let b = &self.bytes;
+        write!(
+            f,
+            "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
+            b[0], b[1], b[2], b[3], b[4], b[5]
+        )
+    }
+}
+
+// Requried to remove the `bytes` member from the serialized JSON!
+impl Serialize for MacAddr {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        self.to_string().serialize(serializer)
+    }
+}

src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs

@@ -0,0 +1,19 @@
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
+pub struct TokenBucketConfig {
+    pub size: u64,
+    pub one_time_burst: Option<u64>,
+    pub refill_time: u64,
+}
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct RateLimiterConfig {
+    pub bandwidth: Option<TokenBucketConfig>,
+    pub ops: Option<TokenBucketConfig>,
+}

src/runtime-rs/crates/hypervisor/src/ch/inner.rs

@@ -0,0 +1,148 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use super::HypervisorState;
+use crate::device::Device;
+use crate::VmmState;
+use anyhow::Result;
+use async_trait::async_trait;
+use kata_types::capabilities::{Capabilities, CapabilityBits};
+use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
+use kata_types::config::hypervisor::HYPERVISOR_NAME_CH;
+use persist::sandbox_persist::Persist;
+use std::os::unix::net::UnixStream;
+use tokio::process::Child;
+use tokio::sync::watch::{channel, Receiver, Sender};
+use tokio::task::JoinHandle;
+
+#[derive(Debug)]
+pub struct CloudHypervisorInner {
+    pub(crate) state: VmmState,
+    pub(crate) id: String,
+
+    pub(crate) api_socket: Option<UnixStream>,
+    pub(crate) extra_args: Option<Vec<String>>,
+
+    pub(crate) config: Option<HypervisorConfig>,
+
+    pub(crate) process: Option<Child>,
+    pub(crate) pid: Option<u32>,
+
+    pub(crate) timeout_secs: i32,
+
+    pub(crate) netns: Option<String>,
+
+    // Sandbox-specific directory
+    pub(crate) vm_path: String,
+
+    // Hypervisor runtime directory
+    pub(crate) run_dir: String,
+
+    // Subdirectory of vm_path.
+    pub(crate) jailer_root: String,
+
+    /// List of devices that will be added to the VM once it boots
+    pub(crate) pending_devices: Option<Vec<Device>>,
+
+    pub(crate) _capabilities: Capabilities,
+
+    pub(crate) shutdown_tx: Option<Sender<bool>>,
+    pub(crate) shutdown_rx: Option<Receiver<bool>>,
+    pub(crate) tasks: Option<Vec<JoinHandle<Result<()>>>>,
+}
+
+unsafe impl Send for CloudHypervisorInner {}
+unsafe impl Sync for CloudHypervisorInner {}
+
+const CH_DEFAULT_TIMEOUT_SECS: u32 = 10;
+
+impl CloudHypervisorInner {
+    pub fn new() -> Self {
+        let mut capabilities = Capabilities::new();
+        capabilities.set(
+            CapabilityBits::BlockDeviceSupport
+                | CapabilityBits::BlockDeviceHotplugSupport
+                | CapabilityBits::FsSharingSupport,
+        );
+
+        let (tx, rx) = channel(true);
+
+        Self {
+            api_socket: None,
+            extra_args: None,
+
+            process: None,
+            pid: None,
+
+            config: None,
+            state: VmmState::NotReady,
+            timeout_secs: CH_DEFAULT_TIMEOUT_SECS as i32,
+            id: String::default(),
+            jailer_root: String::default(),
+            vm_path: String::default(),
+            run_dir: String::default(),
+            netns: None,
+            pending_devices: None,
+            _capabilities: capabilities,
+            shutdown_tx: Some(tx),
+            shutdown_rx: Some(rx),
+            tasks: None,
+        }
+    }
+
+    pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        self.config = Some(config);
+    }
+
+    pub fn hypervisor_config(&self) -> HypervisorConfig {
+        self.config.clone().unwrap_or_default()
+    }
+}
+
+impl Default for CloudHypervisorInner {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl Persist for CloudHypervisorInner {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    // Return a state object that will be saved by the caller.
+    async fn save(&self) -> Result<Self::State> {
+        Ok(HypervisorState {
+            hypervisor_type: HYPERVISOR_NAME_CH.to_string(),
+            id: self.id.clone(),
+            vm_path: self.vm_path.clone(),
+            jailed: false,
+            jailer_root: String::default(),
+            netns: None,
+            config: self.hypervisor_config(),
+            run_dir: self.run_dir.clone(),
+            cached_block_devices: Default::default(),
+            ..Default::default()
+        })
+    }
+
+    // Set the hypervisor state to the specified state
+    async fn restore(
+        _hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let ch = Self {
+            config: Some(hypervisor_state.config),
+            state: VmmState::NotReady,
+            id: hypervisor_state.id,
+            vm_path: hypervisor_state.vm_path,
+            run_dir: hypervisor_state.run_dir,
+
+            ..Default::default()
+        };
+
+        Ok(ch)
+    }
+}

src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs

@@ -0,0 +1,235 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use super::inner::CloudHypervisorInner;
+use crate::device::{Device, ShareFsDeviceConfig};
+use crate::HybridVsockConfig;
+use crate::VmmState;
+use anyhow::{anyhow, Context, Result};
+use ch_config::ch_api::cloud_hypervisor_vm_fs_add;
+use ch_config::{FsConfig, PmemConfig};
+use safe_path::scoped_join;
+use std::convert::TryFrom;
+use std::path::PathBuf;
+
+const VIRTIO_FS: &str = "virtio-fs";
+
+impl CloudHypervisorInner {
+    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+        if self.state != VmmState::VmRunning {
+            let mut devices: Vec<Device> = if let Some(devices) = self.pending_devices.take() {
+                devices
+            } else {
+                vec![]
+            };
+
+            devices.insert(0, device);
+
+            self.pending_devices = Some(devices);
+
+            return Ok(());
+        }
+
+        self.handle_add_device(device).await?;
+
+        Ok(())
+    }
+
+    async fn handle_add_device(&mut self, device: Device) -> Result<()> {
+        match device {
+            Device::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
+            Device::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
+            _ => return Err(anyhow!("unhandled device: {:?}", device)),
+        }
+    }
+
+    /// Add the device that were requested to be added before the VMM was
+    /// started.
+    #[allow(dead_code)]
+    pub(crate) async fn handle_pending_devices_after_boot(&mut self) -> Result<()> {
+        if self.state != VmmState::VmRunning {
+            return Err(anyhow!(
+                "cannot handle pending devices with VMM state {:?}",
+                self.state
+            ));
+        }
+
+        if let Some(mut devices) = self.pending_devices.take() {
+            while let Some(dev) = devices.pop() {
+                self.add_device(dev).await.context("add_device")?;
+            }
+        }
+
+        Ok(())
+    }
+
+    pub(crate) async fn remove_device(&mut self, _device: Device) -> Result<()> {
+        Ok(())
+    }
+
+    async fn handle_share_fs_device(&mut self, cfg: ShareFsDeviceConfig) -> Result<()> {
+        if cfg.fs_type != VIRTIO_FS {
+            return Err(anyhow!("cannot handle share fs type: {:?}", cfg.fs_type));
+        }
+
+        let socket = self
+            .api_socket
+            .as_ref()
+            .ok_or("missing socket")
+            .map_err(|e| anyhow!(e))?;
+
+        let num_queues: usize = if cfg.queue_num > 0 {
+            cfg.queue_num as usize
+        } else {
+            1
+        };
+
+        let queue_size: u16 = if cfg.queue_num > 0 {
+            u16::try_from(cfg.queue_size)?
+        } else {
+            1024
+        };
+
+        let socket_path = if cfg.sock_path.starts_with('/') {
+            PathBuf::from(cfg.sock_path)
+        } else {
+            scoped_join(&self.vm_path, cfg.sock_path)?
+        };
+
+        let fs_config = FsConfig {
+            tag: cfg.mount_tag,
+            socket: socket_path,
+            num_queues,
+            queue_size,
+            ..Default::default()
+        };
+
+        let response = cloud_hypervisor_vm_fs_add(
+            socket.try_clone().context("failed to clone socket")?,
+            fs_config,
+        )
+        .await?;
+
+        if let Some(detail) = response {
+            debug!(sl!(), "fs add response: {:?}", detail);
+        }
+
+        Ok(())
+    }
+
+    async fn handle_hvsock_device(&mut self, _cfg: &HybridVsockConfig) -> Result<()> {
+        Ok(())
+    }
+
+    pub(crate) async fn get_shared_fs_devices(&mut self) -> Result<Option<Vec<FsConfig>>> {
+        let pending_root_devices = self.pending_devices.take();
+
+        let mut root_devices = Vec::<FsConfig>::new();
+
+        if let Some(devices) = pending_root_devices {
+            for dev in devices {
+                match dev {
+                    Device::ShareFsDevice(dev) => {
+                        let settings = ShareFsSettings::new(dev, self.vm_path.clone());
+
+                        let fs_cfg = FsConfig::try_from(settings)?;
+
+                        root_devices.push(fs_cfg);
+                    }
+                    _ => continue,
+                };
+            }
+
+            Ok(Some(root_devices))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub(crate) async fn get_boot_file(&mut self) -> Result<PathBuf> {
+        if let Some(ref config) = self.config {
+            let boot_info = &config.boot_info;
+
+            let file = if !boot_info.initrd.is_empty() {
+                boot_info.initrd.clone()
+            } else if !boot_info.image.is_empty() {
+                boot_info.image.clone()
+            } else {
+                return Err(anyhow!("missing boot file (no image or initrd)"));
+            };
+
+            Ok(PathBuf::from(file))
+        } else {
+            Err(anyhow!("no hypervisor config"))
+        }
+    }
+
+    pub(crate) async fn get_pmem_devices(&mut self) -> Result<Option<Vec<PmemConfig>>> {
+        let file = self.get_boot_file().await?;
+
+        let pmem_cfg = PmemConfig {
+            file,
+            size: None,
+            iommu: false,
+            discard_writes: true,
+            id: None,
+            pci_segment: 0,
+        };
+
+        let pmem_devices = vec![pmem_cfg];
+
+        Ok(Some(pmem_devices))
+    }
+}
+
+#[derive(Debug)]
+pub struct ShareFsSettings {
+    cfg: ShareFsDeviceConfig,
+    vm_path: String,
+}
+
+impl ShareFsSettings {
+    pub fn new(cfg: ShareFsDeviceConfig, vm_path: String) -> Self {
+        ShareFsSettings { cfg, vm_path }
+    }
+}
+
+impl TryFrom<ShareFsSettings> for FsConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(settings: ShareFsSettings) -> Result<Self, Self::Error> {
+        let cfg = settings.cfg;
+        let vm_path = settings.vm_path;
+
+        let num_queues: usize = if cfg.queue_num > 0 {
+            cfg.queue_num as usize
+        } else {
+            1
+        };
+
+        let queue_size: u16 = if cfg.queue_num > 0 {
+            u16::try_from(cfg.queue_size)?
+        } else {
+            1024
+        };
+
+        let socket_path = if cfg.sock_path.starts_with('/') {
+            PathBuf::from(cfg.sock_path)
+        } else {
+            PathBuf::from(vm_path).join(cfg.sock_path)
+        };
+
+        let fs_cfg = FsConfig {
+            tag: cfg.mount_tag,
+            socket: socket_path,
+            num_queues,
+            queue_size,
+            ..Default::default()
+        };
+
+        Ok(fs_cfg)
+    }
+}

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -0,0 +1,486 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use super::inner::CloudHypervisorInner;
+use crate::ch::utils::get_api_socket_path;
+use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path};
+use crate::Device;
+use crate::VsockConfig;
+use crate::{VcpuThreadIds, VmmState};
+use anyhow::{anyhow, Context, Result};
+use ch_config::ch_api::{
+    cloud_hypervisor_vm_create, cloud_hypervisor_vm_start, cloud_hypervisor_vmm_ping,
+    cloud_hypervisor_vmm_shutdown,
+};
+use core::future::poll_fn;
+use futures::executor::block_on;
+use futures::future::join_all;
+use kata_types::capabilities::{Capabilities, CapabilityBits};
+use std::fs::create_dir_all;
+use std::os::unix::net::UnixStream;
+use std::path::Path;
+use std::process::Stdio;
+use tokio::io::AsyncBufReadExt;
+use tokio::io::BufReader;
+use tokio::process::{Child, Command};
+use tokio::sync::watch::Receiver;
+use tokio::task;
+use tokio::task::JoinHandle;
+use tokio::time::Duration;
+
+const CH_NAME: &str = "cloud-hypervisor";
+
+/// Number of milliseconds to wait before retrying a CH operation.
+const CH_POLL_TIME_MS: u64 = 50;
+
+impl CloudHypervisorInner {
+    async fn start_hypervisor(&mut self, timeout_secs: i32) -> Result<()> {
+        self.cloud_hypervisor_launch(timeout_secs)
+            .await
+            .context("launch failed")?;
+
+        self.cloud_hypervisor_setup_comms()
+            .await
+            .context("comms setup failed")?;
+
+        self.cloud_hypervisor_check_running()
+            .await
+            .context("hypervisor running check failed")?;
+
+        self.state = VmmState::VmmServerReady;
+
+        Ok(())
+    }
+
+    async fn boot_vm(&mut self) -> Result<()> {
+        let shared_fs_devices = self.get_shared_fs_devices().await?;
+
+        let pmem_devices = self.get_pmem_devices().await?;
+
+        let socket = self
+            .api_socket
+            .as_ref()
+            .ok_or("missing socket")
+            .map_err(|e| anyhow!(e))?;
+
+        let sandbox_path = get_sandbox_path(&self.id)?;
+
+        std::fs::create_dir_all(sandbox_path.clone()).context("failed to create sandbox path")?;
+
+        let vsock_socket_path = get_vsock_path(&self.id)?;
+
+        let response = cloud_hypervisor_vm_create(
+            sandbox_path,
+            vsock_socket_path,
+            socket.try_clone().context("failed to clone socket")?,
+            shared_fs_devices,
+            pmem_devices,
+        )
+        .await?;
+
+        if let Some(detail) = response {
+            debug!(sl!(), "vm boot response: {:?}", detail);
+        }
+
+        let response =
+            cloud_hypervisor_vm_start(socket.try_clone().context("failed to clone socket")?)
+                .await?;
+
+        if let Some(detail) = response {
+            debug!(sl!(), "vm start response: {:?}", detail);
+        }
+
+        self.state = VmmState::VmRunning;
+
+        Ok(())
+    }
+
+    async fn cloud_hypervisor_setup_comms(&mut self) -> Result<()> {
+        let api_socket_path = get_api_socket_path(&self.id)?;
+
+        // The hypervisor has just been spawned, but may not yet have created
+        // the API socket, so repeatedly try to connect for up to
+        // timeout_secs.
+        let join_handle: JoinHandle<Result<UnixStream>> =
+            task::spawn_blocking(move || -> Result<UnixStream> {
+                let api_socket: UnixStream;
+
+                loop {
+                    let result = UnixStream::connect(api_socket_path.clone());
+
+                    if let Ok(result) = result {
+                        api_socket = result;
+                        break;
+                    }
+
+                    std::thread::sleep(Duration::from_millis(CH_POLL_TIME_MS));
+                }
+
+                Ok(api_socket)
+            });
+
+        let timeout_msg = format!(
+            "API socket connect timed out after {} seconds",
+            self.timeout_secs
+        );
+
+        let result =
+            tokio::time::timeout(Duration::from_secs(self.timeout_secs as u64), join_handle)
+                .await
+                .context(timeout_msg)?;
+
+        let result = result?;
+
+        let api_socket = result?;
+
+        self.api_socket = Some(api_socket);
+
+        Ok(())
+    }
+
+    async fn cloud_hypervisor_check_running(&mut self) -> Result<()> {
+        let timeout_secs = self.timeout_secs;
+
+        let timeout_msg = format!(
+            "API socket connect timed out after {} seconds",
+            timeout_secs
+        );
+
+        let join_handle = self.cloud_hypervisor_ping_until_ready(CH_POLL_TIME_MS);
+
+        let result = tokio::time::timeout(Duration::new(timeout_secs as u64, 0), join_handle)
+            .await
+            .context(timeout_msg)?;
+
+        result
+    }
+
+    async fn cloud_hypervisor_ensure_not_launched(&self) -> Result<()> {
+        if let Some(child) = &self.process {
+            return Err(anyhow!(
+                "{} already running with PID {}",
+                CH_NAME,
+                child.id().unwrap_or(0)
+            ));
+        }
+
+        Ok(())
+    }
+
+    async fn cloud_hypervisor_launch(&mut self, _timeout_secs: i32) -> Result<()> {
+        self.cloud_hypervisor_ensure_not_launched().await?;
+
+        let debug = false;
+
+        let disable_seccomp = true;
+
+        let api_socket_path = get_api_socket_path(&self.id)?;
+
+        let _ = std::fs::remove_file(api_socket_path.clone());
+
+        let binary_path = self
+            .config
+            .as_ref()
+            .ok_or("no hypervisor config for CH")
+            .map_err(|e| anyhow!(e))?
+            .path
+            .to_string();
+
+        let path = Path::new(&binary_path).canonicalize()?;
+
+        let mut cmd = Command::new(path);
+
+        cmd.current_dir("/");
+
+        cmd.stdin(Stdio::null());
+        cmd.stdout(Stdio::piped());
+        cmd.stderr(Stdio::piped());
+
+        cmd.env("RUST_BACKTRACE", "full");
+
+        cmd.args(["--api-socket", &api_socket_path]);
+
+        if let Some(extra_args) = &self.extra_args {
+            cmd.args(extra_args);
+        }
+
+        if debug {
+            cmd.arg("-v");
+        }
+
+        if disable_seccomp {
+            cmd.args(["--seccomp", "false"]);
+        }
+
+        let child = cmd.spawn().context(format!("{} spawn failed", CH_NAME))?;
+
+        // Save process PID
+        self.pid = child.id();
+
+        let shutdown = self
+            .shutdown_rx
+            .as_ref()
+            .ok_or("no receiver channel")
+            .map_err(|e| anyhow!(e))?
+            .clone();
+
+        let ch_outputlogger_task = tokio::spawn(cloud_hypervisor_log_output(child, shutdown));
+
+        let tasks = vec![ch_outputlogger_task];
+
+        self.tasks = Some(tasks);
+
+        Ok(())
+    }
+
+    async fn cloud_hypervisor_shutdown(&mut self) -> Result<()> {
+        let socket = self
+            .api_socket
+            .as_ref()
+            .ok_or("missing socket")
+            .map_err(|e| anyhow!(e))?;
+
+        let response =
+            cloud_hypervisor_vmm_shutdown(socket.try_clone().context("shutdown failed")?).await?;
+
+        if let Some(detail) = response {
+            debug!(sl!(), "shutdown response: {:?}", detail);
+        }
+
+        // Trigger a controlled shutdown
+        self.shutdown_tx
+            .as_mut()
+            .ok_or("no shutdown channel")
+            .map_err(|e| anyhow!(e))?
+            .send(true)
+            .map_err(|e| anyhow!(e).context("failed to request shutdown"))?;
+
+        let tasks = self
+            .tasks
+            .take()
+            .ok_or("no tasks")
+            .map_err(|e| anyhow!(e))?;
+
+        let results = join_all(tasks).await;
+
+        let mut wait_errors: Vec<tokio::task::JoinError> = vec![];
+
+        for result in results {
+            if let Err(e) = result {
+                eprintln!("wait task error: {:#?}", e);
+
+                wait_errors.push(e);
+            }
+        }
+
+        if wait_errors.is_empty() {
+            Ok(())
+        } else {
+            Err(anyhow!("wait all tasks failed: {:#?}", wait_errors))
+        }
+    }
+
+    #[allow(dead_code)]
+    async fn cloud_hypervisor_wait(&mut self) -> Result<()> {
+        let mut child = self
+            .process
+            .take()
+            .ok_or(format!("{} not running", CH_NAME))
+            .map_err(|e| anyhow!(e))?;
+
+        let _pid = child
+            .id()
+            .ok_or(format!("{} missing PID", CH_NAME))
+            .map_err(|e| anyhow!(e))?;
+
+        // Note that this kills _and_ waits for the process!
+        child.kill().await?;
+
+        Ok(())
+    }
+
+    async fn cloud_hypervisor_ping_until_ready(&mut self, _poll_time_ms: u64) -> Result<()> {
+        let socket = self
+            .api_socket
+            .as_ref()
+            .ok_or("missing socket")
+            .map_err(|e| anyhow!(e))?;
+
+        loop {
+            let response =
+                cloud_hypervisor_vmm_ping(socket.try_clone().context("failed to clone socket")?)
+                    .await
+                    .context("ping failed");
+
+            if let Ok(response) = response {
+                if let Some(detail) = response {
+                    debug!(sl!(), "ping response: {:?}", detail);
+                }
+                break;
+            }
+
+            tokio::time::sleep(Duration::from_millis(CH_POLL_TIME_MS)).await;
+        }
+
+        Ok(())
+    }
+
+    pub(crate) async fn prepare_vm(&mut self, id: &str, netns: Option<String>) -> Result<()> {
+        self.id = id.to_string();
+        self.state = VmmState::NotReady;
+
+        self.setup_environment().await?;
+
+        self.netns = netns;
+
+        let vsock_cfg = VsockConfig::new(self.id.clone()).await?;
+
+        let dev = Device::Vsock(vsock_cfg);
+        self.add_device(dev).await.context("add vsock device")?;
+
+        self.start_hypervisor(self.timeout_secs).await?;
+
+        Ok(())
+    }
+
+    async fn setup_environment(&mut self) -> Result<()> {
+        // run_dir and vm_path are the same (shared)
+        self.run_dir = get_sandbox_path(&self.id)?;
+        self.vm_path = self.run_dir.to_string();
+
+        create_dir_all(&self.run_dir)
+            .with_context(|| anyhow!("failed to create sandbox directory {}", self.run_dir))?;
+
+        if !self.jailer_root.is_empty() {
+            create_dir_all(self.jailer_root.as_str())
+                .map_err(|e| anyhow!("Failed to create dir {} err : {:?}", self.jailer_root, e))?;
+        }
+
+        Ok(())
+    }
+
+    pub(crate) async fn start_vm(&mut self, timeout_secs: i32) -> Result<()> {
+        self.setup_environment().await?;
+
+        self.timeout_secs = timeout_secs;
+
+        self.boot_vm().await?;
+
+        Ok(())
+    }
+
+    pub(crate) fn stop_vm(&mut self) -> Result<()> {
+        block_on(self.cloud_hypervisor_shutdown())?;
+
+        Ok(())
+    }
+
+    pub(crate) fn pause_vm(&self) -> Result<()> {
+        Ok(())
+    }
+
+    pub(crate) fn resume_vm(&self) -> Result<()> {
+        Ok(())
+    }
+
+    pub(crate) async fn save_vm(&self) -> Result<()> {
+        Ok(())
+    }
+
+    pub(crate) async fn get_agent_socket(&self) -> Result<String> {
+        const HYBRID_VSOCK_SCHEME: &str = "hvsock";
+
+        let vsock_path = get_vsock_path(&self.id)?;
+
+        let uri = format!("{}://{}", HYBRID_VSOCK_SCHEME, vsock_path);
+
+        Ok(uri)
+    }
+
+    pub(crate) async fn disconnect(&mut self) {
+        self.state = VmmState::NotReady;
+    }
+
+    pub(crate) async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        Ok(VcpuThreadIds::default())
+    }
+
+    pub(crate) async fn cleanup(&self) -> Result<()> {
+        Ok(())
+    }
+
+    pub(crate) async fn get_pids(&self) -> Result<Vec<u32>> {
+        Ok(Vec::<u32>::new())
+    }
+
+    pub(crate) async fn check(&self) -> Result<()> {
+        Ok(())
+    }
+
+    pub(crate) async fn get_jailer_root(&self) -> Result<String> {
+        let root_path = get_jailer_root(&self.id)?;
+
+        std::fs::create_dir_all(&root_path)?;
+
+        Ok(root_path)
+    }
+
+    pub(crate) async fn capabilities(&self) -> Result<Capabilities> {
+        let mut caps = Capabilities::default();
+        caps.set(CapabilityBits::FsSharingSupport);
+        Ok(caps)
+    }
+}
+
+// Log all output from the CH process until a shutdown signal is received.
+// When that happens, stop logging and wait for the child process to finish
+// before returning.
+async fn cloud_hypervisor_log_output(mut child: Child, mut shutdown: Receiver<bool>) -> Result<()> {
+    let stdout = child
+        .stdout
+        .as_mut()
+        .ok_or("failed to get child stdout")
+        .map_err(|e| anyhow!(e))?;
+
+    let stdout_reader = BufReader::new(stdout);
+    let mut stdout_lines = stdout_reader.lines();
+
+    let stderr = child
+        .stderr
+        .as_mut()
+        .ok_or("failed to get child stderr")
+        .map_err(|e| anyhow!(e))?;
+
+    let stderr_reader = BufReader::new(stderr);
+    let mut stderr_lines = stderr_reader.lines();
+
+    loop {
+        tokio::select! {
+            _ = shutdown.changed() => {
+                info!(sl!(), "got shutdown request");
+                break;
+            },
+            stderr_line = poll_fn(|cx| Pin::new(&mut stderr_lines).poll_next_line(cx)) => {
+                if let Ok(line) = stderr_line {
+                    let line = line.ok_or("missing stderr line").map_err(|e| anyhow!(e))?;
+
+                    info!(sl!(), "{:?}", line; "stream" => "stderr");
+                }
+            },
+            stdout_line = poll_fn(|cx| Pin::new(&mut stdout_lines).poll_next_line(cx)) => {
+                if let Ok(line) = stdout_line {
+                    let line = line.ok_or("missing stdout line").map_err(|e| anyhow!(e))?;
+
+                    info!(sl!(), "{:?}", line; "stream" => "stdout");
+                }
+            },
+        };
+    }
+
+    // Note that this kills _and_ waits for the process!
+    child.kill().await?;
+
+    Ok(())
+}

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -0,0 +1,163 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use super::HypervisorState;
+use crate::{device::Device, Hypervisor, VcpuThreadIds};
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use kata_types::capabilities::Capabilities;
+use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
+use persist::sandbox_persist::Persist;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+// Convenience macro to obtain the scope logger
+#[macro_export]
+macro_rules! sl {
+      () => {
+          slog_scope::logger().new(o!("subsystem" => "cloud-hypervisor"))
+      };
+  }
+
+mod inner;
+mod inner_device;
+mod inner_hypervisor;
+mod utils;
+
+use inner::CloudHypervisorInner;
+
+#[derive(Debug, Default, Clone)]
+pub struct CloudHypervisor {
+    inner: Arc<RwLock<CloudHypervisorInner>>,
+}
+
+unsafe impl Send for CloudHypervisor {}
+unsafe impl Sync for CloudHypervisor {}
+
+impl CloudHypervisor {
+    pub fn new() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(CloudHypervisorInner::new())),
+        }
+    }
+
+    pub async fn set_hypervisor_config(&mut self, config: HypervisorConfig) {
+        let mut inner = self.inner.write().await;
+        inner.set_hypervisor_config(config)
+    }
+}
+
+#[async_trait]
+impl Hypervisor for CloudHypervisor {
+    async fn prepare_vm(&self, id: &str, netns: Option<String>) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.prepare_vm(id, netns).await
+    }
+
+    async fn start_vm(&self, timeout: i32) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.start_vm(timeout).await
+    }
+
+    async fn stop_vm(&self) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.stop_vm()
+    }
+
+    async fn pause_vm(&self) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.pause_vm()
+    }
+
+    async fn resume_vm(&self) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.resume_vm()
+    }
+
+    async fn save_vm(&self) -> Result<()> {
+        let inner = self.inner.write().await;
+        inner.save_vm().await
+    }
+
+    async fn add_device(&self, device: Device) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.add_device(device).await
+    }
+
+    async fn remove_device(&self, device: Device) -> Result<()> {
+        let mut inner = self.inner.write().await;
+        inner.remove_device(device).await
+    }
+
+    async fn get_agent_socket(&self) -> Result<String> {
+        let inner = self.inner.write().await;
+        inner.get_agent_socket().await
+    }
+
+    async fn disconnect(&self) {
+        let mut inner = self.inner.write().await;
+        inner.disconnect().await
+    }
+
+    async fn hypervisor_config(&self) -> HypervisorConfig {
+        let inner = self.inner.write().await;
+        inner.hypervisor_config()
+    }
+
+    async fn get_thread_ids(&self) -> Result<VcpuThreadIds> {
+        let inner = self.inner.read().await;
+        inner.get_thread_ids().await
+    }
+
+    async fn cleanup(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.cleanup().await
+    }
+
+    async fn get_pids(&self) -> Result<Vec<u32>> {
+        let inner = self.inner.read().await;
+        inner.get_pids().await
+    }
+
+    async fn check(&self) -> Result<()> {
+        let inner = self.inner.read().await;
+        inner.check().await
+    }
+
+    async fn get_jailer_root(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_jailer_root().await
+    }
+
+    async fn save_state(&self) -> Result<HypervisorState> {
+        self.save().await
+    }
+
+    async fn capabilities(&self) -> Result<Capabilities> {
+        let inner = self.inner.read().await;
+        inner.capabilities().await
+    }
+}
+
+#[async_trait]
+impl Persist for CloudHypervisor {
+    type State = HypervisorState;
+    type ConstructorArgs = ();
+
+    async fn save(&self) -> Result<Self::State> {
+        let inner = self.inner.read().await;
+        inner.save().await.context("save CH hypervisor state")
+    }
+
+    async fn restore(
+        hypervisor_args: Self::ConstructorArgs,
+        hypervisor_state: Self::State,
+    ) -> Result<Self> {
+        let inner = CloudHypervisorInner::restore(hypervisor_args, hypervisor_state).await?;
+        Ok(Self {
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+}

src/runtime-rs/crates/hypervisor/src/ch/utils.rs

@@ -0,0 +1,53 @@
+// Copyright (c) 2022-2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::Result;
+use shim_interface::KATA_PATH;
+
+// The socket used to connect to CH. This is used for CH API communications.
+const CH_API_SOCKET_NAME: &str = "ch-api.sock";
+
+// The socket that allows runtime-rs to connect direct through to the Kata
+// Containers agent running inside the CH hosted VM.
+const CH_VM_SOCKET_NAME: &str = "ch-vm.sock";
+
+const CH_JAILER_DIR: &str = "root";
+
+// Return the path for a _hypothetical_ sandbox: the path does *not* exist
+// yet, and for this reason safe-path cannot be used.
+pub fn get_sandbox_path(id: &str) -> Result<String> {
+    let path = [KATA_PATH, id].join("/");
+
+    Ok(path)
+}
+
+// Return the path for a _hypothetical_ API socket path:
+// the path does *not* exist yet, and for this reason safe-path cannot be
+// used.
+pub fn get_api_socket_path(id: &str) -> Result<String> {
+    let sandbox_path = get_sandbox_path(id)?;
+
+    let path = [&sandbox_path, CH_API_SOCKET_NAME].join("/");
+
+    Ok(path)
+}
+
+// Return the path for a _hypothetical_ sandbox specific VSOCK socket path:
+// the path does *not* exist yet, and for this reason safe-path cannot be
+// used.
+pub fn get_vsock_path(id: &str) -> Result<String> {
+    let sandbox_path = get_sandbox_path(id)?;
+
+    let path = [&sandbox_path, CH_VM_SOCKET_NAME].join("/");
+
+    Ok(path)
+}
+
+pub fn get_jailer_root(id: &str) -> Result<String> {
+    let sandbox_path = get_sandbox_path(id)?;
+
+    let path = [&sandbox_path, CH_JAILER_DIR].join("/");
+
+    Ok(path)
+}

src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs

@@ -8,7 +8,7 @@ use crate::HypervisorConfig;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 
-#[derive(Serialize, Deserialize, Default)]
+#[derive(Serialize, Deserialize, Default, Clone, Debug)]
 pub struct HypervisorState {
     // Type of hypervisor, E.g. dragonball/qemu/firecracker/acrn.
     pub hypervisor_type: String,

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -19,11 +19,17 @@ pub use kernel_param::Param;
 mod utils;
 use std::collections::HashMap;
 
+#[cfg(feature = "cloud-hypervisor")]
+pub mod ch;
+
 use anyhow::Result;
 use async_trait::async_trait;
 use hypervisor_persist::HypervisorState;
 use kata_types::capabilities::Capabilities;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
+
+pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH;
+
 // Config which driver to use as vm root dev
 const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk";
 const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem";
@@ -48,7 +54,7 @@ const SHMEM: &str = "shmem";
 pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 pub const HYPERVISOR_QEMU: &str = "qemu";
 
-#[derive(PartialEq)]
+#[derive(PartialEq, Debug, Clone)]
 pub(crate) enum VmmState {
     NotReady,
     VmmServerReady,
@@ -56,7 +62,7 @@ pub(crate) enum VmmState {
 }
 
 // vcpu mapping from vcpu number to thread number
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct VcpuThreadIds {
     pub vcpus: HashMap<u32, u32>,
 }

src/runtime-rs/crates/runtimes/virt_container/Cargo.toml

@@ -35,3 +35,9 @@ oci = { path = "../../../../libs/oci" }
 persist = { path = "../../persist"}
 resource = { path = "../../resource" }
 
+[features]
+default = []
+
+# Feature is not yet complete, so not enabled by default.
+# See https://github.com/kata-containers/kata-containers/issues/6264.
+cloud-hypervisor = []

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -25,6 +25,12 @@ use hypervisor::{qemu::Qemu, HYPERVISOR_QEMU};
 use kata_types::config::{
     hypervisor::register_hypervisor_plugin, DragonballConfig, QemuConfig, TomlConfig,
 };
+
+#[cfg(feature = "cloud-hypervisor")]
+use hypervisor::ch::CloudHypervisor;
+#[cfg(feature = "cloud-hypervisor")]
+use kata_types::config::{hypervisor::HYPERVISOR_NAME_CH, CloudHypervisorConfig};
+
 use resource::ResourceManager;
 use sandbox::VIRTCONTAINER;
 use tokio::sync::mpsc::Sender;
@@ -39,8 +45,16 @@ impl RuntimeHandler for VirtContainer {
         // register
         let dragonball_config = Arc::new(DragonballConfig::new());
         register_hypervisor_plugin("dragonball", dragonball_config);
+
         let qemu_config = Arc::new(QemuConfig::new());
         register_hypervisor_plugin("qemu", qemu_config);
+
+        #[cfg(feature = "cloud-hypervisor")]
+        {
+            let ch_config = Arc::new(CloudHypervisorConfig::new());
+            register_hypervisor_plugin(HYPERVISOR_NAME_CH, ch_config);
+        }
+
         Ok(())
     }
 
@@ -118,6 +132,17 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
                 .await;
             Ok(Arc::new(hypervisor))
         }
+
+        #[cfg(feature = "cloud-hypervisor")]
+        HYPERVISOR_NAME_CH => {
+            let mut hypervisor = CloudHypervisor::new();
+
+            hypervisor
+                .set_hypervisor_config(hypervisor_config.clone())
+                .await;
+
+            Ok(Arc::new(hypervisor))
+        }
         _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)),
     }
 }