CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #3843
Signed-off-by: stevenhorsman <steven@uk.ibm.com>

snap/snapcraft.yaml

@@ -264,12 +264,12 @@ parts:
       # download source
       qemu_dir=${SNAPCRAFT_STAGE}/qemu
       rm -rf "${qemu_dir}"
-      git clone --branch ${branch} --single-branch ${url} "${qemu_dir}"
+      git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
       cd ${qemu_dir}
       [ -z "${commit}" ] || git checkout ${commit}
 
-      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb ui/keycodemapdb/
-      [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone
+      [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
+      [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
 
       # Apply branch patches
       [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}"
@@ -320,20 +320,23 @@ parts:
     plugin: nil
     after: [godeps]
     override-build: |
-      sudo apt-get -y update
-      sudo apt-get -y install ca-certificates curl gnupg lsb-release
-      curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-      echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      sudo apt-get -y update
-      sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-      sudo systemctl start docker.socket
+      arch=$(uname -m)
+      if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then 
+          sudo apt-get -y update
+          sudo apt-get -y install ca-certificates curl gnupg lsb-release
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get -y update
+          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+          sudo systemctl start docker.socket
 
-      export GOPATH=${SNAPCRAFT_STAGE}/gopath
-      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
-      cd ${kata_dir}
-      sudo -E NO_TTY=true make cloud-hypervisor-tarball
-      tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/
-      install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+          export GOPATH=${SNAPCRAFT_STAGE}/gopath
+          kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+          cd ${kata_dir}
+          sudo -E NO_TTY=true make cloud-hypervisor-tarball
+          tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/
+          install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
+      fi
 
 apps:
   runtime:

src/agent/Cargo.lock

@@ -367,6 +367,30 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.5"
@@ -993,6 +1017,7 @@ dependencies = [
  "slog",
  "slog-scope",
  "slog-stdlog",
+ "sysinfo",
  "tempfile",
  "thiserror",
  "tokio",
@@ -1879,6 +1904,31 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.10"
@@ -2281,6 +2331,21 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e757000a4bed2b1be9be65a3f418b9696adf30bb419214c73997422de73a591"
+dependencies = [
+ "cfg-if 1.0.0",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"

src/agent/Cargo.toml

@@ -20,6 +20,7 @@ scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1.5.4"
 serial_test = "0.5.1"
+sysinfo = "0.23.0"
 
 # Async helpers
 async-trait = "0.1.42"

src/agent/rustjail/src/container.rs

@@ -215,7 +215,6 @@ pub trait BaseContainer {
     async fn start(&mut self, p: Process) -> Result<()>;
     async fn run(&mut self, p: Process) -> Result<()>;
     async fn destroy(&mut self) -> Result<()>;
-    fn signal(&self, sig: Signal, all: bool) -> Result<()>;
     fn exec(&mut self) -> Result<()>;
 }
 
@@ -1057,18 +1056,6 @@ impl BaseContainer for LinuxContainer {
         Ok(())
     }
 
-    fn signal(&self, sig: Signal, all: bool) -> Result<()> {
-        if all {
-            for pid in self.processes.keys() {
-                signal::kill(Pid::from_raw(*pid), Some(sig))?;
-            }
-        }
-
-        signal::kill(Pid::from_raw(self.init_process_pid), Some(sig))?;
-
-        Ok(())
-    }
-
     fn exec(&mut self) -> Result<()> {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
@@ -2049,14 +2036,6 @@ mod tests {
         assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
     }
 
-    #[test]
-    fn test_linuxcontainer_signal() {
-        let ret = new_linux_container_and_then(|c: LinuxContainer| {
-            c.signal(nix::sys::signal::SIGCONT, true)
-        });
-        assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
-    }
-
     #[test]
     fn test_linuxcontainer_exec() {
         let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec());

src/agent/samples/configuration-all-endpoints.toml

@@ -25,6 +25,7 @@ allowed = [
         "ReadStreamRequest",
         "RemoveContainerRequest",
         "ReseedRandomDevRequest",
+        "ResizeVolumeRequest",
         "ResumeContainerRequest",
         "SetGuestDateTimeRequest",
         "SignalProcessRequest",
@@ -34,6 +35,7 @@ allowed = [
         "UpdateContainerRequest",
         "UpdateInterfaceRequest",
         "UpdateRoutesRequest",
+        "VolumeStatsRequest",
         "WaitProcessRequest",
         "WriteStreamRequest"
 ]

src/agent/src/mount.rs

@@ -193,13 +193,6 @@ async fn ephemeral_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut sb = sandbox.lock().await;
-    let new_storage = sb.set_sandbox_storage(&storage.mount_point);
-
-    if !new_storage {
-        return Ok("".to_string());
-    }
-
     // hugetlbfs
     if storage.fstype == FS_TYPE_HUGETLB {
         return handle_hugetlbfs_storage(logger, storage).await;
@@ -255,13 +248,6 @@ async fn local_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut sb = sandbox.lock().await;
-    let new_storage = sb.set_sandbox_storage(&storage.mount_point);
-
-    if !new_storage {
-        return Ok("".to_string());
-    }
-
     fs::create_dir_all(&storage.mount_point).context(format!(
         "failed to create dir all {:?}",
         &storage.mount_point
@@ -401,7 +387,7 @@ fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> {
 async fn virtiommio_blk_storage_handler(
     logger: &Logger,
     storage: &Storage,
-    _sandbox: Arc<Mutex<Sandbox>>,
+    sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
     //The source path is VmPath
     common_storage_handler(logger, storage)
@@ -641,6 +627,14 @@ pub async fn add_storages(
             "subsystem" => "storage",
             "storage-type" => handler_name.to_owned()));
 
+        {
+            let mut sb = sandbox.lock().await;
+            let new_storage = sb.set_sandbox_storage(&storage.mount_point);
+            if !new_storage {
+                continue;
+            }
+        }
+
         let res = match handler_name.as_str() {
             DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await,
             DRIVER_BLK_CCW_TYPE => {

src/agent/src/rpc.rs

@@ -23,9 +23,10 @@ use oci::{LinuxNamespace, Root, Spec};
 use protobuf::{Message, RepeatedField, SingularPtrField};
 use protocols::agent::{
     AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics,
-    OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse,
-    WriteStreamResponse,
+    OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, VolumeStatsRequest,
+    WaitProcessResponse, WriteStreamResponse,
 };
+use protocols::csi::{VolumeCondition, VolumeStatsResponse, VolumeUsage, VolumeUsage_Unit};
 use protocols::empty::Empty;
 use protocols::health::{
     HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse,
@@ -43,13 +44,15 @@ use nix::sys::stat;
 use nix::unistd::{self, Pid};
 use rustjail::process::ProcessOperations;
 
+use sysinfo::{DiskExt, System, SystemExt};
+
 use crate::device::{
     add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
 };
 use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
-use crate::mount::{add_storages, baremount, remove_mounts, STORAGE_HANDLER_LIST};
+use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST};
 use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
 use crate::network::setup_guest_dns;
 use crate::pci;
@@ -69,6 +72,7 @@ use tracing::instrument;
 use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ};
 use std::convert::TryFrom;
 use std::fs;
+use std::os::unix::fs::MetadataExt;
 use std::os::unix::prelude::PermissionsExt;
 use std::process::{Command, Stdio};
 use std::time::Duration;
@@ -302,8 +306,6 @@ impl AgentService {
             // Find the sandbox storage used by this container
             let mounts = sandbox.container_mounts.get(&cid);
             if let Some(mounts) = mounts {
-                remove_mounts(mounts)?;
-
                 for m in mounts.iter() {
                     if sandbox.storages.get(m).is_some() {
                         cmounts.push(m.to_string());
@@ -1321,6 +1323,47 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         Err(ttrpc_error!(ttrpc::Code::INTERNAL, ""))
     }
 
+    async fn get_volume_stats(
+        &self,
+        ctx: &TtrpcContext,
+        req: VolumeStatsRequest,
+    ) -> ttrpc::Result<VolumeStatsResponse> {
+        trace_rpc_call!(ctx, "get_volume_stats", req);
+        is_allowed!(req);
+
+        info!(sl!(), "get volume stats!");
+        let mut resp = VolumeStatsResponse::new();
+
+        let mut condition = VolumeCondition::new();
+
+        match File::open(&req.volume_guest_path) {
+            Ok(_) => {
+                condition.abnormal = false;
+                condition.message = String::from("OK");
+            }
+            Err(e) => {
+                info!(sl!(), "failed to open the volume");
+                return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e));
+            }
+        };
+
+        let mut usage_vec = Vec::new();
+
+        // to get volume capacity stats
+        get_volume_capacity_stats(&req.volume_guest_path)
+            .map(|u| usage_vec.push(u))
+            .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
+
+        // to get volume inode stats
+        get_volume_inode_stats(&req.volume_guest_path)
+            .map(|u| usage_vec.push(u))
+            .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
+
+        resp.usage = RepeatedField::from_vec(usage_vec);
+        resp.volume_condition = SingularPtrField::some(condition);
+        Ok(resp)
+    }
+
     async fn add_swap(
         &self,
         ctx: &TtrpcContext,
@@ -1408,6 +1451,48 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
     Ok((size, plug))
 }
 
+fn get_volume_capacity_stats(path: &str) -> Result<VolumeUsage> {
+    let mut usage = VolumeUsage::new();
+
+    let s = System::new();
+    for disk in s.disks() {
+        if let Some(v) = disk.name().to_str() {
+            if v.to_string().eq(path) {
+                usage.available = disk.available_space();
+                usage.total = disk.total_space();
+                usage.used = usage.total - usage.available;
+                usage.unit = VolumeUsage_Unit::BYTES; // bytes
+                break;
+            }
+        } else {
+            return Err(anyhow!(nix::Error::EINVAL));
+        }
+    }
+
+    Ok(usage)
+}
+
+fn get_volume_inode_stats(path: &str) -> Result<VolumeUsage> {
+    let mut usage = VolumeUsage::new();
+
+    let s = System::new();
+    for disk in s.disks() {
+        if let Some(v) = disk.name().to_str() {
+            if v.to_string().eq(path) {
+                let meta = fs::metadata(disk.mount_point())?;
+                let inode = meta.ino();
+                usage.used = inode;
+                usage.unit = VolumeUsage_Unit::INODES;
+                break;
+            }
+        } else {
+            return Err(anyhow!(nix::Error::EINVAL));
+        }
+    }
+
+    Ok(usage)
+}
+
 pub fn have_seccomp() -> bool {
     if cfg!(feature = "seccomp") {
         return true;

src/agent/src/sandbox.rs

@@ -553,7 +553,7 @@ mod tests {
 
         assert!(
             s.remove_sandbox_storage(srcdir_path).is_err(),
-            "Expect Err as the directory i not a mountpoint"
+            "Expect Err as the directory is not a mountpoint"
         );
 
         assert!(s.remove_sandbox_storage("").is_err());
@@ -588,7 +588,6 @@ mod tests {
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
 
-        // FIX: This test fails, not sure why yet.
         assert!(
             s.unset_and_remove_sandbox_storage("/tmp/testEphePath")
                 .is_err(),

src/libs/protocols/build.rs

@@ -95,6 +95,7 @@ fn real_main() -> Result<(), std::io::Error> {
 
     let protos = vec![
         "protos/agent.proto",
+        "protos/csi.proto",
         "protos/google/protobuf/empty.proto",
         "protos/health.proto",
         "protos/oci.proto",

src/libs/protocols/hack/update-generated-proto.sh

@@ -51,6 +51,8 @@ generate_go_sources() {
 --gogottrpc_out=plugins=ttrpc+fieldpath,\
 import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
 \
+Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
+\
 Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\
 \
 Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
@@ -69,7 +71,7 @@ if [ "$(basename $(pwd))" != "agent" ]; then
 fi
 
 # Protocol buffer files required to generate golang/rust bindings.
-proto_files_list=(agent.proto health.proto oci.proto types.proto)
+proto_files_list=(agent.proto csi.proto health.proto oci.proto types.proto)
 
 if [ "$1" = "" ]; then
     show_usage "${proto_files_list[@]}"

src/libs/protocols/protos/agent.proto

@@ -12,6 +12,7 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
 package grpc;
 
 import "oci.proto";
+import "csi.proto";
 import "types.proto";
 
 import "google/protobuf/empty.proto";
@@ -65,6 +66,8 @@ service AgentService {
 	rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty);
 	rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent);
 	rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty);
+	rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse);
+	rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty);
 }
 
 message CreateContainerRequest {
@@ -505,3 +508,14 @@ message GetMetricsRequest {}
 message Metrics {
 	string metrics = 1;
 }
+
+message VolumeStatsRequest {
+	// The volume path on the guest outside the container
+	string volume_guest_path = 1;
+}
+
+message ResizeVolumeRequest {
+	// Full VM guest path of the volume (outside the container)
+	string volume_guest_path = 1;
+	uint64 size = 2;
+}

src/libs/protocols/protos/csi.proto

@@ -0,0 +1,60 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
+package grpc;
+import "gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.equal_all) = true;
+option (gogoproto.populate_all) = true;
+option (gogoproto.testgen_all) = true;
+option (gogoproto.benchgen_all) = true;
+
+// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto)
+message VolumeStatsResponse {
+  // This field is OPTIONAL.
+  repeated VolumeUsage usage = 1;
+  // Information about the current condition of the volume.
+  // This field is OPTIONAL.
+  // This field MUST be specified if the VOLUME_CONDITION node
+  // capability is supported.
+  VolumeCondition volume_condition = 2;
+}
+message VolumeUsage {
+  enum Unit {
+    UNKNOWN = 0;
+    BYTES = 1;
+    INODES = 2;
+  }
+  // The available capacity in specified Unit. This field is OPTIONAL.
+  // The value of this field MUST NOT be negative.
+  uint64 available = 1;
+
+  // The total capacity in specified Unit. This field is REQUIRED.
+  // The value of this field MUST NOT be negative.
+  uint64 total = 2;
+
+  // The used capacity in specified Unit. This field is OPTIONAL.
+  // The value of this field MUST NOT be negative.
+  uint64 used = 3;
+
+  // Units by which values are measured. This field is REQUIRED.
+  Unit unit = 4;
+}
+
+// VolumeCondition represents the current condition of a volume.
+message VolumeCondition {
+
+  // Normal volumes are available for use and operating optimally.
+  // An abnormal volume does not meet these criteria.
+  // This field is REQUIRED.
+  bool abnormal = 1;
+
+  // The message describing the condition of the volume.
+  // This field is REQUIRED.
+  string message = 2;
+}

src/libs/protocols/protos/oci.proto

@@ -6,11 +6,9 @@
 //
 
 syntax = "proto3";
-
 option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
 
 package grpc;
-
 import "gogo/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.equal_all) = true;

src/libs/protocols/src/lib.rs

@@ -7,6 +7,7 @@
 
 pub mod agent;
 pub mod agent_ttrpc;
+pub mod csi;
 pub mod empty;
 pub mod health;
 pub mod health_ttrpc;

src/runtime/Makefile

@@ -1,5 +1,6 @@
 #
 # Copyright (c) 2018-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -28,6 +29,9 @@ ARCH_FILE = $(ARCH_DIR)/$(ARCH)$(ARCH_FILE_SUFFIX)
 ARCH_FILES = $(wildcard arch/*$(ARCH_FILE_SUFFIX))
 ALL_ARCHES = $(patsubst $(ARCH_DIR)/%$(ARCH_FILE_SUFFIX),%,$(ARCH_FILES))
 
+# Build as safely as possible
+export CGO_CPPFLAGS = -D_FORTIFY_SOURCE=2 -fstack-protector
+
 ifeq (,$(realpath $(ARCH_FILE)))
     $(error "ERROR: invalid architecture: '$(ARCH)'")
 else
@@ -158,6 +162,7 @@ DEFMEMSLOTS := 10
 DEFBRIDGES := 1
 DEFENABLEANNOTATIONS := []
 DEFDISABLEGUESTSECCOMP := true
+DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled
 DEFAULTEXPFEATURES := []
 
@@ -434,6 +439,7 @@ USER_VARS += DEFNETWORKMODEL_ACRN
 USER_VARS += DEFNETWORKMODEL_CLH
 USER_VARS += DEFNETWORKMODEL_FC
 USER_VARS += DEFNETWORKMODEL_QEMU
+USER_VARS += DEFDISABLEGUESTEMPTYDIR
 USER_VARS += DEFDISABLEGUESTSECCOMP
 USER_VARS += DEFDISABLESELINUX
 USER_VARS += DEFAULTEXPFEATURES

src/runtime/cmd/kata-runtime/kata-exec.go

@@ -19,8 +19,8 @@ import (
 	"time"
 
 	"github.com/containerd/console"
-	kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
 	clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
@@ -154,7 +154,7 @@ func (s *iostream) Read(data []byte) (n int, err error) {
 }
 
 func getConn(sandboxID string, port uint64) (net.Conn, error) {
-	client, err := kataMonitor.BuildShimClient(sandboxID, defaultTimeout)
+	client, err := shimclient.BuildShimClient(sandboxID, defaultTimeout)
 	if err != nil {
 		return nil, err
 	}

src/runtime/cmd/kata-runtime/kata-volume.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"net/url"
+
+	containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
+
+	"github.com/urfave/cli"
+)
+
+var volumeSubCmds = []cli.Command{
+	addCommand,
+	removeCommand,
+	statsCommand,
+	resizeCommand,
+}
+
+var (
+	mountInfo  string
+	volumePath string
+	size       uint64
+)
+
+var kataVolumeCommand = cli.Command{
+	Name:        "direct-volume",
+	Usage:       "directly assign a volume to Kata Containers to manage",
+	Subcommands: volumeSubCmds,
+	Action: func(context *cli.Context) {
+		cli.ShowSubcommandHelp(context)
+	},
+}
+
+var addCommand = cli.Command{
+	Name:  "add",
+	Usage: "add a direct assigned block volume device to the Kata Containers runtime",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+		cli.StringFlag{
+			Name:        "mount-info",
+			Usage:       "the mount info for the Kata Containers runtime to manage the volume",
+			Destination: &mountInfo,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		return volume.Add(volumePath, mountInfo)
+	},
+}
+
+var removeCommand = cli.Command{
+	Name:  "remove",
+	Usage: "remove a direct assigned block volume device from the Kata Containers runtime",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		return volume.Remove(volumePath)
+	},
+}
+
+var statsCommand = cli.Command{
+	Name:  "stats",
+	Usage: "get the filesystem stat of a direct assigned volume",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+	},
+	Action: func(c *cli.Context) (string, error) {
+		stats, err := Stats(volumePath)
+		if err != nil {
+			return "", err
+		}
+
+		return string(stats), nil
+	},
+}
+
+var resizeCommand = cli.Command{
+	Name:  "resize",
+	Usage: "resize a direct assigned block volume",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "volume-path",
+			Usage:       "the target volume path the volume is published to",
+			Destination: &volumePath,
+		},
+		cli.Uint64Flag{
+			Name:        "size",
+			Usage:       "the new size of the volume",
+			Destination: &size,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		return Resize(volumePath, size)
+	},
+}
+
+// Stats retrieves the filesystem stats of the direct volume inside the guest.
+func Stats(volumePath string) ([]byte, error) {
+	sandboxId, err := volume.GetSandboxIdForVolume(volumePath)
+	if err != nil {
+		return nil, err
+	}
+	urlSafeDevicePath := url.PathEscape(volumePath)
+	body, err := shimclient.DoGet(sandboxId, defaultTimeout, containerdshim.DirectVolumeStatUrl+"/"+urlSafeDevicePath)
+	if err != nil {
+		return nil, err
+	}
+	return body, nil
+}
+
+// Resize resizes a direct volume inside the guest.
+func Resize(volumePath string, size uint64) error {
+	sandboxId, err := volume.GetSandboxIdForVolume(volumePath)
+	if err != nil {
+		return err
+	}
+	resizeReq := containerdshim.ResizeRequest{
+		VolumePath: volumePath,
+		Size:       size,
+	}
+	encoded, err := json.Marshal(resizeReq)
+	if err != nil {
+		return err
+	}
+	return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, encoded)
+}

src/runtime/cmd/kata-runtime/main.go

@@ -124,6 +124,7 @@ var runtimeCommands = []cli.Command{
 	kataExecCLICommand,
 	kataMetricsCLICommand,
 	factoryCLICommand,
+	kataVolumeCommand,
 }
 
 // runtimeBeforeSubcommands is the function to run before command-line

src/runtime/config/configuration-acrn.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -219,6 +220,10 @@ disable_selinux=@DEFDISABLESELINUX@
 # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
 sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-clh.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -25,9 +26,12 @@ image = "@IMAGEPATH@"
 # Known limitations:
 # * Does not work by design:
 #   - CPU Hotplug 
-#   - Device Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
+#   - SharedFS, such as virtio-fs and virtio-fs-nydus
+#
+# Requirements:
+# * virtio-block used as rootfs, thus the usage of devmapper snapshotter.
 #
 # Supported TEEs:
 # * Intel TDX
@@ -306,6 +310,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@
 #
 vfio_mode="@DEFVFIOMODE@"
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-fc.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -103,14 +104,6 @@ default_memory = @DEFMEMSZ@
 # Default 0
 #memory_offset = 0
 
-# Disable block device from being used for a container's rootfs.
-# In case of a storage driver like devicemapper where a container's
-# root file system is backed by a block device, the block device is passed
-# directly to the hypervisor for performance reasons.
-# This flag prevents the block device from being passed to the hypervisor,
-# 9pfs is used instead to pass the rootfs.
-disable_block_device_use = @DEFDISABLEBLOCK@
-
 # Block storage driver to be used for the hypervisor in case the container
 # rootfs is backed by a block device. This is virtio-scsi, virtio-blk
 # or nvdimm.
@@ -352,6 +345,10 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running single containers using a tool like ctr, container sizing information will be available.
 static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_FC@
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/config/configuration-qemu.toml.in

@@ -1,4 +1,5 @@
 # Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -26,7 +27,6 @@ machine_type = "@MACHINETYPE@"
 # Known limitations:
 # * Does not work by design:
 #   - CPU Hotplug 
-#   - Device Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -144,7 +144,7 @@ default_memory = @DEFMEMSZ@
 # root file system is backed by a block device, the block device is passed
 # directly to the hypervisor for performance reasons.
 # This flag prevents the block device from being passed to the hypervisor,
-# 9pfs is used instead to pass the rootfs.
+# virtio-fs is used instead to pass the rootfs.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
 # Shared file system type:
@@ -574,6 +574,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@
 #
 vfio_mode="@DEFVFIOMODE@"
 
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/pkg/containerd-shim-v2/create.go

@@ -1,6 +1,7 @@
 // Copyright (c) 2014,2015,2016 Docker, Inc.
 // Copyright (c) 2017 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
+// Copyright (c) 2021 Adobe Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -57,10 +58,10 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 
 	detach := !r.Terminal
 	ociSpec, bundlePath, err := loadSpec(r)
+
 	if err != nil {
 		return nil, err
 	}
-
 	containerType, err := oci.ContainerType(*ociSpec)
 	if err != nil {
 		return nil, err
@@ -69,16 +70,18 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
 	rootfs := filepath.Join(r.Bundle, "rootfs")
 
+	runtimeConfig, err := loadRuntimeConfig(s, r, ociSpec.Annotations)
+	if err != nil {
+		return nil, err
+	}
+
 	switch containerType {
 	case vc.PodSandbox, vc.SingleContainer:
 		if s.sandbox != nil {
 			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
 		}
 
-		s.config, err = loadRuntimeConfig(s, r, ociSpec.Annotations)
-		if err != nil {
-			return nil, err
-		}
+		s.config = runtimeConfig
 
 		// create tracer
 		// This is the earliest location we can create the tracer because we must wait
@@ -176,7 +179,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 			}
 		}()
 
-		_, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput)
+		_, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, runtimeConfig.DisableGuestEmptyDir)
 		if err != nil {
 			return nil, err
 		}

src/runtime/pkg/containerd-shim-v2/shim_management.go

@@ -7,26 +7,33 @@ package containerdshim
 
 import (
 	"context"
+	"encoding/json"
 	"expvar"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/pprof"
+	"net/url"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"google.golang.org/grpc/codes"
+
 	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+	mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
+)
 
-	"google.golang.org/grpc/codes"
-
-	mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
+const (
+	DirectVolumeStatUrl   = "/direct-volume/stats"
+	DirectVolumeResizeUrl = "/direct-volume/resize"
 )
 
 var (
@@ -34,6 +41,11 @@ var (
 	shimMgtLog               = shimLog.WithField("subsystem", "shim-management")
 )
 
+type ResizeRequest struct {
+	VolumePath string
+	Size       uint64
+}
+
 // agentURL returns URL for agent
 func (s *service) agentURL(w http.ResponseWriter, r *http.Request) {
 	url, err := s.sandbox.GetAgentURL()
@@ -126,6 +138,52 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily {
 	return list
 }
 
+func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) {
+	volumePath, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, DirectVolumeStatUrl))
+	if err != nil {
+		shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	buf, err := s.sandbox.GuestVolumeStats(context.Background(), volumePath)
+	if err != nil {
+		shimMgtLog.WithError(err).WithField("volume-path", volumePath).Error("failed to get volume stats")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Write(buf)
+}
+
+func (s *service) serveVolumeResize(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		shimMgtLog.WithError(err).Error("failed to read request body")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	var resizeReq ResizeRequest
+	err = json.Unmarshal(body, &resizeReq)
+	if err != nil {
+		shimMgtLog.WithError(err).Error("failed to unmarshal the http request body")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	err = s.sandbox.ResizeGuestVolume(context.Background(), resizeReq.VolumePath, resizeReq.Size)
+	if err != nil {
+		shimMgtLog.WithError(err).WithField("volume-path", resizeReq.VolumePath).Error("failed to resize the volume")
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Write([]byte(""))
+}
+
 func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) {
 	// metrics socket will under sandbox's bundle path
 	metricsAddress := SocketAddress(s.id)
@@ -148,6 +206,8 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec
 	m := http.NewServeMux()
 	m.Handle("/metrics", http.HandlerFunc(s.serveMetrics))
 	m.Handle("/agent-url", http.HandlerFunc(s.agentURL))
+	m.Handle(DirectVolumeStatUrl, http.HandlerFunc(s.serveVolumeStats))
+	m.Handle(DirectVolumeResizeUrl, http.HandlerFunc(s.serveVolumeResize))
 	s.mountPprofHandle(m, ociSpec)
 
 	// register shim metrics

src/runtime/pkg/direct-volume/utils.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package volume
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	mountInfoFileName = "mountInfo.json"
+)
+
+var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes"
+
+// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM.
+type MountInfo struct {
+	// The type of the volume (ie. block)
+	VolumeType string `json:"volume-type"`
+	// The device backing the volume.
+	Device string `json:"device"`
+	// The filesystem type to be mounted on the volume.
+	FsType string `json:"fstype"`
+	// Additional metadata to pass to the agent regarding this volume.
+	Metadata map[string]string `json:"metadata,omitempty"`
+	// Additional mount options.
+	Options []string `json:"options,omitempty"`
+}
+
+// Add writes the mount info of a direct volume into a filesystem path known to Kata Container.
+func Add(volumePath string, mountInfo string) error {
+	volumeDir := filepath.Join(kataDirectVolumeRootPath, volumePath)
+	stat, err := os.Stat(volumeDir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if stat != nil && !stat.IsDir() {
+		return fmt.Errorf("%s should be a directory", volumeDir)
+	}
+	if errors.Is(err, os.ErrNotExist) {
+		if err := os.MkdirAll(volumeDir, 0700); err != nil {
+			return err
+		}
+	}
+	var deserialized MountInfo
+	if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600)
+}
+
+// Remove deletes the direct volume path including all the files inside it.
+func Remove(volumePath string) error {
+	// Find the base of the volume path to delete the whole volume path
+	base := strings.SplitN(volumePath, string(os.PathSeparator), 2)[0]
+	return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, base))
+}
+
+// VolumeMountInfo retrieves the mount info of a direct volume.
+func VolumeMountInfo(volumePath string) (*MountInfo, error) {
+	mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName)
+	if _, err := os.Stat(mountInfoFilePath); err != nil {
+		return nil, err
+	}
+	buf, err := ioutil.ReadFile(mountInfoFilePath)
+	if err != nil {
+		return nil, err
+	}
+	var mountInfo MountInfo
+	if err := json.Unmarshal(buf, &mountInfo); err != nil {
+		return nil, err
+	}
+	return &mountInfo, nil
+}
+
+// RecordSandboxId associates a sandbox id with a direct volume.
+func RecordSandboxId(sandboxId string, volumePath string) error {
+	mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName)
+	if _, err := os.Stat(mountInfoFilePath); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, volumePath, sandboxId), []byte(""), 0600)
+}
+
+func GetSandboxIdForVolume(volumePath string) (string, error) {
+	files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, volumePath))
+	if err != nil {
+		return "", err
+	}
+	// Find the id of the first sandbox.
+	// We expect a direct-assigned volume is associated with only a sandbox at a time.
+	for _, file := range files {
+		if file.Name() != mountInfoFileName {
+			return file.Name(), nil
+		}
+	}
+	return "", fmt.Errorf("no sandbox found for %s", volumePath)
+}

src/runtime/pkg/direct-volume/utils_test.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package volume
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAdd(t *testing.T) {
+	var err error
+	kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "add-test")
+	assert.Nil(t, err)
+	defer os.RemoveAll(kataDirectVolumeRootPath)
+	var volumePath = "/a/b/c"
+	var basePath = "a"
+	actual := MountInfo{
+		VolumeType: "block",
+		Device:     "/dev/sda",
+		FsType:     "ext4",
+		Options:    []string{"journal_dev", "noload"},
+	}
+	buf, err := json.Marshal(actual)
+	assert.Nil(t, err)
+
+	// Add the mount info
+	assert.Nil(t, Add(volumePath, string(buf)))
+
+	// Validate the mount info
+	expected, err := VolumeMountInfo(volumePath)
+	assert.Nil(t, err)
+	assert.Equal(t, expected.Device, actual.Device)
+	assert.Equal(t, expected.FsType, actual.FsType)
+	assert.Equal(t, expected.Options, actual.Options)
+
+	// Remove the file
+	err = Remove(volumePath)
+	assert.Nil(t, err)
+	_, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, basePath))
+	assert.True(t, errors.Is(err, os.ErrNotExist))
+
+	// Test invalid mount info json
+	assert.Error(t, Add(volumePath, "{invalid json}"))
+}
+
+func TestRecordSandboxId(t *testing.T) {
+	var err error
+	kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test")
+	assert.Nil(t, err)
+	defer os.RemoveAll(kataDirectVolumeRootPath)
+
+	var volumePath = "/a/b/c"
+	mntInfo := MountInfo{
+		VolumeType: "block",
+		Device:     "/dev/sda",
+		FsType:     "ext4",
+		Options:    []string{"journal_dev", "noload"},
+	}
+	buf, err := json.Marshal(mntInfo)
+	assert.Nil(t, err)
+
+	// Add the mount info
+	assert.Nil(t, Add(volumePath, string(buf)))
+
+	sandboxId := uuid.Generate().String()
+	err = RecordSandboxId(sandboxId, volumePath)
+	assert.Nil(t, err)
+
+	id, err := GetSandboxIdForVolume(volumePath)
+	assert.Nil(t, err)
+	assert.Equal(t, sandboxId, id)
+}
+
+func TestRecordSandboxIdNoMountInfoFile(t *testing.T) {
+	var err error
+	kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test")
+	assert.Nil(t, err)
+	defer os.RemoveAll(kataDirectVolumeRootPath)
+
+	var volumePath = "/a/b/c"
+	sandboxId := uuid.Generate().String()
+	err = RecordSandboxId(sandboxId, volumePath)
+	assert.Error(t, err)
+	assert.True(t, errors.Is(err, os.ErrNotExist))
+}

src/runtime/pkg/kata-monitor/metrics.go

@@ -16,7 +16,7 @@ import (
 	"time"
 
 	mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
-
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/expfmt"
 
@@ -224,7 +224,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
 }
 
 func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*dto.MetricFamily, error) {
-	body, err := doGet(sandboxID, defaultTimeout, "metrics")
+	body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics")
 	if err != nil {
 		return nil, err
 	}
@@ -234,7 +234,7 @@ func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*
 
 // GetSandboxMetrics will get sandbox's metrics from shim
 func GetSandboxMetrics(sandboxID string) (string, error) {
-	body, err := doGet(sandboxID, defaultTimeout, "metrics")
+	body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics")
 	if err != nil {
 		return "", err
 	}

src/runtime/pkg/kata-monitor/monitor.go

@@ -14,6 +14,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
+
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 )
@@ -180,7 +182,7 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	data, err := doGet(sandboxID, defaultTimeout, "agent-url")
+	data, err := shimclient.DoGet(sandboxID, defaultTimeout, "agent-url")
 	if err != nil {
 		commonServeError(w, http.StatusBadRequest, err)
 		return

src/runtime/pkg/kata-monitor/shim_client.go

@@ -7,13 +7,9 @@ package katamonitor
 
 import (
 	"fmt"
-	"io"
-	"net"
 	"net/http"
 	"time"
 
-	cdshim "github.com/containerd/containerd/runtime/v2/shim"
-
 	shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
 )
 
@@ -40,51 +36,3 @@ func getSandboxIDFromReq(r *http.Request) (string, error) {
 func getSandboxFS() string {
 	return shim.GetSandboxesStoragePath()
 }
-
-// BuildShimClient builds and returns an http client for communicating with the provided sandbox
-func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) {
-	return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout)
-}
-
-// buildUnixSocketClient build http client for Unix socket
-func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) {
-	transport := &http.Transport{
-		DisableKeepAlives: true,
-		Dial: func(proto, addr string) (conn net.Conn, err error) {
-			return cdshim.AnonDialer(socketAddr, timeout)
-		},
-	}
-
-	client := &http.Client{
-		Transport: transport,
-	}
-
-	if timeout > 0 {
-		client.Timeout = timeout
-	}
-
-	return client, nil
-}
-
-func doGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) {
-	client, err := BuildShimClient(sandboxID, timeoutInSeconds)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath))
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		resp.Body.Close()
-	}()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	return body, nil
-}

src/runtime/pkg/katatestutils/README.md

@@ -102,36 +102,6 @@ func TestOnlyRunWhenNotRoot(t *testing.T) {
 }
 ```
 
-#### Skip tests based on distro
-
-Use the `NeedDistro()` constraint to skip a test unless running on a
-particular Linux distribution:
-
-```go
-func TestOnlyRunOnUbuntu(t *testing.T) {
-
-    if tc.NotValid(ktu.NeedDistro("ubuntu")) {
-        t.Skip("skipping test as not running on ubuntu")
-    }
-
-    // Test code to run on Ubuntu only ...
-}
-```
-
-Use the `NeedDistroNotEquals()` constraint to skip a test unless running
-on a Linux distribution other than the one specified:
-
-```go
-func TestDontRunOnFedora(t *testing.T) {
-
-    if tc.NotValid(ktu.NeedDistroNotEquals("fedora")) {
-        t.Skip("skipping test as running on fedora")
-    }
-
-    // Test code to run on any distro apart from Fedora ...
-}
-```
-
 #### Skip tests based on kernel version
 
 Use the `NeedKernelVersionGE()` constraint to skip a test unless running on a

src/runtime/pkg/katatestutils/constraints.go

@@ -18,17 +18,9 @@ import (
 const (
 	TestDisabledNeedRoot    = "Test disabled as requires root user"
 	TestDisabledNeedNonRoot = "Test disabled as requires non-root user"
-
-	// See https://www.freedesktop.org/software/systemd/man/os-release.html
-	osRelease            = "/etc/os-release"
-	osReleaseAlternative = "/usr/lib/os-release"
 )
 
-var (
-	errUnknownDistroName      = errors.New("unknown distro name")
-	errUnknownDistroVersion   = errors.New("unknown distro version")
-	errInvalidOpForConstraint = errors.New("invalid operator for constraint type")
-)
+var errInvalidOpForConstraint = errors.New("invalid operator for constraint type")
 
 // String converts the operator to a human-readable value.
 func (o Operator) String() (s string) {
@@ -87,54 +79,6 @@ func getKernelVersion() (string, error) {
 	return fixKernelVersion(fields[2]), nil
 }
 
-// getDistroDetails returns the distributions name and version string.
-// If it is not possible to determine both values an error is
-// returned.
-func getDistroDetails() (name, version string, err error) {
-	files := []string{osRelease, osReleaseAlternative}
-	name = ""
-	version = ""
-
-	for _, file := range files {
-		contents, err := getFileContents(file)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-
-			return "", "", err
-		}
-
-		lines := strings.Split(contents, "\n")
-
-		for _, line := range lines {
-			if strings.HasPrefix(line, "ID=") && name == "" {
-				fields := strings.Split(line, "=")
-				name = strings.Trim(fields[1], `"`)
-				name = strings.ToLower(name)
-			} else if strings.HasPrefix(line, "VERSION_ID=") && version == "" {
-				fields := strings.Split(line, "=")
-				version = strings.Trim(fields[1], `"`)
-				version = strings.ToLower(version)
-			}
-		}
-
-		if name != "" && version != "" {
-			return name, version, nil
-		}
-	}
-
-	if name == "" {
-		return "", "", errUnknownDistroName
-	}
-
-	if version == "" {
-		return "", "", errUnknownDistroVersion
-	}
-
-	return name, version, nil
-}
-
 // fixKernelVersion replaces underscores with dashes in a version string.
 // This change is primarily for Fedora, RHEL and CentOS version numbers which
 // can contain underscores. By replacing them with dashes, a valid semantic
@@ -157,42 +101,6 @@ func fixKernelVersion(version string) string {
 	return strings.Replace(version, "+", "", -1)
 }
 
-// handleDistroName checks that the current distro is compatible with
-// the constraint specified by the arguments.
-func (tc *TestConstraint) handleDistroName(name string, op Operator) (result Result, err error) {
-	if name == "" {
-		return Result{}, fmt.Errorf("distro name cannot be blank")
-	}
-
-	name = strings.ToLower(name)
-
-	var success bool
-
-	switch op {
-	case eqOperator:
-		success = name == tc.DistroName
-	case neOperator:
-		success = name != tc.DistroName
-	default:
-		return Result{}, errInvalidOpForConstraint
-	}
-
-	descr := fmt.Sprintf("need distro %s %q, got distro %q", op, name, tc.DistroName)
-
-	result = Result{
-		Description: descr,
-		Success:     success,
-	}
-
-	return result, nil
-}
-
-// handleDistroVersion checks that the current distro version is compatible with
-// the constraint specified by the arguments.
-func (tc *TestConstraint) handleDistroVersion(version string, op Operator) (result Result, err error) {
-	return handleVersionType("distro", tc.DistroVersion, op, version)
-}
-
 // handleKernelVersion checks that the current kernel version is compatible with
 // the constraint specified by the arguments.
 func (tc *TestConstraint) handleKernelVersion(version string, op Operator) (result Result, err error) {
@@ -459,23 +367,6 @@ func (tc *TestConstraint) constraintValid(fn Constraint) bool {
 		}
 	}
 
-	if c.DistroName != "" {
-		result, err := tc.handleDistroName(c.DistroName, c.Operator)
-		tc.handleResults(result, err)
-		if !result.Success {
-			return false
-		}
-	}
-
-	if c.DistroVersion != "" {
-		result, err := tc.handleDistroVersion(c.DistroVersion, c.Operator)
-		tc.handleResults(result, err)
-		if !result.Success {
-			return false
-		}
-
-	}
-
 	if c.KernelVersion != "" {
 		result, err := tc.handleKernelVersion(c.KernelVersion, c.Operator)
 		tc.handleResults(result, err)

src/runtime/pkg/katatestutils/constraints_api.go

@@ -9,7 +9,6 @@ package katatestutils
 
 import (
 	"os"
-	"strings"
 )
 
 // Operator represents an operator to apply to a test constraint value.
@@ -28,13 +27,6 @@ const (
 type Constraints struct {
 	Issue string
 
-	// DistroName is the name of a distro in all lower-case letters.
-	DistroName string
-
-	// DistroVersion is the version of the particular distro in string
-	// format. It may contain periods and dashes.
-	DistroVersion string
-
 	// KernelVersion is the version of a particular kernel.
 	KernelVersion string
 
@@ -54,8 +46,6 @@ type Constraint func(c *Constraints)
 
 // TestConstraint records details about test constraints.
 type TestConstraint struct {
-	DistroName    string
-	DistroVersion string
 	KernelVersion string
 
 	// Optionally used to record an issue number that relates to the
@@ -76,11 +66,6 @@ type TestConstraint struct {
 // NewKataTest creates a new TestConstraint object and is the main interface
 // to the test constraints feature.
 func NewTestConstraint(debug bool) TestConstraint {
-	distroName, distroVersion, err := getDistroDetails()
-	if err != nil {
-		panic(err)
-	}
-
 	kernelVersion, err := getKernelVersion()
 	if err != nil {
 		panic(err)
@@ -90,8 +75,6 @@ func NewTestConstraint(debug bool) TestConstraint {
 		Debug: debug,
 
 		ActualEUID:    os.Geteuid(),
-		DistroName:    distroName,
-		DistroVersion: distroVersion,
 		KernelVersion: kernelVersion,
 	}
 }
@@ -102,7 +85,7 @@ func NewTestConstraint(debug bool) TestConstraint {
 // Notes:
 //
 // - Constraints are applied in the order specified.
-// - A constraint type (user, distro) can only be specified once.
+// - A constraint type (user, kernel) can only be specified once.
 // - If the function fails to determine whether it can check the constraints,
 //   it will panic. Since this is facility is used for testing, this seems like
 //   the best approach as it unburdens the caller from checking for an error
@@ -146,99 +129,7 @@ func NeedNonRoot() Constraint {
 	return NeedUID(0, neOperator)
 }
 
-// NeedDistroWithOp skips the test unless the distro constraint specified by
-// the arguments is true.
-func NeedDistroWithOp(distro string, op Operator) Constraint {
-	return func(c *Constraints) {
-		c.DistroName = strings.ToLower(distro)
-		c.Operator = op
-	}
-}
-
-// NeedDistroEquals will skip the test unless running on the specified distro.
-func NeedDistroEquals(distro string) Constraint {
-	return NeedDistroWithOp(distro, eqOperator)
-}
-
-// NeedDistroNotEquals will skip the test unless run a distro that does not
-// match the specified name.
-func NeedDistroNotEquals(distro string) Constraint {
-	return NeedDistroWithOp(distro, neOperator)
-}
-
-// NeedDistro will skip the test unless running on the specified distro.
-func NeedDistro(distro string) Constraint {
-	return NeedDistroEquals(distro)
-}
-
-// NeedDistroVersionWithOp skips the test unless the distro version constraint
-// specified by the arguments is true.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionWithOp(version string, op Operator) Constraint {
-	return func(c *Constraints) {
-		c.DistroVersion = version
-		c.Operator = op
-	}
-}
-
-// NeedDistroVersionEquals will skip the test unless the distro version is the
-// same as the specified version.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionEquals(version string) Constraint {
-	return NeedDistroVersionWithOp(version, eqOperator)
-}
-
-// NeedDistroVersionNotEquals will skip the test unless the distro version is
-// different to the specified version.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionNotEquals(version string) Constraint {
-	return NeedDistroVersionWithOp(version, neOperator)
-}
-
-// NeedDistroVersionLE will skip the test unless the distro version is older
-// than or the same as the specified version.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionLE(version string) Constraint {
-	return NeedDistroVersionWithOp(version, leOperator)
-}
-
-// NeedDistroVersionLT will skip the test unless the distro version is older
-// than the specified version.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionLT(version string) Constraint {
-	return NeedDistroVersionWithOp(version, ltOperator)
-}
-
-// NeedDistroVersionGE will skip the test unless the distro version is newer
-// than or the same as the specified version.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionGE(version string) Constraint {
-	return NeedDistroVersionWithOp(version, geOperator)
-}
-
-// NeedDistroVersionGT will skip the test unless the distro version is newer
-// than the specified version.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersionGT(version string) Constraint {
-	return NeedDistroVersionWithOp(version, gtOperator)
-}
-
-// NeedDistroVersion will skip the test unless running on the specified
-// (exact) version of some distro.
-//
-// Note: distro versions vary in format.
-func NeedDistroVersion(version string) Constraint {
-	return NeedDistroVersionEquals(version)
-}
-
-// NeedKernelVersionWithOp skips the test unless the distro version constraint
+// NeedKernelVersionWithOp skips the test unless the kernel version constraint
 // specified by the arguments is true.
 func NeedKernelVersionWithOp(version string, op Operator) Constraint {
 	return func(c *Constraints) {
@@ -247,54 +138,44 @@ func NeedKernelVersionWithOp(version string, op Operator) Constraint {
 	}
 }
 
-// NeedKernelVersionLT will skip the test unless the distro version is older
-// than the specified version.
+// NeedKernelVersionEquals will skip the test unless the kernel version is same as
+// the specified version.
 func NeedKernelVersionEquals(version string) Constraint {
 	return NeedKernelVersionWithOp(version, eqOperator)
 }
 
-// NeedKernelVersionNotEquals will skip the test unless the distro version is
+// NeedKernelVersionNotEquals will skip the test unless the kernel version is
 // different to the specified version.
 func NeedKernelVersionNotEquals(version string) Constraint {
 	return NeedKernelVersionWithOp(version, neOperator)
 }
 
-// NeedKernelVersionLT will skip the test unless the distro version is older
+// NeedKernelVersionLT will skip the test unless the kernel version is older
 // than the specified version.
-//
-// Note: distro versions vary in format.
 func NeedKernelVersionLT(version string) Constraint {
 	return NeedKernelVersionWithOp(version, ltOperator)
 }
 
-// NeedKernelVersionLE will skip the test unless the distro version is older
+// NeedKernelVersionLE will skip the test unless the kernel version is older
 // than or the same as the specified version.
-//
-// Note: distro versions vary in format.
 func NeedKernelVersionLE(version string) Constraint {
 	return NeedKernelVersionWithOp(version, leOperator)
 }
 
-// NeedKernelVersionGT will skip the test unless the distro version is newer
+// NeedKernelVersionGT will skip the test unless the kernel version is newer
 // than the specified version.
-//
-// Note: distro versions vary in format.
 func NeedKernelVersionGT(version string) Constraint {
 	return NeedKernelVersionWithOp(version, gtOperator)
 }
 
-// NeedKernelVersionGE will skip the test unless the distro version is newer
+// NeedKernelVersionGE will skip the test unless the kernel version is newer
 // than or the same as the specified version.
-//
-// Note: distro versions vary in format.
 func NeedKernelVersionGE(version string) Constraint {
 	return NeedKernelVersionWithOp(version, geOperator)
 }
 
-// NeedKernelVersion will skip the test unless running on the specified
-// (exact) version of some distro.
-//
-// Note: distro versions vary in format.
+// NeedKernelVersion will skip the test unless the kernel version is same as
+// the specified version.
 func NeedKernelVersion(version string) Constraint {
 	return NeedKernelVersionEquals(version)
 }

src/runtime/pkg/katatestutils/constraints_test.go

@@ -9,7 +9,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -21,8 +20,6 @@ import (
 
 const (
 	invalidOperator = 1234
-
-	skipUnknownDistroName = "skipping test as cannot determine distro name"
 )
 
 // nolint: govet
@@ -32,36 +29,16 @@ type testDataUID struct {
 	c   Constraints
 }
 
-// nolint: govet
-type testDataDistro struct {
-	distro string
-	op     Operator
-	c      Constraints
-}
+var (
+	thisUID = os.Getuid()
+	rootUID = 0
+)
 
-var distros = []string{
-	"centos",
-	"clear-linux-os",
-	"debian",
-	"fedora",
-	"opensuse",
-	"rhel",
-	"sles",
-	"ubuntu",
-}
-
-var thisUID = os.Getuid()
-var rootUID = 0
-
-// name and version of current distro and kernel version of system tests are
+// name and version of current kernel version of system tests are
 // running on
-var distroName string
-var distroVersion string
 var kernelVersion string
 
-// error saved when attempting to determine distro name+version and kernel
-// version.
-var getDistroErr error
+// error saved when attempting to determine kernel version.
 var getKernelErr error
 
 // true if running as root
@@ -87,49 +64,8 @@ var uidNotEqualsRootData = testDataUID{
 	},
 }
 
-var distroEqualsCurrentData testDataDistro
-var distroNotEqualsCurrentData testDataDistro
-
 func init() {
-	distroName, distroVersion, getDistroErr = testGetDistro()
 	kernelVersion, getKernelErr = testGetKernelVersion()
-
-	distroEqualsCurrentData = testDataDistro{
-		distro: distroName,
-		op:     eqOperator,
-		c: Constraints{
-			DistroName: distroName,
-			Operator:   eqOperator,
-		},
-	}
-
-	distroNotEqualsCurrentData = testDataDistro{
-		distro: distroName,
-		op:     neOperator,
-		c: Constraints{
-			DistroName: distroName,
-			Operator:   neOperator,
-		},
-	}
-}
-
-func fileExists(path string) bool {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return false
-	}
-
-	return true
-}
-
-// getAnotherDistro returns a distro name not equal to the one specified.
-func getAnotherDistro(distro string) string {
-	for _, d := range distros {
-		if d != distro {
-			return d
-		}
-	}
-
-	panic(fmt.Sprintf("failed to find a distro different to %s", distro))
 }
 
 func checkUIDConstraints(assert *assert.Assertions, a, b Constraints, desc string) {
@@ -140,13 +76,6 @@ func checkUIDConstraints(assert *assert.Assertions, a, b Constraints, desc strin
 	assert.Equal(a.UIDSet, b.UIDSet, msg)
 }
 
-func checkDistroConstraints(assert *assert.Assertions, a, b Constraints, desc string) {
-	msg := fmt.Sprintf("%s: a: %+v, b: %+v", desc, a, b)
-
-	assert.Equal(a.DistroName, b.DistroName, msg)
-	assert.Equal(a.Operator, b.Operator, msg)
-}
-
 func checkKernelConstraint(assert *assert.Assertions, f Constraint, version string, op Operator, msg string) {
 	c := Constraints{}
 
@@ -156,19 +85,6 @@ func checkKernelConstraint(assert *assert.Assertions, f Constraint, version stri
 	assert.Equal(c.Operator, op, msg)
 }
 
-// runCommand runs a command and returns its output
-func runCommand(args ...string) ([]string, error) {
-	cmd := exec.Command(args[0], args[1:]...)
-	bytes, err := cmd.Output()
-	if err != nil {
-		return []string{}, err
-	}
-
-	output := strings.Split(string(bytes), "\n")
-
-	return output, nil
-}
-
 // semverBumpVersion takes an existing semantic version and increments one or
 // more parts of it, returning the new version number as a string.
 func semverBumpVersion(ver semver.Version, bumpMajor, bumpMinor, bumpPatch bool) (string, error) {
@@ -263,56 +179,6 @@ func decrementVersion(version string) (string, error) {
 	return changeVersion(version, true)
 }
 
-// testGetDistro is an alternative implementation of getDistroDetails() used
-// for testing.
-func testGetDistro() (name, version string, err error) {
-	files := []string{"/etc/os-release", "/usr/lib/os-release"}
-
-	for _, file := range files {
-		if !fileExists(file) {
-			continue
-		}
-
-		output, err := runCommand("grep", "^ID=", file)
-		if err != nil {
-			return "", "", err
-		}
-
-		line := output[0]
-		fields := strings.Split(line, "=")
-		if name == "" {
-			name = strings.Trim(fields[1], `"`)
-			name = strings.ToLower(name)
-		}
-
-		output, err = runCommand("grep", "^VERSION_ID=", file)
-		if err != nil {
-			return "", "", err
-		}
-
-		line = output[0]
-		fields = strings.Split(line, "=")
-		if version == "" {
-			version = strings.Trim(fields[1], `"`)
-			version = strings.ToLower(version)
-		}
-	}
-
-	if name != "" && version != "" {
-		return name, version, nil
-	}
-
-	if name == "" {
-		return "", "", errUnknownDistroName
-	}
-
-	if version == "" {
-		return "", "", errUnknownDistroVersion
-	}
-
-	return "", "", errors.New("BUG: something bad happened")
-}
-
 func testGetKernelVersion() (version string, err error) {
 	const file = "/proc/version"
 
@@ -360,11 +226,6 @@ func TestOperatorString(t *testing.T) {
 }
 
 func TestNewTestConstraint(t *testing.T) {
-	if getDistroErr != nil {
-		t.Skipf("skipping as unable to determine distro name/version: %v",
-			getDistroErr)
-	}
-
 	if getKernelErr != nil {
 		t.Skipf("skipping as unable to determine kernel version: %v",
 			getKernelErr)
@@ -379,17 +240,11 @@ func TestNewTestConstraint(t *testing.T) {
 
 		assert.Equal(debug, c.Debug, msg)
 
-		assert.Equal(distroName, c.DistroName, msg)
-		assert.Equal(distroVersion, c.DistroVersion, msg)
 		assert.Equal(kernelVersion, c.KernelVersion, msg)
 		assert.Equal(thisUID, c.ActualEUID)
 
 		toCheck := []string{
-			distroName,
-			distroVersion,
 			kernelVersion,
-			c.DistroName,
-			c.DistroVersion,
 			c.KernelVersion,
 		}
 
@@ -438,26 +293,6 @@ func TestGetFileContents(t *testing.T) {
 	}
 }
 
-func TestGetDistroDetails(t *testing.T) {
-	assert := assert.New(t)
-
-	if getDistroErr == errUnknownDistroName {
-		t.Skip(skipUnknownDistroName)
-	}
-
-	assert.NoError(getDistroErr)
-	assert.NotNil(distroName)
-	assert.NotNil(distroVersion)
-
-	name, version, err := getDistroDetails()
-	assert.NoError(err)
-	assert.NotNil(name)
-	assert.NotNil(version)
-
-	assert.Equal(name, distroName)
-	assert.Equal(version, distroVersion)
-}
-
 func TestGetKernelVersion(t *testing.T) {
 	assert := assert.New(t)
 
@@ -471,158 +306,6 @@ func TestGetKernelVersion(t *testing.T) {
 	assert.Equal(version, kernelVersion)
 }
 
-func TestConstraintHandleDistroName(t *testing.T) {
-	assert := assert.New(t)
-
-	// nolint: govet
-	type testData struct {
-		distro      string
-		op          Operator
-		result      Result
-		expectError bool
-	}
-
-	distroName, _, err := testGetDistro()
-	if err != nil && err == errUnknownDistroName {
-		t.Skip(skipUnknownDistroName)
-	}
-
-	// Look for the first distro that is not the same as the distro this
-	// test is currently running on.
-	differentDistro := getAnotherDistro(distroName)
-
-	data := []testData{
-		{"", eqOperator, Result{}, true},
-		{"", neOperator, Result{}, true},
-		{"", invalidOperator, Result{}, true},
-		{distroName, invalidOperator, Result{}, true},
-		{distroName, invalidOperator, Result{}, true},
-
-		{
-			distroName,
-			eqOperator,
-			Result{
-				Description: distroName,
-				Success:     true,
-			},
-			false,
-		},
-		{
-			distroName,
-			neOperator,
-			Result{
-				Description: distroName,
-				Success:     false,
-			},
-			false,
-		},
-		{
-			differentDistro,
-			eqOperator,
-			Result{
-				Description: differentDistro,
-				Success:     false,
-			},
-			false,
-		},
-
-		{
-			differentDistro,
-			neOperator,
-			Result{
-				Description: differentDistro,
-				Success:     true,
-			},
-			false,
-		},
-	}
-
-	for _, debug := range []bool{true, false} {
-		tc := NewTestConstraint(debug)
-
-		for i, d := range data {
-			result, err := tc.handleDistroName(d.distro, d.op)
-
-			msg := fmt.Sprintf("test[%d]: %+v, result: %+v", i, d, result)
-
-			if d.expectError {
-				assert.Error(err, msg)
-				continue
-
-			}
-
-			assert.NoError(err, msg)
-			assert.Equal(result.Success, d.result.Success, msg)
-			assert.NotNil(result.Description, msg)
-		}
-	}
-}
-
-func TestConstraintHandleDistroVersion(t *testing.T) {
-	assert := assert.New(t)
-
-	assert.NotNil(distroVersion)
-
-	// Generate a new distro version for testing purposes. Since we don't
-	// know the format of this particular distros versioning scheme, we
-	// need to calculate it.
-	higherVersion, err := incrementVersion(distroVersion)
-	assert.NoError(err)
-	assert.NotEqual(distroVersion, higherVersion)
-
-	// nolint: govet
-	type testData struct {
-		version     string
-		op          Operator
-		result      Result
-		expectError bool
-	}
-
-	data := []testData{
-		{"", eqOperator, Result{}, true},
-		{"", geOperator, Result{}, true},
-		{"", gtOperator, Result{}, true},
-		{"", leOperator, Result{}, true},
-		{"", ltOperator, Result{}, true},
-		{"", neOperator, Result{}, true},
-
-		{distroVersion, eqOperator, Result{Success: true}, false},
-		{higherVersion, eqOperator, Result{Success: false}, false},
-
-		{distroVersion, gtOperator, Result{Success: false}, false},
-		{higherVersion, gtOperator, Result{Success: false}, false},
-
-		{distroVersion, geOperator, Result{Success: true}, false},
-		{higherVersion, geOperator, Result{Success: false}, false},
-
-		{distroVersion, ltOperator, Result{Success: false}, false},
-		{higherVersion, ltOperator, Result{Success: true}, false},
-
-		{distroVersion, leOperator, Result{Success: true}, false},
-		{higherVersion, leOperator, Result{Success: true}, false},
-
-		{distroVersion, neOperator, Result{Success: false}, false},
-		{higherVersion, neOperator, Result{Success: true}, false},
-	}
-
-	for _, debug := range []bool{true, false} {
-		tc := NewTestConstraint(debug)
-
-		for i, d := range data {
-			result, err := tc.handleDistroVersion(d.version, d.op)
-
-			msg := fmt.Sprintf("test[%d]: %+v, result: %+v", i, d, result)
-
-			if d.expectError {
-				assert.Error(err, msg)
-				continue
-			}
-
-			assert.Equal(d.result.Success, result.Success, msg)
-		}
-	}
-}
-
 func TestConstraintHandleVersionType(t *testing.T) {
 	assert := assert.New(t)
 
@@ -638,7 +321,6 @@ func TestConstraintHandleVersionType(t *testing.T) {
 
 	data := []testData{
 		//----------
-
 		{"", "", eqOperator, "", Result{}, true},
 
 		{"name", "foo", eqOperator, "", Result{}, true},
@@ -960,10 +642,12 @@ func TestNeedUID(t *testing.T) {
 	data := []testDataUID{
 		uidEqualsRootData,
 		uidNotEqualsRootData,
-		{thisUID, eqOperator, Constraints{
-			Operator: eqOperator,
-			UID:      thisUID,
-			UIDSet:   true},
+		{
+			thisUID, eqOperator, Constraints{
+				Operator: eqOperator,
+				UID:      thisUID,
+				UIDSet:   true,
+			},
 		},
 	}
 
@@ -1000,62 +684,6 @@ func TestNeedNonRoot(t *testing.T) {
 	checkUIDConstraints(assert, c, uidNotEqualsRootData.c, "TestNeedNonRoot")
 }
 
-func TestNeedDistroWithOp(t *testing.T) {
-	assert := assert.New(t)
-
-	if getDistroErr == errUnknownDistroName {
-		t.Skip(skipUnknownDistroName)
-	}
-
-	data := []testDataDistro{
-		distroEqualsCurrentData,
-		distroNotEqualsCurrentData,
-
-		// check name provided is lower-cased
-		{
-			strings.ToUpper(distroName),
-			eqOperator,
-			Constraints{
-				DistroName: distroName,
-				Operator:   eqOperator,
-			},
-		},
-	}
-
-	for i, d := range data {
-
-		c := Constraints{}
-
-		f := NeedDistroWithOp(d.distro, d.op)
-		f(&c)
-
-		desc := fmt.Sprintf("test[%d]: %+v, constraints: %+v", i, d, c)
-		checkDistroConstraints(assert, d.c, c, desc)
-	}
-}
-
-func TestNeedDistroEquals(t *testing.T) {
-	assert := assert.New(t)
-
-	c := Constraints{}
-
-	f := NeedDistroEquals(distroName)
-	f(&c)
-
-	checkDistroConstraints(assert, c, distroEqualsCurrentData.c, "TestNeedDistroEquals")
-}
-
-func TestNeedDistroNotEquals(t *testing.T) {
-	assert := assert.New(t)
-
-	c := Constraints{}
-
-	f := NeedDistroNotEquals(distroName)
-	f(&c)
-
-	checkDistroConstraints(assert, c, distroNotEqualsCurrentData.c, "TestNeedDistroNotEquals")
-}
-
 func TestWithIssue(t *testing.T) {
 	assert := assert.New(t)
 
@@ -1171,22 +799,7 @@ func TestConstraintNotValid(t *testing.T) {
 			assert.False(result)
 		}
 
-		// Now test specification of multiple constraints
-		if root {
-			result := tc.NotValid(NeedRoot(), NeedDistro(distroName))
-			assert.False(result)
-
-			result = tc.NotValid(NeedNonRoot(), NeedDistro(distroName))
-			assert.True(result)
-		} else {
-			result := tc.NotValid(NeedRoot(), NeedDistro(distroName))
-			assert.True(result)
-
-			result = tc.NotValid(NeedNonRoot(), NeedDistro(distroName))
-			assert.False(result)
-		}
 	}
-
 }
 
 func TestConstraintNotValidKernelVersion(t *testing.T) {
@@ -1278,62 +891,6 @@ func TestConstraintNotValidKernelVersion(t *testing.T) {
 	}
 }
 
-func TestConstraintNotValidDistroVersion(t *testing.T) {
-	assert := assert.New(t)
-
-	assert.NotNil(distroVersion)
-
-	// Generate new distro versions for testing purposes based on the
-	// current kernel version.
-	higherVersion, err := incrementVersion(distroVersion)
-	assert.NoError(err)
-	assert.NotEqual(distroVersion, higherVersion)
-
-	lowerVersion, err := decrementVersion(distroVersion)
-	assert.NoError(err)
-	assert.NotEqual(distroVersion, lowerVersion)
-
-	for _, debug := range []bool{true, false} {
-		tc := NewTestConstraint(debug)
-
-		result := tc.NotValid(NeedDistroVersionEquals(higherVersion))
-		assert.True(result)
-
-		result = tc.NotValid(NeedDistroVersionEquals(distroVersion))
-		assert.False(result)
-
-		result = tc.NotValid(NeedDistroVersionLE(higherVersion))
-		assert.False(result)
-
-		result = tc.NotValid(NeedDistroVersionLE(distroVersion))
-		assert.False(result)
-
-		result = tc.NotValid(NeedDistroVersionLT(higherVersion))
-		assert.False(result)
-
-		result = tc.NotValid(NeedDistroVersionLT(distroVersion))
-		assert.True(result)
-
-		result = tc.NotValid(NeedDistroVersionGE(higherVersion))
-		assert.True(result)
-
-		result = tc.NotValid(NeedDistroVersionGE(distroVersion))
-		assert.False(result)
-
-		result = tc.NotValid(NeedDistroVersionGT(higherVersion))
-		assert.True(result)
-
-		result = tc.NotValid(NeedDistroVersionGT(distroVersion))
-		assert.True(result)
-
-		result = tc.NotValid(NeedDistroVersionNotEquals(higherVersion))
-		assert.False(result)
-
-		result = tc.NotValid(NeedDistroVersionNotEquals(distroVersion))
-		assert.True(result)
-	}
-}
-
 func TestConstraintConstraintValid(t *testing.T) {
 	assert := assert.New(t)
 
@@ -1352,100 +909,6 @@ func TestConstraintConstraintValid(t *testing.T) {
 			true,
 			TestConstraint{Issue: issue},
 		},
-
-		{
-			NeedDistroWithOp(distroName, eqOperator),
-			true,
-			TestConstraint{
-				Passed: []Result{
-					{Success: true},
-				},
-			},
-		},
-		{
-			NeedDistroWithOp(distroName, neOperator),
-			false,
-			TestConstraint{
-				Failed: []Result{
-					{Success: false},
-				},
-			},
-		},
-		{
-			NeedDistroWithOp(getAnotherDistro(distroName), eqOperator),
-			false,
-			TestConstraint{
-				Failed: []Result{
-					{Success: false},
-				},
-			},
-		},
-		{
-			NeedDistroWithOp(getAnotherDistro(distroName), neOperator),
-			true,
-			TestConstraint{
-				Failed: []Result{
-					{Success: true},
-				},
-			},
-		},
-
-		{
-			NeedDistroEquals(distroName),
-			true,
-			TestConstraint{
-				Passed: []Result{
-					{Success: true},
-				},
-			},
-		},
-		{
-			NeedDistroEquals(getAnotherDistro(distroName)),
-			false,
-			TestConstraint{
-				Failed: []Result{
-					{Success: false},
-				},
-			},
-		},
-
-		{
-			NeedDistroNotEquals(getAnotherDistro(distroName)),
-			true,
-			TestConstraint{
-				Passed: []Result{
-					{Success: true},
-				},
-			},
-		},
-		{
-			NeedDistroNotEquals(distroName),
-			false,
-			TestConstraint{
-				Failed: []Result{
-					{Success: false},
-				},
-			},
-		},
-
-		{
-			NeedDistro(distroName),
-			true,
-			TestConstraint{
-				Passed: []Result{
-					{Success: true},
-				},
-			},
-		},
-		{
-			NeedDistro(getAnotherDistro(distroName)),
-			false,
-			TestConstraint{
-				Failed: []Result{
-					{Success: false},
-				},
-			},
-		},
 	}
 
 	if root {
@@ -1541,7 +1004,6 @@ func TestEvalIntVersion(t *testing.T) {
 
 	data := []testData{
 		//----------
-
 		{"", eqOperator, "", false, true},
 		{"", eqOperator, "1", false, true},
 		{"1", eqOperator, "", false, true},
@@ -1647,7 +1109,6 @@ func TestEvalFloatVersion(t *testing.T) {
 
 	data := []testData{
 		//----------
-
 		{"", eqOperator, "", false, true},
 		{"foo", eqOperator, "", false, true},
 		{"", eqOperator, "foo", false, true},
@@ -1763,7 +1224,6 @@ func TestEvalSemverVersion(t *testing.T) {
 
 	data := []testData{
 		//----------
-
 		{"", eqOperator, "", false, true},
 		{"foo", eqOperator, "", false, true},
 		{"", eqOperator, "foo", false, true},

src/runtime/pkg/katautils/config.go

@@ -1,5 +1,6 @@
 // Copyright (c) 2018-2022 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
+// Copyright (c) 2021 Adobe Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -154,6 +155,7 @@ type runtime struct {
 	SandboxCgroupOnly         bool     `toml:"sandbox_cgroup_only"`
 	StaticSandboxResourceMgmt bool     `toml:"static_sandbox_resource_mgmt"`
 	EnablePprof               bool     `toml:"enable_pprof"`
+	DisableGuestEmptyDir      bool     `toml:"disable_guest_empty_dir"`
 }
 
 type agent struct {
@@ -568,7 +570,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EntropySource:         h.GetEntropySource(),
 		EntropySourceList:     h.EntropySourceList,
 		DefaultBridges:        h.defaultBridges(),
-		DisableBlockDeviceUse: h.DisableBlockDeviceUse,
+		DisableBlockDeviceUse: false, // shared fs is not supported in Firecracker,
 		HugePages:             h.HugePages,
 		Debug:                 h.Debug,
 		DisableNestingChecks:  h.DisableNestingChecks,
@@ -1174,6 +1176,8 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 	}
 	config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts
 
+	config.DisableGuestEmptyDir = tomlConf.Runtime.DisableGuestEmptyDir
+
 	if err := checkConfig(config); err != nil {
 		return "", config, err
 	}

src/runtime/pkg/katautils/create.go

@@ -1,5 +1,6 @@
 // Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
+// Copyright (c) 2021 Adobe Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -96,12 +97,12 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon
 // For the given pod ephemeral volume is created only once
 // backed by tmpfs inside the VM. For successive containers
 // of the same pod the already existing volume is reused.
-func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
+func SetEphemeralStorageType(ociSpec specs.Spec, disableGuestEmptyDir bool) specs.Spec {
 	for idx, mnt := range ociSpec.Mounts {
 		if vc.IsEphemeralStorage(mnt.Source) {
 			ociSpec.Mounts[idx].Type = vc.KataEphemeralDevType
 		}
-		if vc.Isk8sHostEmptyDir(mnt.Source) {
+		if vc.Isk8sHostEmptyDir(mnt.Source) && !disableGuestEmptyDir {
 			ociSpec.Mounts[idx].Type = vc.KataLocalDevType
 		}
 	}
@@ -218,14 +219,14 @@ func checkForFIPS(sandboxConfig *vc.SandboxConfig) error {
 }
 
 // CreateContainer create a container
-func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) {
+func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool, disableGuestEmptyDir bool) (vc.Process, error) {
 	var c vc.VCContainer
 
 	span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags)
 	katatrace.AddTags(span, "container_id", containerID)
 	defer span.End()
 
-	ociSpec = SetEphemeralStorageType(ociSpec)
+	ociSpec = SetEphemeralStorageType(ociSpec, disableGuestEmptyDir)
 
 	contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput)
 	if err != nil {

src/runtime/pkg/katautils/create_test.go

@@ -1,5 +1,6 @@
 // Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
+// Copyright (c) 2021 Adobe Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -145,7 +146,7 @@ func TestSetEphemeralStorageType(t *testing.T) {
 
 	ociMounts = append(ociMounts, mount)
 	ociSpec.Mounts = ociMounts
-	ociSpec = SetEphemeralStorageType(ociSpec)
+	ociSpec = SetEphemeralStorageType(ociSpec, false)
 
 	mountType := ociSpec.Mounts[0].Type
 	assert.Equal(mountType, "ephemeral",
@@ -367,7 +368,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 	rootFs := vc.RootFs{Mounted: true}
 
 	for _, disableOutput := range []bool{true, false} {
-		_, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput)
+		_, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false)
 		assert.Error(err)
 		assert.False(vcmock.IsMockError(err))
 		assert.True(strings.Contains(err.Error(), containerType))
@@ -395,7 +396,7 @@ func TestCreateContainerFail(t *testing.T) {
 	rootFs := vc.RootFs{Mounted: true}
 
 	for _, disableOutput := range []bool{true, false} {
-		_, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput)
+		_, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false)
 		assert.Error(err)
 		assert.True(vcmock.IsMockError(err))
 	}
@@ -430,7 +431,7 @@ func TestCreateContainer(t *testing.T) {
 	rootFs := vc.RootFs{Mounted: true}
 
 	for _, disableOutput := range []bool{true, false} {
-		_, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput)
+		_, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false)
 		assert.NoError(err)
 	}
 }

src/runtime/pkg/oci/utils.go

@@ -1,4 +1,5 @@
 // Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2021 Adobe Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -146,6 +147,9 @@ type RuntimeConfig struct {
 	// Determines if enable pprof
 	EnablePprof bool
 
+	// Determines if Kata creates emptyDir on the guest
+	DisableGuestEmptyDir bool
+
 	// Offload the CRI image management service to the Kata agent.
 	ServiceOffload bool
 }

src/runtime/pkg/resourcecontrol/controller.go

@@ -22,14 +22,14 @@ func SetLogger(logger *logrus.Entry) {
 	controllerLogger = logger.WithFields(fields)
 }
 
-// HypervisorType describes an hypervisor type.
+// ResourceControllerType describes a resource controller type.
 type ResourceControllerType string
 
 const (
 	LinuxCgroups ResourceControllerType = "cgroups"
 )
 
-// String converts an hypervisor type to a string.
+// String converts a resource type to a string.
 func (rType *ResourceControllerType) String() string {
 	switch *rType {
 	case LinuxCgroups:

src/runtime/pkg/resourcecontrol/utils_linux.go

@@ -20,7 +20,7 @@ import (
 // DefaultResourceControllerID runtime-determined location in the cgroups hierarchy.
 const DefaultResourceControllerID = "/vc"
 
-// validCgroupPath returns a valid cgroup path.
+// ValidCgroupPath returns a valid cgroup path.
 // see https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#cgroups-path
 func ValidCgroupPath(path string, systemdCgroup bool) (string, error) {
 	if IsSystemdCgroup(path) {

src/runtime/pkg/utils/shimclient/shim_management_client.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2022 Databricks Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package shimclient
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+	shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
+)
+
+// BuildShimClient builds and returns an http client for communicating with the provided sandbox
+func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) {
+	return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout)
+}
+
+// buildUnixSocketClient build http client for Unix socket
+func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) {
+	transport := &http.Transport{
+		DisableKeepAlives: true,
+		Dial: func(proto, addr string) (conn net.Conn, err error) {
+			return cdshim.AnonDialer(socketAddr, timeout)
+		},
+	}
+
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	if timeout > 0 {
+		client.Timeout = timeout
+	}
+
+	return client, nil
+}
+
+func DoGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) {
+	client, err := BuildShimClient(sandboxID, timeoutInSeconds)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath))
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		resp.Body.Close()
+	}()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+func DoPost(sandboxID string, timeoutInSeconds time.Duration, urlPath string, payload []byte) error {
+	client, err := BuildShimClient(sandboxID, timeoutInSeconds)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Post(fmt.Sprintf("http://shim/%s", urlPath), "application/json", bytes.NewBuffer(payload))
+	defer func() {
+		resp.Body.Close()
+	}()
+	return err
+}

src/runtime/virtcontainers/agent.go

@@ -192,6 +192,12 @@ type agent interface {
 	// getAgentMetrics get metrics of agent and guest through agent
 	getAgentMetrics(context.Context, *grpc.GetMetricsRequest) (*grpc.Metrics, error)
 
+	//getGuestVolumeStats get the filesystem stats of a volume specified by the volume mount path on the guest.
+	getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error)
+
+	// resizeGuestVolume resizes a volume specified by the volume mount path on the guest.
+	resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error
+
 	// pullImage will tell the agent to pull an image inside the Pod Sandbox
 	image.ImageService
 }

src/runtime/virtcontainers/clh.go

@@ -198,10 +198,125 @@ func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error {
 	return nil
 }
 
+func (clh *cloudHypervisor) createVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) {
+	if !clh.supportsSharedFS() {
+		clh.Logger().Info("SharedFS is not supported")
+		return nil, nil
+	}
+
+	virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id)
+	if err != nil {
+		return nil, err
+	}
+
+	if clh.config.SharedFS == config.VirtioFSNydus {
+		apiSockPath, err := clh.nydusdAPISocketPath(clh.id)
+		if err != nil {
+			clh.Logger().WithError(err).Error("Invalid api socket path for nydusd")
+			return nil, err
+		}
+		nd := &nydusd{
+			path:        clh.config.VirtioFSDaemon,
+			sockPath:    virtiofsdSocketPath,
+			apiSockPath: apiSockPath,
+			sourcePath:  sharedPath,
+			debug:       clh.config.Debug,
+			extraArgs:   clh.config.VirtioFSExtraArgs,
+			startFn:     startInShimNS,
+		}
+		nd.setupShareDirFn = nd.setupPassthroughFS
+		return nd, nil
+	}
+
+	// default: use virtiofsd
+	return &virtiofsd{
+		path:       clh.config.VirtioFSDaemon,
+		sourcePath: sharedPath,
+		socketPath: virtiofsdSocketPath,
+		extraArgs:  clh.config.VirtioFSExtraArgs,
+		debug:      clh.config.Debug,
+		cache:      clh.config.VirtioFSCache,
+	}, nil
+}
+
+func (clh *cloudHypervisor) setupVirtiofsDaemon(ctx context.Context) error {
+	if !clh.supportsSharedFS() {
+		clh.Logger().Info("SharedFS is not supported")
+		return nil
+	}
+
+	if clh.config.SharedFS == config.Virtio9P {
+		return errors.New("cloud-hypervisor only supports virtio based file sharing")
+	}
+
+	// virtioFS or virtioFsNydus
+	clh.Logger().WithField("function", "setupVirtiofsDaemon").Info("Starting virtiofsDaemon")
+
+	if clh.virtiofsDaemon == nil {
+		return errors.New("Missing virtiofsDaemon configuration")
+	}
+
+	pid, err := clh.virtiofsDaemon.Start(ctx, func() {
+		clh.StopVM(ctx, false)
+	})
+	if err != nil {
+		return err
+	}
+	clh.state.VirtiofsDaemonPid = pid
+
+	return nil
+}
+
+func (clh *cloudHypervisor) stopVirtiofsDaemon(ctx context.Context) (err error) {
+	if !clh.supportsSharedFS() {
+		clh.Logger().Info("SharedFS is not supported")
+		return nil
+	}
+
+	if clh.state.VirtiofsDaemonPid == 0 {
+		clh.Logger().Warn("The virtiofsd had stopped")
+		return nil
+	}
+
+	err = clh.virtiofsDaemon.Stop(ctx)
+	if err != nil {
+		return err
+	}
+
+	clh.state.VirtiofsDaemonPid = 0
+
+	return nil
+}
+
+func (clh *cloudHypervisor) loadVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) {
+	if !clh.supportsSharedFS() {
+		clh.Logger().Info("SharedFS is not supported")
+		return nil, nil
+	}
+
+	virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id)
+	if err != nil {
+		return nil, err
+	}
+
+	return &virtiofsd{
+		PID:        clh.state.VirtiofsDaemonPid,
+		sourcePath: sharedPath,
+		debug:      clh.config.Debug,
+		socketPath: virtiofsdSocketPath,
+	}, nil
+}
+
 func (clh *cloudHypervisor) nydusdAPISocketPath(id string) (string, error) {
 	return utils.BuildSocketPath(clh.config.VMStorePath, id, nydusdAPISock)
 }
 
+func (clh *cloudHypervisor) supportsSharedFS() bool {
+	caps := clh.Capabilities(clh.ctx)
+
+	return caps.IsFsSharingSupported()
+}
+
 func (clh *cloudHypervisor) enableProtection() error {
 	protection, err := availableGuestProtection()
 	if err != nil {
@@ -248,19 +363,15 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 
 	clh.Logger().WithField("function", "CreateVM").Info("creating Sandbox")
 
-	virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id)
-	if err != nil {
-		return nil
-	}
-
 	if clh.state.PID > 0 {
 		clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state")
-		clh.virtiofsDaemon = &virtiofsd{
-			PID:        clh.state.VirtiofsDaemonPid,
-			sourcePath: hypervisorConfig.SharedPath,
-			debug:      clh.config.Debug,
-			socketPath: virtiofsdSocketPath,
+
+		virtiofsDaemon, err := clh.loadVirtiofsDaemon(hypervisorConfig.SharedFS)
+		if err != nil {
+			return err
 		}
+		clh.virtiofsDaemon = virtiofsDaemon
+
 		return nil
 	}
 
@@ -402,32 +513,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 		ApiInternal: chclient.NewAPIClient(cfg).DefaultApi,
 	}
 
-	clh.virtiofsDaemon = &virtiofsd{
-		path:       clh.config.VirtioFSDaemon,
-		sourcePath: filepath.Join(GetSharePath(clh.id)),
-		socketPath: virtiofsdSocketPath,
-		extraArgs:  clh.config.VirtioFSExtraArgs,
-		debug:      clh.config.Debug,
-		cache:      clh.config.VirtioFSCache,
-	}
-
-	if clh.config.SharedFS == config.VirtioFSNydus {
-		apiSockPath, err := clh.nydusdAPISocketPath(clh.id)
-		if err != nil {
-			clh.Logger().WithError(err).Error("Invalid api socket path for nydusd")
-			return err
-		}
-		nd := &nydusd{
-			path:        clh.config.VirtioFSDaemon,
-			sockPath:    virtiofsdSocketPath,
-			apiSockPath: apiSockPath,
-			sourcePath:  filepath.Join(GetSharePath(clh.id)),
-			debug:       clh.config.Debug,
-			extraArgs:   clh.config.VirtioFSExtraArgs,
-			startFn:     startInShimNS,
-		}
-		nd.setupShareDirFn = nd.setupPassthroughFS
-		clh.virtiofsDaemon = nd
+	clh.virtiofsDaemon, err = clh.createVirtiofsDaemon(filepath.Join(GetSharePath(clh.id)))
+	if err != nil {
+		return err
 	}
 
 	if clh.config.SGXEPCSize > 0 {
@@ -461,10 +549,6 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error {
 		return err
 	}
 
-	if clh.virtiofsDaemon == nil {
-		return errors.New("Missing virtiofsDaemon configuration")
-	}
-
 	// This needs to be done as late as possible, just before launching
 	// virtiofsd are executed by kata-runtime after this call, run with
 	// the SELinux label. If these processes require privileged, we do
@@ -477,24 +561,20 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error {
 		defer label.SetProcessLabel("")
 	}
 
-	if clh.config.SharedFS == config.VirtioFS || clh.config.SharedFS == config.VirtioFSNydus {
-		clh.Logger().WithField("function", "StartVM").Info("Starting virtiofsDaemon")
-		pid, err := clh.virtiofsDaemon.Start(ctx, func() {
-			clh.StopVM(ctx, false)
-		})
-		if err != nil {
-			return err
-		}
-		clh.state.VirtiofsDaemonPid = pid
-	} else {
-		return errors.New("cloud-hypervisor only supports virtio based file sharing")
+	err = clh.setupVirtiofsDaemon(ctx)
+	if err != nil {
+		return err
 	}
+	defer func() {
+		if err != nil {
+			if shutdownErr := clh.stopVirtiofsDaemon(ctx); shutdownErr != nil {
+				clh.Logger().WithError(shutdownErr).Warn("error shutting down VirtiofsDaemon")
+			}
+		}
+	}()
 
 	pid, err := clh.launchClh()
 	if err != nil {
-		if shutdownErr := clh.virtiofsDaemon.Stop(ctx); shutdownErr != nil {
-			clh.Logger().WithError(shutdownErr).Warn("error shutting down VirtiofsDaemon")
-		}
 		return fmt.Errorf("failed to launch cloud-hypervisor: %q", err)
 	}
 	clh.state.PID = pid
@@ -638,10 +718,6 @@ func (clh *cloudHypervisor) HotplugAddDevice(ctx context.Context, devInfo interf
 	span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugAddDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id})
 	defer span.End()
 
-	if clh.config.ConfidentialGuest {
-		return nil, errors.New("Device hotplug addition is not supported in confidential mode")
-	}
-
 	switch devType {
 	case BlockDev:
 		drive := devInfo.(*config.BlockDrive)
@@ -659,10 +735,6 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int
 	span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugRemoveDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id})
 	defer span.End()
 
-	if clh.config.ConfidentialGuest {
-		return nil, errors.New("Device hotplug removal is not supported in confidential mode")
-	}
-
 	var deviceID string
 
 	switch devType {
@@ -890,6 +962,10 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{},
 	case types.HybridVSock:
 		clh.addVSock(defaultGuestVSockCID, v.UdsPath)
 	case types.Volume:
+		if !clh.supportsSharedFS() {
+			return fmt.Errorf("SharedFS is not supported")
+		}
+
 		err = clh.addVolume(v)
 	default:
 		clh.Logger().WithField("function", "AddDevice").Warnf("Add device of type %v is not supported.", v)
@@ -916,10 +992,10 @@ func (clh *cloudHypervisor) Capabilities(ctx context.Context) types.Capabilities
 
 	clh.Logger().WithField("function", "Capabilities").Info("get Capabilities")
 	var caps types.Capabilities
-	caps.SetFsSharingSupport()
 	if !clh.config.ConfidentialGuest {
-		caps.SetBlockDeviceHotplugSupport()
+		caps.SetFsSharingSupport()
 	}
+	caps.SetBlockDeviceHotplugSupport()
 	return caps
 }
 
@@ -957,12 +1033,9 @@ func (clh *cloudHypervisor) terminate(ctx context.Context, waitOnly bool) (err e
 		return err
 	}
 
-	if clh.virtiofsDaemon == nil {
-		return errors.New("virtiofsDaemon config is nil, failed to stop it")
-	}
-
 	clh.Logger().Debug("stop virtiofsDaemon")
-	if err = clh.virtiofsDaemon.Stop(ctx); err != nil {
+
+	if err = clh.stopVirtiofsDaemon(ctx); err != nil {
 		clh.Logger().WithError(err).Error("failed to stop virtiofsDaemon")
 	}
 

src/runtime/virtcontainers/container.go

@@ -15,6 +15,7 @@ import (
 	"syscall"
 	"time"
 
+	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
@@ -598,22 +599,50 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
 	}
 
 	// iterate all mounts and create block device if it's block based.
-	for i, m := range c.mounts {
-		if len(m.BlockDeviceID) > 0 {
+	for i := range c.mounts {
+		if len(c.mounts[i].BlockDeviceID) > 0 {
 			// Non-empty m.BlockDeviceID indicates there's already one device
 			// associated with the mount,so no need to create a new device for it
 			// and we only create block device for bind mount
 			continue
 		}
 
-		if m.Type != "bind" {
+		if c.mounts[i].Type != "bind" {
 			// We only handle for bind-mounts
 			continue
 		}
 
+		// Handle directly assigned volume. Update the mount info based on the mount info json.
+		mntInfo, e := volume.VolumeMountInfo(c.mounts[i].Source)
+		if e != nil && !os.IsNotExist(e) {
+			c.Logger().WithError(e).WithField("mount-source", c.mounts[i].Source).
+				Error("failed to parse the mount info file for a direct assigned volume")
+			continue
+		}
+
+		if mntInfo != nil {
+			// Write out sandbox info file on the mount source to allow CSI to communicate with the runtime
+			if err := volume.RecordSandboxId(c.sandboxID, c.mounts[i].Source); err != nil {
+				c.Logger().WithError(err).Error("error writing sandbox info")
+			}
+
+			readonly := false
+			for _, flag := range mntInfo.Options {
+				if flag == "ro" {
+					readonly = true
+					break
+				}
+			}
+
+			c.mounts[i].Source = mntInfo.Device
+			c.mounts[i].Type = mntInfo.FsType
+			c.mounts[i].Options = mntInfo.Options
+			c.mounts[i].ReadOnly = readonly
+		}
+
 		var stat unix.Stat_t
-		if err := unix.Stat(m.Source, &stat); err != nil {
-			return fmt.Errorf("stat %q failed: %v", m.Source, err)
+		if err := unix.Stat(c.mounts[i].Source, &stat); err != nil {
+			return fmt.Errorf("stat %q failed: %v", c.mounts[i].Source, err)
 		}
 
 		var di *config.DeviceInfo
@@ -623,17 +652,17 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
 		// instead of passing this as a shared mount.
 		if stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
 			di = &config.DeviceInfo{
-				HostPath:      m.Source,
-				ContainerPath: m.Destination,
+				HostPath:      c.mounts[i].Source,
+				ContainerPath: c.mounts[i].Destination,
 				DevType:       "b",
 				Major:         int64(unix.Major(uint64(stat.Rdev))),
 				Minor:         int64(unix.Minor(uint64(stat.Rdev))),
-				ReadOnly:      m.ReadOnly,
+				ReadOnly:      c.mounts[i].ReadOnly,
 			}
 			// Check whether source can be used as a pmem device
-		} else if di, err = config.PmemDeviceInfo(m.Source, m.Destination); err != nil {
+		} else if di, err = config.PmemDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination); err != nil {
 			c.Logger().WithError(err).
-				WithField("mount-source", m.Source).
+				WithField("mount-source", c.mounts[i].Source).
 				Debug("no loop device")
 		}
 
@@ -642,7 +671,7 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
 			if err != nil {
 				// Do not return an error, try to create
 				// devices for other mounts
-				c.Logger().WithError(err).WithField("mount-source", m.Source).
+				c.Logger().WithError(err).WithField("mount-source", c.mounts[i].Source).
 					Error("device manager failed to create new device")
 				continue
 

src/runtime/virtcontainers/interfaces.go

@@ -78,6 +78,9 @@ type VCSandbox interface {
 	GetAgentMetrics(ctx context.Context) (string, error)
 	GetAgentURL() (string, error)
 
+	GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error)
+	ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error
+
 	// Image management inside Sandbox
 	image.ImageService
 }

src/runtime/virtcontainers/kata_agent.go

@@ -6,6 +6,7 @@
 package virtcontainers
 
 import (
+	b64 "encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -140,6 +141,8 @@ const (
 	grpcGetOOMEventRequest       = "grpc.GetOOMEventRequest"
 	grpcGetMetricsRequest        = "grpc.GetMetricsRequest"
 	grpcAddSwapRequest           = "grpc.AddSwapRequest"
+	grpcVolumeStatsRequest       = "grpc.VolumeStatsRequest"
+	grpcResizeVolumeRequest      = "grpc.ResizeVolumeRequest"
 )
 
 // newKataAgent returns an agent from an agent type.
@@ -884,35 +887,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st
 	return nil
 }
 
-func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error {
-	ociMounts := spec.Mounts
-	var index int
-	var m specs.Mount
-
-	for i, v := range volumeStorages {
-		for index, m = range ociMounts {
-			if m.Destination != v.MountPoint {
-				continue
-			}
-
-			// Create a temporary location to mount the Storage. Mounting to the correct location
-			// will be handled by the OCI mount structure.
-			filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
-			path := filepath.Join(kataGuestSandboxStorageDir(), filename)
-
-			k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
-			ociMounts[index].Source = path
-			volumeStorages[i].MountPoint = path
-
-			break
-		}
-		if index == len(ociMounts) {
-			return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint)
-		}
-	}
-	return nil
-}
-
 func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) {
 	// Disable Hooks since they have been handled on the host and there is
 	// no reason to send them to the agent. It would make no sense to try
@@ -1249,19 +1223,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
 	// Append container devices for block devices passed with --device.
 	ctrDevices = k.appendDevices(ctrDevices, c)
 
-	// Handle all the volumes that are block device files.
-	// Note this call modifies the list of container devices to make sure
-	// all hotplugged devices are unplugged, so this needs be done
-	// after devices passed with --device are handled.
-	volumeStorages, err := k.handleBlockVolumes(c)
+	// Block based volumes will require some adjustments in the OCI spec, and creation of
+	// storage objects to pass to the agent.
+	volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
-		return nil, err
-	}
-
 	ctrStorages = append(ctrStorages, volumeStorages...)
 
 	grpcSpec, err := grpc.OCItoGRPC(ociSpec)
@@ -1524,16 +1492,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D
 	vol.Options = []string{"bind"}
 	vol.MountPoint = m.Destination
 
+	// Assign the type from the mount, if it's specified (e.g. direct assigned volume)
+	if m.Type != "" {
+		vol.Fstype = m.Type
+		vol.Options = m.Options
+	}
+
 	return vol, nil
 }
 
-// handleBlockVolumes handles volumes that are block devices files
-// by passing the block devices as Storage to the agent.
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
+func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) {
+	var vol *grpc.Storage
+
+	id := m.BlockDeviceID
+	device := c.sandbox.devManager.GetDeviceByID(id)
+	if device == nil {
+		k.Logger().WithField("device", id).Error("failed to find device by id")
+		return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
+	}
+
+	var err error
+	switch device.DeviceType() {
+	case config.DeviceBlock:
+		vol, err = k.handleDeviceBlockVolume(c, m, device)
+	case config.VhostUserBlk:
+		vol, err = k.handleVhostUserBlkVolume(c, m, device)
+	default:
+		return nil, fmt.Errorf("Unknown device type")
+	}
+
+	return vol, err
+}
+
+// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the
+// given container and will update the OCI spec to utilize this mount point as the new source for the
+// container volume. The container mount structure is updated to store the guest destination mountpoint.
+func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) {
 
 	var volumeStorages []*grpc.Storage
 
-	for _, m := range c.mounts {
+	for i, m := range c.mounts {
 		id := m.BlockDeviceID
 
 		if len(id) == 0 {
@@ -1544,29 +1542,39 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
 		// device is detached with detachDevices() for a container.
 		c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
 
-		var vol *grpc.Storage
-
-		device := c.sandbox.devManager.GetDeviceByID(id)
-		if device == nil {
-			k.Logger().WithField("device", id).Error("failed to find device by id")
-			return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
-		}
-
-		var err error
-		switch device.DeviceType() {
-		case config.DeviceBlock:
-			vol, err = k.handleDeviceBlockVolume(c, m, device)
-		case config.VhostUserBlk:
-			vol, err = k.handleVhostUserBlkVolume(c, m, device)
-		default:
-			k.Logger().Error("Unknown device type")
-			continue
-		}
-
+		// Create Storage structure
+		vol, err := k.createBlkStorageObject(c, m)
 		if vol == nil || err != nil {
 			return nil, err
 		}
 
+		// Each device will be mounted at a unique location within the VM only once. Mounting
+		// to the container specific location is handled within the OCI spec. Let's ensure that
+		// the storage mount point is unique for each device. This is then utilized as the source
+		// in the OCI spec. If multiple containers mount the same block device, it's refcounted inside
+		// the guest by Kata agent.
+		filename := b64.StdEncoding.EncodeToString([]byte(vol.Source))
+		// Make the base64 encoding path safe.
+		filename = strings.ReplaceAll(filename, "/", "_")
+		path := filepath.Join(kataGuestSandboxStorageDir(), filename)
+
+		// Update applicable OCI mount source
+		for idx, ociMount := range spec.Mounts {
+			if ociMount.Destination != vol.MountPoint {
+				continue
+			}
+			k.Logger().WithFields(logrus.Fields{
+				"original-source": ociMount.Source,
+				"new-source":      path,
+			}).Debug("Replacing OCI mount source")
+			spec.Mounts[idx].Source = path
+			break
+		}
+
+		// Update storage mountpoint, and save guest device mount path to container mount struct:
+		vol.MountPoint = path
+		c.mounts[i].GuestDeviceMount = path
+
 		volumeStorages = append(volumeStorages, vol)
 	}
 
@@ -1955,6 +1963,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest))
 	}
+	k.reqHandlers[grpcVolumeStatsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.GetVolumeStats(ctx, req.(*grpc.VolumeStatsRequest))
+	}
+	k.reqHandlers[grpcResizeVolumeRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.ResizeVolume(ctx, req.(*grpc.ResizeVolumeRequest))
+	}
 }
 
 func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) {
@@ -2173,6 +2187,24 @@ func (k *kataAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsReq
 	return resp.(*grpc.Metrics), nil
 }
 
+func (k *kataAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) {
+	result, err := k.sendReq(ctx, &grpc.VolumeStatsRequest{VolumeGuestPath: volumeGuestPath})
+	if err != nil {
+		return nil, err
+	}
+
+	buf, err := json.Marshal(result.(*grpc.VolumeStatsResponse))
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+func (k *kataAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error {
+	_, err := k.sendReq(ctx, &grpc.ResizeVolumeRequest{VolumeGuestPath: volumeGuestPath, Size_: size})
+	return err
+
 func (k *kataAgent) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) {
 	r := &grpc.PullImageRequest{
 		Image: req.Image,

src/runtime/virtcontainers/kata_agent_test.go

@@ -405,24 +405,28 @@ func TestHandleBlockVolume(t *testing.T) {
 	containers[c.id].sandbox = &sandbox
 	containers[c.id].mounts = mounts
 
-	volumeStorages, err := k.handleBlockVolumes(c)
+	vStorage, err := k.createBlkStorageObject(c, vMount)
+	assert.Nil(t, err, "Error while handling block volumes")
+	bStorage, err := k.createBlkStorageObject(c, bMount)
+	assert.Nil(t, err, "Error while handling block volumes")
+	dStorage, err := k.createBlkStorageObject(c, dMount)
 	assert.Nil(t, err, "Error while handling block volumes")
 
-	vStorage := &pb.Storage{
+	vStorageExpected := &pb.Storage{
 		MountPoint: vDestination,
 		Fstype:     "bind",
 		Options:    []string{"bind"},
 		Driver:     kataBlkDevType,
 		Source:     vPCIPath.String(),
 	}
-	bStorage := &pb.Storage{
+	bStorageExpected := &pb.Storage{
 		MountPoint: bDestination,
 		Fstype:     "bind",
 		Options:    []string{"bind"},
 		Driver:     kataBlkDevType,
 		Source:     bPCIPath.String(),
 	}
-	dStorage := &pb.Storage{
+	dStorageExpected := &pb.Storage{
 		MountPoint: dDestination,
 		Fstype:     "ext4",
 		Options:    []string{"ro"},
@@ -430,9 +434,9 @@ func TestHandleBlockVolume(t *testing.T) {
 		Source:     dPCIPath.String(),
 	}
 
-	assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume")
-	assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume")
-	assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume")
+	assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume")
+	assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume")
+	assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume")
 }
 
 func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) {

src/runtime/virtcontainers/mock_agent.go

@@ -244,6 +244,14 @@ func (n *mockAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsReq
 	return nil, nil
 }
 
+func (n *mockAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) {
+	return nil, nil
+}
+
+func (n *mockAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error {
+	return nil
+}
+
 func (k *mockAgent) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) {
 	return nil, nil
 }

src/runtime/virtcontainers/mount.go

@@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) {
 		}, nil
 	}
 
-	// We get the mount point by recursively peforming stat on the path
+	// We get the mount point by recursively performing stat on the path
 	// The point where the device changes indicates the mountpoint
 	for {
 		if mountPoint == "/" {
@@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string
 
 // Mount describes a container mount.
 type Mount struct {
-	Source      string
+	// Source is the source of the mount.
+	Source string
+	// Destination is the destination of the mount (within the container).
 	Destination string
 
 	// Type specifies the type of filesystem to mount.
@@ -335,6 +337,11 @@ type Mount struct {
 	// HostPath used to store host side bind mount path
 	HostPath string
 
+	// GuestDeviceMount represents the path within the VM that the device
+	// is mounted. Only relevant for block devices. This is tracked in the event
+	// runtime wants to query the agent for mount stats.
+	GuestDeviceMount string
+
 	// BlockDeviceID represents block device that is attached to the
 	// VM in case this mount is a block device file or a directory
 	// backed by a block device.

src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go

@@ -2415,6 +2415,87 @@ func (m *Metrics) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_Metrics proto.InternalMessageInfo
 
+type VolumeStatsRequest struct {
+	// The volume path on the guest outside the container
+	VolumeGuestPath      string   `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *VolumeStatsRequest) Reset()      { *m = VolumeStatsRequest{} }
+func (*VolumeStatsRequest) ProtoMessage() {}
+func (*VolumeStatsRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_712ce9a559fda969, []int{56}
+}
+func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *VolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_VolumeStatsRequest.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *VolumeStatsRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VolumeStatsRequest.Merge(m, src)
+}
+func (m *VolumeStatsRequest) XXX_Size() int {
+	return m.Size()
+}
+func (m *VolumeStatsRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_VolumeStatsRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VolumeStatsRequest proto.InternalMessageInfo
+
+type ResizeVolumeRequest struct {
+	// Full VM guest path of the volume (outside the container)
+	VolumeGuestPath      string   `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"`
+	Size_                uint64   `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ResizeVolumeRequest) Reset()      { *m = ResizeVolumeRequest{} }
+func (*ResizeVolumeRequest) ProtoMessage() {}
+func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_712ce9a559fda969, []int{57}
+}
+func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ResizeVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ResizeVolumeRequest.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ResizeVolumeRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ResizeVolumeRequest.Merge(m, src)
+}
+func (m *ResizeVolumeRequest) XXX_Size() int {
+	return m.Size()
+}
+func (m *ResizeVolumeRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ResizeVolumeRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ResizeVolumeRequest proto.InternalMessageInfo
+
 func init() {
 	proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
 	proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
@@ -2474,6 +2555,8 @@ func init() {
 	proto.RegisterType((*AddSwapRequest)(nil), "grpc.AddSwapRequest")
 	proto.RegisterType((*GetMetricsRequest)(nil), "grpc.GetMetricsRequest")
 	proto.RegisterType((*Metrics)(nil), "grpc.Metrics")
+	proto.RegisterType((*VolumeStatsRequest)(nil), "grpc.VolumeStatsRequest")
+	proto.RegisterType((*ResizeVolumeRequest)(nil), "grpc.ResizeVolumeRequest")
 }
 
 func init() {
@@ -2481,194 +2564,198 @@ func init() {
 }
 
 var fileDescriptor_712ce9a559fda969 = []byte{
-	// 2978 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x23, 0xc7,
-	0xd1, 0xe6, 0x43, 0x22, 0x59, 0x7c, 0x89, 0x23, 0xad, 0x96, 0x4b, 0xdb, 0xfa, 0xd6, 0xb3, 0xf6,
-	0x7a, 0x6d, 0x7f, 0xa6, 0xec, 0xb5, 0xf1, 0xad, 0x1f, 0xf0, 0xb7, 0x58, 0x69, 0x65, 0x49, 0xb6,
-	0xe5, 0x65, 0x46, 0x16, 0x1c, 0x24, 0x48, 0x06, 0xc3, 0x99, 0x16, 0xd9, 0x16, 0x67, 0x7a, 0xdc,
-	0xd3, 0xa3, 0x15, 0x1d, 0x20, 0xc8, 0x29, 0xb9, 0xe5, 0x98, 0x5b, 0xfe, 0x40, 0x90, 0x5b, 0x80,
-	0x5c, 0x72, 0xcd, 0xc1, 0xc8, 0x29, 0xc7, 0x9c, 0x82, 0x78, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf,
-	0x79, 0xf0, 0x21, 0x27, 0x82, 0x80, 0x5c, 0x88, 0xae, 0xea, 0xea, 0x7a, 0x75, 0x57, 0x75, 0x55,
-	0x0f, 0x61, 0x30, 0xc2, 0x6c, 0x1c, 0x0f, 0xfb, 0x2e, 0xf1, 0xb7, 0xcf, 0x1c, 0xe6, 0xbc, 0xe9,
-	0x92, 0x80, 0x39, 0x38, 0x40, 0x34, 0x9a, 0x83, 0x23, 0xea, 0x6e, 0x4f, 0xf0, 0x30, 0xda, 0x0e,
-	0x29, 0x61, 0xc4, 0x25, 0x13, 0x35, 0x8a, 0xb6, 0x9d, 0x11, 0x0a, 0x58, 0x5f, 0x00, 0x46, 0x79,
-	0x44, 0x43, 0xb7, 0x57, 0x23, 0x2e, 0x96, 0x88, 0x5e, 0x9d, 0x4d, 0x43, 0x14, 0x29, 0xe0, 0xf9,
-	0x11, 0x21, 0xa3, 0x09, 0x92, 0x0b, 0x87, 0xf1, 0xe9, 0x36, 0xf2, 0x43, 0x36, 0x95, 0x93, 0xe6,
-	0x6f, 0x8b, 0xb0, 0xb9, 0x4b, 0x91, 0xc3, 0xd0, 0xae, 0x96, 0x6a, 0xa1, 0xaf, 0x63, 0x14, 0x31,
-	0xe3, 0x25, 0x68, 0x24, 0x9a, 0xd8, 0xd8, 0xeb, 0x16, 0x6e, 0x17, 0xee, 0xd5, 0xac, 0x7a, 0x82,
-	0x3b, 0xf4, 0x8c, 0x9b, 0x50, 0x41, 0x17, 0xc8, 0xe5, 0xb3, 0x45, 0x31, 0xbb, 0xca, 0xc1, 0x43,
-	0xcf, 0x78, 0x1b, 0xea, 0x11, 0xa3, 0x38, 0x18, 0xd9, 0x71, 0x84, 0x68, 0xb7, 0x74, 0xbb, 0x70,
-	0xaf, 0x7e, 0x7f, 0xad, 0xcf, 0xf5, 0xec, 0x1f, 0x8b, 0x89, 0x93, 0x08, 0x51, 0x0b, 0xa2, 0x64,
-	0x6c, 0xdc, 0x85, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x6e, 0xf9, 0x76, 0xe9, 0x5e, 0xfd, 0x7e,
-	0x43, 0x92, 0x3f, 0x16, 0x48, 0x4b, 0x4f, 0x1a, 0xaf, 0x41, 0x35, 0x62, 0x84, 0x3a, 0x23, 0x14,
-	0x75, 0x57, 0x04, 0x61, 0x53, 0xf3, 0x15, 0x58, 0x2b, 0x99, 0x36, 0x5e, 0x80, 0xd2, 0x93, 0xdd,
-	0xc3, 0xee, 0xaa, 0x90, 0x0e, 0x8a, 0x2a, 0x44, 0xae, 0xc5, 0xd1, 0xc6, 0x1d, 0x68, 0x46, 0x4e,
-	0xe0, 0x0d, 0xc9, 0x85, 0x1d, 0x62, 0x2f, 0x88, 0xba, 0x95, 0xdb, 0x85, 0x7b, 0x55, 0xab, 0xa1,
-	0x90, 0x03, 0x8e, 0x33, 0x3f, 0x80, 0x1b, 0xc7, 0xcc, 0xa1, 0xec, 0x0a, 0xde, 0x31, 0x4f, 0x60,
-	0xd3, 0x42, 0x3e, 0x39, 0xbf, 0x92, 0x6b, 0xbb, 0x50, 0x61, 0xd8, 0x47, 0x24, 0x66, 0xc2, 0xb5,
-	0x4d, 0x4b, 0x83, 0xe6, 0xef, 0x0b, 0x60, 0xec, 0x5d, 0x20, 0x77, 0x40, 0x89, 0x8b, 0xa2, 0xe8,
-	0xbf, 0xb4, 0x5d, 0xaf, 0x42, 0x25, 0x94, 0x0a, 0x74, 0xcb, 0x82, 0x5c, 0xed, 0x82, 0xd6, 0x4a,
-	0xcf, 0x9a, 0x5f, 0xc1, 0xc6, 0x31, 0x1e, 0x05, 0xce, 0xe4, 0x1a, 0xf5, 0xdd, 0x84, 0xd5, 0x48,
-	0xf0, 0x14, 0xaa, 0x36, 0x2d, 0x05, 0x99, 0x03, 0x30, 0xbe, 0x74, 0x30, 0xbb, 0x3e, 0x49, 0xe6,
-	0x9b, 0xb0, 0x9e, 0xe3, 0x18, 0x85, 0x24, 0x88, 0x90, 0x50, 0x80, 0x39, 0x2c, 0x8e, 0x04, 0xb3,
-	0x15, 0x4b, 0x41, 0x26, 0x81, 0xcd, 0x93, 0xd0, 0xbb, 0x62, 0x34, 0xdd, 0x87, 0x1a, 0x45, 0x11,
-	0x89, 0x29, 0x8f, 0x81, 0xa2, 0x70, 0xea, 0x86, 0x74, 0xea, 0x67, 0x38, 0x88, 0x2f, 0x2c, 0x3d,
-	0x67, 0xa5, 0x64, 0xea, 0x7c, 0xb2, 0xe8, 0x2a, 0xe7, 0xf3, 0x03, 0xb8, 0x31, 0x70, 0xe2, 0xe8,
-	0x2a, 0xba, 0x9a, 0x1f, 0xf2, 0xb3, 0x1d, 0xc5, 0xfe, 0x95, 0x16, 0xff, 0xae, 0x00, 0xd5, 0xdd,
-	0x30, 0x3e, 0x89, 0x9c, 0x11, 0x32, 0xfe, 0x07, 0xea, 0x8c, 0x30, 0x67, 0x62, 0xc7, 0x1c, 0x14,
-	0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x04, 0x8d, 0x10, 0x51, 0x37, 0x8c, 0x15, 0x45, 0xf1,
-	0x76, 0xe9, 0x5e, 0xd9, 0xaa, 0x4b, 0x9c, 0x24, 0xe9, 0xc3, 0xba, 0x98, 0xb3, 0x71, 0x60, 0x9f,
-	0x21, 0x1a, 0xa0, 0x89, 0x4f, 0x3c, 0x24, 0x0e, 0x47, 0xd9, 0xea, 0x88, 0xa9, 0xc3, 0xe0, 0xd3,
-	0x64, 0xc2, 0x78, 0x1d, 0x3a, 0x09, 0x3d, 0x3f, 0xf1, 0x82, 0xba, 0x2c, 0xa8, 0xdb, 0x8a, 0xfa,
-	0x44, 0xa1, 0xcd, 0x9f, 0x43, 0xeb, 0x8b, 0x31, 0x25, 0x8c, 0x4d, 0x70, 0x30, 0x7a, 0xec, 0x30,
-	0x87, 0x87, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, 0x06, 0x74, 0x98, 0xa4,
-	0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x2d, 0x99, 0x18, 0x28, 0xe2, 0x57, 0xa0, 0x95, 0x12,
-	0xf3, 0xe0, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x17, 0xd8, 0x47, 0xe6, 0xb9, 0xf0, 0x95, 0xd8, 0x64,
-	0xe3, 0x0d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x13, 0xd2, 0x92, 0x27, 0x44, 0xbb, 0xd3, 0xaa, 0x26,
-	0x4e, 0xf9, 0x08, 0xda, 0x2c, 0x51, 0xdc, 0xf6, 0x1c, 0xe6, 0xe4, 0x0f, 0x55, 0xde, 0x2a, 0xab,
-	0xc5, 0x72, 0xb0, 0xf9, 0x21, 0xd4, 0x06, 0xd8, 0x8b, 0xa4, 0xe0, 0x2e, 0x54, 0xdc, 0x98, 0x52,
-	0x14, 0x30, 0x6d, 0xb2, 0x02, 0x8d, 0x0d, 0x58, 0x99, 0x60, 0x1f, 0x33, 0x65, 0xa6, 0x04, 0x4c,
-	0x02, 0x70, 0x84, 0x7c, 0x42, 0xa7, 0xc2, 0x61, 0x1b, 0xb0, 0x92, 0xdd, 0x5c, 0x09, 0x18, 0xcf,
-	0x43, 0xcd, 0x77, 0x2e, 0x92, 0x4d, 0xe5, 0x33, 0x55, 0xdf, 0xb9, 0x90, 0xca, 0x77, 0xa1, 0x72,
-	0xea, 0xe0, 0x89, 0x1b, 0x30, 0xe5, 0x15, 0x0d, 0xa6, 0x02, 0xcb, 0x59, 0x81, 0x7f, 0x2e, 0x42,
-	0x5d, 0x4a, 0x94, 0x0a, 0x6f, 0xc0, 0x8a, 0xeb, 0xb8, 0xe3, 0x44, 0xa4, 0x00, 0x8c, 0xbb, 0x5a,
-	0x91, 0x62, 0x36, 0xc3, 0xa5, 0x9a, 0x6a, 0xd5, 0xb6, 0x01, 0xa2, 0xa7, 0x4e, 0xa8, 0x74, 0x2b,
-	0x2d, 0x21, 0xae, 0x71, 0x1a, 0xa9, 0xee, 0x3b, 0xd0, 0x90, 0xe7, 0x4e, 0x2d, 0x29, 0x2f, 0x59,
-	0x52, 0x97, 0x54, 0x72, 0xd1, 0x1d, 0x68, 0xc6, 0x11, 0xb2, 0xc7, 0x18, 0x51, 0x87, 0xba, 0xe3,
-	0x69, 0x77, 0x45, 0x5e, 0x40, 0x71, 0x84, 0x0e, 0x34, 0xce, 0xb8, 0x0f, 0x2b, 0x3c, 0xb7, 0x44,
-	0xdd, 0x55, 0x71, 0xd7, 0xbd, 0x90, 0x65, 0x29, 0x4c, 0xed, 0x8b, 0xdf, 0xbd, 0x80, 0xd1, 0xa9,
-	0x25, 0x49, 0x7b, 0xef, 0x01, 0xa4, 0x48, 0x63, 0x0d, 0x4a, 0x67, 0x68, 0xaa, 0xe2, 0x90, 0x0f,
-	0xb9, 0x73, 0xce, 0x9d, 0x49, 0xac, 0xbd, 0x2e, 0x81, 0x0f, 0x8a, 0xef, 0x15, 0x4c, 0x17, 0xda,
-	0x3b, 0x93, 0x33, 0x4c, 0x32, 0xcb, 0x37, 0x60, 0xc5, 0x77, 0xbe, 0x22, 0x54, 0x7b, 0x52, 0x00,
-	0x02, 0x8b, 0x03, 0x42, 0x35, 0x0b, 0x01, 0x18, 0x2d, 0x28, 0x92, 0x50, 0xf8, 0xab, 0x66, 0x15,
-	0x49, 0x98, 0x0a, 0x2a, 0x67, 0x04, 0x99, 0x7f, 0x2f, 0x03, 0xa4, 0x52, 0x0c, 0x0b, 0x7a, 0x98,
-	0xd8, 0x11, 0xa2, 0xfc, 0x7e, 0xb7, 0x87, 0x53, 0x86, 0x22, 0x9b, 0x22, 0x37, 0xa6, 0x11, 0x3e,
-	0xe7, 0xfb, 0xc7, 0xcd, 0xbe, 0x21, 0xcd, 0x9e, 0xd1, 0xcd, 0xba, 0x89, 0xc9, 0xb1, 0x5c, 0xb7,
-	0xc3, 0x97, 0x59, 0x7a, 0x95, 0x71, 0x08, 0x37, 0x52, 0x9e, 0x5e, 0x86, 0x5d, 0xf1, 0x32, 0x76,
-	0xeb, 0x09, 0x3b, 0x2f, 0x65, 0xb5, 0x07, 0xeb, 0x98, 0xd8, 0x5f, 0xc7, 0x28, 0xce, 0x31, 0x2a,
-	0x5d, 0xc6, 0xa8, 0x83, 0xc9, 0x0f, 0xc4, 0x82, 0x94, 0xcd, 0x00, 0x6e, 0x65, 0xac, 0xe4, 0xe1,
-	0x9e, 0x61, 0x56, 0xbe, 0x8c, 0xd9, 0x66, 0xa2, 0x15, 0xcf, 0x07, 0x29, 0xc7, 0x4f, 0x60, 0x13,
-	0x13, 0xfb, 0xa9, 0x83, 0xd9, 0x2c, 0xbb, 0x95, 0xef, 0x31, 0x92, 0xdf, 0x68, 0x79, 0x5e, 0xd2,
-	0x48, 0x1f, 0xd1, 0x51, 0xce, 0xc8, 0xd5, 0xef, 0x31, 0xf2, 0x48, 0x2c, 0x48, 0xd9, 0x3c, 0x82,
-	0x0e, 0x26, 0xb3, 0xda, 0x54, 0x2e, 0x63, 0xd2, 0xc6, 0x24, 0xaf, 0xc9, 0x0e, 0x74, 0x22, 0xe4,
-	0x32, 0x42, 0xb3, 0x87, 0xa0, 0x7a, 0x19, 0x8b, 0x35, 0x45, 0x9f, 0xf0, 0x30, 0x7f, 0x0c, 0x8d,
-	0x83, 0x78, 0x84, 0xd8, 0x64, 0x98, 0x24, 0x83, 0x6b, 0xcb, 0x3f, 0xe6, 0x3f, 0x8b, 0x50, 0xdf,
-	0x1d, 0x51, 0x12, 0x87, 0xb9, 0x9c, 0x2c, 0x83, 0x74, 0x36, 0x27, 0x0b, 0x12, 0x91, 0x93, 0x25,
-	0xf1, 0xbb, 0xd0, 0xf0, 0x45, 0xe8, 0x2a, 0x7a, 0x99, 0x87, 0x3a, 0x73, 0x41, 0x6d, 0xd5, 0xfd,
-	0x4c, 0x32, 0xeb, 0x03, 0x84, 0xd8, 0x8b, 0xd4, 0x1a, 0x99, 0x8e, 0xda, 0xaa, 0xdc, 0xd2, 0x29,
-	0xda, 0xaa, 0x85, 0x49, 0xb6, 0x7e, 0x1b, 0xea, 0x43, 0xee, 0x24, 0xb5, 0x20, 0x97, 0x8c, 0x52,
-	0xef, 0x59, 0x30, 0x4c, 0x83, 0xf0, 0x00, 0x9a, 0x63, 0xe9, 0x32, 0xb5, 0x48, 0x9e, 0xa1, 0x3b,
-	0xca, 0x92, 0xd4, 0xde, 0x7e, 0xd6, 0xb3, 0x72, 0x03, 0x1a, 0xe3, 0x0c, 0xaa, 0x77, 0x0c, 0x9d,
-	0x39, 0x92, 0x05, 0x39, 0xe8, 0x5e, 0x36, 0x07, 0xd5, 0xef, 0x1b, 0x52, 0x50, 0x76, 0x65, 0x36,
-	0x2f, 0xfd, 0xba, 0x08, 0x8d, 0xcf, 0x11, 0x7b, 0x4a, 0xe8, 0x99, 0xd4, 0xd7, 0x80, 0x72, 0xe0,
-	0xf8, 0x48, 0x71, 0x14, 0x63, 0xe3, 0x16, 0x54, 0xe9, 0x85, 0x4c, 0x20, 0x6a, 0x3f, 0x2b, 0xf4,
-	0x42, 0x24, 0x06, 0xe3, 0x45, 0x00, 0x7a, 0x61, 0x87, 0x8e, 0x7b, 0x86, 0x94, 0x07, 0xcb, 0x56,
-	0x8d, 0x5e, 0x0c, 0x24, 0x82, 0x1f, 0x05, 0x7a, 0x61, 0x23, 0x4a, 0x09, 0x8d, 0x54, 0xae, 0xaa,
-	0xd2, 0x8b, 0x3d, 0x01, 0xab, 0xb5, 0x1e, 0x25, 0x61, 0x88, 0x3c, 0x91, 0xa3, 0xc5, 0xda, 0xc7,
-	0x12, 0xc1, 0xa5, 0x32, 0x2d, 0x75, 0x55, 0x4a, 0x65, 0xa9, 0x54, 0x96, 0x4a, 0xad, 0xc8, 0x95,
-	0x2c, 0x2b, 0x95, 0x25, 0x52, 0xab, 0x52, 0x2a, 0xcb, 0x48, 0x65, 0xa9, 0xd4, 0x9a, 0x5e, 0xab,
-	0xa4, 0x9a, 0xbf, 0x2a, 0xc0, 0xe6, 0x6c, 0xe1, 0xa7, 0x6a, 0xd3, 0x77, 0xa1, 0xe1, 0x8a, 0xfd,
-	0xca, 0x9d, 0xc9, 0xce, 0xdc, 0x4e, 0x5a, 0x75, 0x37, 0x73, 0x8c, 0x1f, 0x40, 0x33, 0x90, 0x0e,
-	0x4e, 0x8e, 0x66, 0x29, 0xdd, 0x97, 0xac, 0xef, 0xad, 0x46, 0x90, 0x81, 0x4c, 0x0f, 0x8c, 0x2f,
-	0x29, 0x66, 0xe8, 0x98, 0x51, 0xe4, 0xf8, 0xd7, 0x51, 0xdd, 0x1b, 0x50, 0x16, 0xd5, 0x0a, 0xdf,
-	0xa6, 0x86, 0x25, 0xc6, 0xe6, 0xab, 0xb0, 0x9e, 0x93, 0xa2, 0x6c, 0x5d, 0x83, 0xd2, 0x04, 0x05,
-	0x82, 0x7b, 0xd3, 0xe2, 0x43, 0xd3, 0x81, 0x8e, 0x85, 0x1c, 0xef, 0xfa, 0xb4, 0x51, 0x22, 0x4a,
-	0xa9, 0x88, 0x7b, 0x60, 0x64, 0x45, 0x28, 0x55, 0xb4, 0xd6, 0x85, 0x8c, 0xd6, 0x4f, 0xa0, 0xb3,
-	0x3b, 0x21, 0x11, 0x3a, 0x66, 0x1e, 0x0e, 0xae, 0xa3, 0x1d, 0xf9, 0x19, 0xac, 0x7f, 0xc1, 0xa6,
-	0x5f, 0x72, 0x66, 0x11, 0xfe, 0x06, 0x5d, 0x93, 0x7d, 0x94, 0x3c, 0xd5, 0xf6, 0x51, 0xf2, 0x94,
-	0x37, 0x37, 0x2e, 0x99, 0xc4, 0x7e, 0x20, 0x42, 0xa1, 0x69, 0x29, 0xc8, 0xdc, 0x81, 0x86, 0xac,
-	0xa1, 0x8f, 0x88, 0x17, 0x4f, 0xd0, 0xc2, 0x18, 0xdc, 0x02, 0x08, 0x1d, 0xea, 0xf8, 0x88, 0x21,
-	0x2a, 0xcf, 0x50, 0xcd, 0xca, 0x60, 0xcc, 0xdf, 0x14, 0x61, 0x43, 0xbe, 0x37, 0x1c, 0xcb, 0x36,
-	0x5b, 0x9b, 0xd0, 0x83, 0xea, 0x98, 0x44, 0x2c, 0xc3, 0x30, 0x81, 0xb9, 0x8a, 0xbc, 0x3f, 0x97,
-	0xdc, 0xf8, 0x30, 0xf7, 0x08, 0x50, 0xba, 0xfc, 0x11, 0x60, 0xae, 0xcd, 0x2f, 0xcf, 0xb7, 0xf9,
-	0x3c, 0xda, 0x34, 0x11, 0x96, 0x31, 0x5e, 0xb3, 0x6a, 0x0a, 0x73, 0xe8, 0x19, 0x77, 0xa1, 0x3d,
-	0xe2, 0x5a, 0xda, 0x63, 0x42, 0xce, 0xec, 0xd0, 0x61, 0x63, 0x11, 0xea, 0x35, 0xab, 0x29, 0xd0,
-	0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0xde, 0x87, 0x96, 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48,
-	0x5d, 0x7e, 0x2a, 0x8a, 0xb2, 0xde, 0xb3, 0x9a, 0x67, 0x19, 0x28, 0x32, 0x6f, 0xc2, 0x8d, 0xc7,
-	0x28, 0x62, 0x94, 0x4c, 0xf3, 0x8e, 0x31, 0xff, 0x1f, 0xe0, 0x30, 0x60, 0x88, 0x9e, 0x3a, 0x2e,
-	0x8a, 0x8c, 0xb7, 0xb2, 0x90, 0x2a, 0x8e, 0xd6, 0xfa, 0xf2, 0xb9, 0x27, 0x99, 0xb0, 0x32, 0x34,
-	0x66, 0x1f, 0x56, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0xb2, 0x1e, 0xa9, 0x75, 0x0d, 0xb5, 0x4e, 0x20,
-	0x2d, 0x35, 0x67, 0x1e, 0xe8, 0x16, 0x36, 0x65, 0xa7, 0xb6, 0xa8, 0x0f, 0x35, 0xac, 0x71, 0x2a,
-	0xab, 0xcc, 0x8b, 0x4e, 0x49, 0xcc, 0x0f, 0x61, 0x5d, 0x72, 0x92, 0x9c, 0x35, 0x9b, 0x97, 0x61,
-	0x95, 0x6a, 0x35, 0x0a, 0xe9, 0x3b, 0x8f, 0x22, 0x52, 0x73, 0xdc, 0x1f, 0x9f, 0xe1, 0x88, 0xa5,
-	0x86, 0x68, 0x7f, 0xac, 0x43, 0x87, 0x4f, 0xe4, 0x78, 0x9a, 0x1f, 0x43, 0xe3, 0x91, 0x35, 0xf8,
-	0x1c, 0xe1, 0xd1, 0x78, 0xc8, 0xb3, 0xe7, 0xff, 0xe5, 0x61, 0x65, 0xb0, 0xa1, 0xb4, 0xcd, 0x4c,
-	0x59, 0x39, 0x3a, 0xf3, 0x13, 0xd8, 0x7c, 0xe4, 0x79, 0x59, 0x94, 0xd6, 0xfa, 0x2d, 0xa8, 0x05,
-	0x19, 0x76, 0x99, 0x3b, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x3f, 0x81, 0xf5, 0x27, 0xc1, 0x04, 0x07,
-	0x68, 0x77, 0x70, 0x72, 0x84, 0x92, 0x5c, 0x64, 0x40, 0x99, 0xd7, 0x6c, 0x82, 0x47, 0xd5, 0x12,
-	0x63, 0x1e, 0x9c, 0xc1, 0xd0, 0x76, 0xc3, 0x38, 0x52, 0x8f, 0x3d, 0xab, 0xc1, 0x70, 0x37, 0x8c,
-	0x23, 0x7e, 0xb9, 0xf0, 0xe2, 0x82, 0x04, 0x93, 0xa9, 0x88, 0xd0, 0xaa, 0x55, 0x71, 0xc3, 0xf8,
-	0x49, 0x30, 0x99, 0x9a, 0xff, 0x2b, 0x3a, 0x70, 0x84, 0x3c, 0xcb, 0x09, 0x3c, 0xe2, 0x3f, 0x46,
-	0xe7, 0x19, 0x09, 0x49, 0xb7, 0xa7, 0x33, 0xd1, 0xb7, 0x05, 0x68, 0x3c, 0x1a, 0xa1, 0x80, 0x3d,
-	0x46, 0xcc, 0xc1, 0x13, 0xd1, 0xd1, 0x9d, 0x23, 0x1a, 0x61, 0x12, 0xa8, 0x70, 0xd3, 0x20, 0x6f,
-	0xc8, 0x71, 0x80, 0x99, 0xed, 0x39, 0xc8, 0x27, 0x81, 0xe0, 0x52, 0xb5, 0x80, 0xa3, 0x1e, 0x0b,
-	0x8c, 0xf1, 0x2a, 0xb4, 0xe5, 0x63, 0x9c, 0x3d, 0x76, 0x02, 0x6f, 0xc2, 0x03, 0xbd, 0x24, 0x42,
-	0xb3, 0x25, 0xd1, 0x07, 0x0a, 0x6b, 0xbc, 0x06, 0x6b, 0x2a, 0x0c, 0x53, 0xca, 0xb2, 0xa0, 0x6c,
-	0x2b, 0x7c, 0x8e, 0x34, 0x0e, 0x43, 0x42, 0x59, 0x64, 0x47, 0xc8, 0x75, 0x89, 0x1f, 0xaa, 0x76,
-	0xa8, 0xad, 0xf1, 0xc7, 0x12, 0x6d, 0x8e, 0x60, 0x7d, 0x9f, 0xdb, 0xa9, 0x2c, 0x49, 0x8f, 0x55,
-	0xcb, 0x47, 0xbe, 0x3d, 0x9c, 0x10, 0xf7, 0xcc, 0xe6, 0xc9, 0x51, 0x79, 0x98, 0x17, 0x5c, 0x3b,
-	0x1c, 0x79, 0x8c, 0xbf, 0x11, 0x9d, 0x3f, 0xa7, 0x1a, 0x13, 0x16, 0x4e, 0xe2, 0x91, 0x1d, 0x52,
-	0x32, 0x44, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, 0xa7, 0x02, 0x6c,
-	0xe4, 0x25, 0xa9, 0x54, 0xbf, 0x0d, 0x1b, 0x79, 0x51, 0xea, 0xfa, 0x97, 0xe5, 0x65, 0x27, 0x2b,
-	0x50, 0x16, 0x02, 0x0f, 0xa0, 0x29, 0xde, 0x6b, 0x6d, 0x4f, 0x72, 0xca, 0x17, 0x3d, 0xd9, 0x7d,
-	0xb1, 0x1a, 0x4e, 0x76, 0x97, 0xde, 0x87, 0x5b, 0xca, 0x7c, 0x7b, 0x5e, 0x6d, 0x79, 0x20, 0x36,
-	0x15, 0xc1, 0xd1, 0x8c, 0xf6, 0x9f, 0x41, 0x37, 0x45, 0xed, 0x4c, 0x05, 0x32, 0x3d, 0xcc, 0xeb,
-	0x33, 0xc6, 0x3e, 0xf2, 0x3c, 0x2a, 0xa2, 0xa4, 0x6c, 0x2d, 0x9a, 0x32, 0x1f, 0xc2, 0xcd, 0x63,
-	0xc4, 0xa4, 0x37, 0x1c, 0xa6, 0x3a, 0x11, 0xc9, 0x6c, 0x0d, 0x4a, 0xc7, 0xc8, 0x15, 0xc6, 0x97,
-	0x2c, 0x3e, 0xe4, 0x07, 0xf0, 0x24, 0x42, 0xae, 0xb0, 0xb2, 0x64, 0x89, 0xb1, 0xf9, 0x87, 0x02,
-	0x54, 0x54, 0x72, 0xe6, 0x17, 0x8c, 0x47, 0xf1, 0x39, 0xa2, 0xea, 0xe8, 0x29, 0xc8, 0x78, 0x05,
-	0x5a, 0x72, 0x64, 0x93, 0x90, 0x61, 0x92, 0xa4, 0xfc, 0xa6, 0xc4, 0x3e, 0x91, 0x48, 0xf1, 0xf8,
-	0x26, 0x9e, 0xbf, 0x54, 0xa7, 0xa9, 0x20, 0x8e, 0x3f, 0x8d, 0x78, 0x84, 0x8b, 0x14, 0x5f, 0xb3,
-	0x14, 0xc4, 0x8f, 0xba, 0xe6, 0xb7, 0x22, 0xf8, 0x69, 0x90, 0x1f, 0x75, 0x9f, 0xc4, 0x01, 0xb3,
-	0x43, 0x82, 0x03, 0xa6, 0x72, 0x3a, 0x08, 0xd4, 0x80, 0x63, 0xcc, 0x5f, 0x16, 0x60, 0x55, 0x3e,
-	0x40, 0xf3, 0xde, 0x36, 0xb9, 0x59, 0x8b, 0x58, 0x54, 0x29, 0x42, 0x96, 0xbc, 0x4d, 0xc5, 0x98,
-	0xc7, 0xf1, 0xb9, 0x2f, 0xef, 0x07, 0xa5, 0xda, 0xb9, 0x2f, 0x2e, 0x86, 0x57, 0xa0, 0x95, 0x5e,
-	0xd0, 0x62, 0x5e, 0xaa, 0xd8, 0x4c, 0xb0, 0x82, 0x6c, 0xa9, 0xa6, 0xe6, 0x0f, 0x79, 0x4b, 0x9f,
-	0x3c, 0xbe, 0xae, 0x41, 0x29, 0x4e, 0x94, 0xe1, 0x43, 0x8e, 0x19, 0x25, 0x57, 0x3b, 0x1f, 0x1a,
-	0x77, 0xa1, 0xe5, 0x78, 0x1e, 0xe6, 0xcb, 0x9d, 0xc9, 0x3e, 0xf6, 0x92, 0x20, 0xcd, 0x63, 0xcd,
-	0xbf, 0x14, 0xa0, 0xbd, 0x4b, 0xc2, 0xe9, 0xc7, 0x78, 0x82, 0x32, 0x19, 0x44, 0x28, 0xa9, 0x6e,
-	0x76, 0x3e, 0xe6, 0xd5, 0xea, 0x29, 0x9e, 0x20, 0x19, 0x5a, 0x72, 0x67, 0xab, 0x1c, 0x21, 0xc2,
-	0x4a, 0x4f, 0x26, 0xcf, 0x6e, 0x4d, 0x39, 0x79, 0x44, 0x3c, 0x51, 0x97, 0x7b, 0x98, 0xda, 0xc9,
-	0x23, 0x5b, 0xd3, 0xaa, 0x78, 0x98, 0x8a, 0x29, 0x65, 0xc8, 0x8a, 0x78, 0x44, 0xcd, 0x1a, 0xb2,
-	0x2a, 0x31, 0xdc, 0x90, 0x4d, 0x58, 0x25, 0xa7, 0xa7, 0x11, 0x62, 0xa2, 0x82, 0x2e, 0x59, 0x0a,
-	0x4a, 0xd2, 0x5c, 0x35, 0x93, 0xe6, 0x36, 0xc0, 0xd8, 0x47, 0xec, 0xc9, 0x93, 0xa3, 0xbd, 0x73,
-	0x14, 0x30, 0x7d, 0x3b, 0xbc, 0x09, 0x55, 0x8d, 0xfa, 0x77, 0x9e, 0x27, 0x5f, 0x87, 0xd6, 0x23,
-	0xcf, 0x3b, 0x7e, 0xea, 0x84, 0xda, 0x1f, 0x5d, 0xa8, 0x0c, 0x76, 0x0f, 0x07, 0xd2, 0x25, 0x25,
-	0x6e, 0x80, 0x02, 0xf9, 0x6d, 0xb4, 0x8f, 0xd8, 0x11, 0x62, 0x14, 0xbb, 0xc9, 0x6d, 0x74, 0x07,
-	0x2a, 0x0a, 0xc3, 0x57, 0xfa, 0x72, 0xa8, 0xd3, 0xac, 0x02, 0xef, 0xff, 0x71, 0x4d, 0x65, 0x64,
-	0xd5, 0xdc, 0x1b, 0xfb, 0xd0, 0x9e, 0xf9, 0x12, 0x63, 0xa8, 0xd7, 0x9e, 0xc5, 0x1f, 0x68, 0x7a,
-	0x9b, 0x7d, 0xf9, 0x65, 0xa7, 0xaf, 0xbf, 0xec, 0xf4, 0xf7, 0xfc, 0x90, 0x4d, 0x8d, 0x3d, 0x68,
-	0xe5, 0xbf, 0x59, 0x18, 0xcf, 0xeb, 0xe2, 0x68, 0xc1, 0x97, 0x8c, 0xa5, 0x6c, 0xf6, 0xa1, 0x3d,
-	0xf3, 0xf9, 0x42, 0xeb, 0xb3, 0xf8, 0xab, 0xc6, 0x52, 0x46, 0x0f, 0xa1, 0x9e, 0xf9, 0x5e, 0x61,
-	0x74, 0x25, 0x93, 0xf9, 0x4f, 0x18, 0x4b, 0x19, 0xec, 0x42, 0x33, 0xf7, 0x09, 0xc1, 0xe8, 0x29,
-	0x7b, 0x16, 0x7c, 0x57, 0x58, 0xca, 0x64, 0x07, 0xea, 0x99, 0x97, 0x7c, 0xad, 0xc5, 0xfc, 0xe7,
-	0x82, 0xde, 0xad, 0x05, 0x33, 0x2a, 0xf1, 0xef, 0x43, 0x7b, 0xe6, 0x79, 0x5f, 0xbb, 0x64, 0xf1,
-	0xab, 0xff, 0x52, 0x65, 0x3e, 0x15, 0x5b, 0x94, 0xe9, 0xde, 0x32, 0x5b, 0x34, 0xff, 0x98, 0xdf,
-	0x7b, 0x61, 0xf1, 0xa4, 0xd2, 0x6a, 0x0f, 0x5a, 0xf9, 0x77, 0x7c, 0xcd, 0x6c, 0xe1, 0xeb, 0xfe,
-	0xe5, 0xfb, 0x9d, 0x7b, 0xd2, 0x4f, 0xf7, 0x7b, 0xd1, 0x4b, 0xff, 0x52, 0x46, 0x8f, 0x00, 0x54,
-	0xaf, 0xe6, 0xe1, 0x20, 0x71, 0xf4, 0x5c, 0x8f, 0x98, 0x38, 0x7a, 0x41, 0x5f, 0xf7, 0x10, 0x40,
-	0xb6, 0x58, 0x1e, 0x89, 0x99, 0x71, 0x53, 0xab, 0x31, 0xd3, 0xd7, 0xf5, 0xba, 0xf3, 0x13, 0x73,
-	0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x11, 0x40, 0xda, 0xba, 0x69, 0x06, 0x73, 0xcd, 0xdc, 0x25,
-	0x3e, 0x68, 0x64, 0x1b, 0x35, 0x43, 0xd9, 0xba, 0xa0, 0x79, 0xbb, 0x84, 0x45, 0x7b, 0xa6, 0x10,
-	0xcf, 0x1f, 0xb6, 0xd9, 0xfa, 0xbc, 0x37, 0x57, 0x8c, 0x1b, 0x0f, 0xa0, 0x91, 0xad, 0xc0, 0xb5,
-	0x16, 0x0b, 0xaa, 0xf2, 0x5e, 0xae, 0x0a, 0x37, 0x1e, 0x42, 0x2b, 0x5f, 0x7d, 0xeb, 0x23, 0xb5,
-	0xb0, 0x26, 0xef, 0xa9, 0xb7, 0xa5, 0x0c, 0xf9, 0x3b, 0x00, 0x69, 0x95, 0xae, 0xdd, 0x37, 0x57,
-	0xb7, 0xcf, 0x48, 0xdd, 0x87, 0xf6, 0x4c, 0xf5, 0xad, 0x2d, 0x5e, 0x5c, 0x94, 0x2f, 0x75, 0xdd,
-	0xbb, 0x00, 0x69, 0x56, 0xd6, 0xd2, 0xe7, 0xf2, 0x74, 0xaf, 0xa9, 0xdf, 0xdd, 0x24, 0xdd, 0x2e,
-	0x34, 0x73, 0xad, 0xa9, 0x4e, 0x33, 0x8b, 0xfa, 0xd5, 0xcb, 0x92, 0x6f, 0xbe, 0x8f, 0xd3, 0x9e,
-	0x5b, 0xd8, 0xdd, 0x5d, 0x76, 0x7e, 0xb2, 0xcd, 0x83, 0xde, 0xb9, 0x05, 0x0d, 0xc5, 0xf7, 0xc4,
-	0x73, 0xb6, 0x41, 0xc8, 0xc4, 0xf3, 0x82, 0xbe, 0x61, 0x29, 0xa3, 0x03, 0x68, 0xef, 0xeb, 0xda,
-	0x4f, 0xd5, 0xa5, 0x4a, 0x9d, 0x05, 0x75, 0x78, 0xaf, 0xb7, 0x68, 0x4a, 0x05, 0xd5, 0xa7, 0xd0,
-	0x99, 0xab, 0x49, 0x8d, 0xad, 0xe4, 0xf5, 0x73, 0x61, 0xb1, 0xba, 0x54, 0xad, 0x43, 0x58, 0x9b,
-	0x2d, 0x49, 0x8d, 0x17, 0x55, 0xa2, 0x5c, 0x5c, 0xaa, 0x2e, 0x65, 0xf5, 0x3e, 0x54, 0x75, 0x09,
-	0x64, 0xa8, 0x57, 0xe6, 0x99, 0x92, 0x68, 0xe9, 0xd2, 0x07, 0x50, 0xcf, 0x54, 0x1c, 0x3a, 0xdb,
-	0xcd, 0x17, 0x21, 0x3d, 0xf5, 0x28, 0x9c, 0x50, 0x3e, 0x80, 0x8a, 0xaa, 0x32, 0x8c, 0x8d, 0xe4,
-	0x90, 0x67, 0x8a, 0x8e, 0x65, 0x12, 0x77, 0x2e, 0xbe, 0xfd, 0x6e, 0xeb, 0xb9, 0xbf, 0x7d, 0xb7,
-	0xf5, 0xdc, 0x2f, 0x9e, 0x6d, 0x15, 0xbe, 0x7d, 0xb6, 0x55, 0xf8, 0xeb, 0xb3, 0xad, 0xc2, 0x3f,
-	0x9e, 0x6d, 0x15, 0x7e, 0xf4, 0xd3, 0xff, 0xf0, 0x9f, 0x25, 0x34, 0x0e, 0x18, 0xf6, 0xd1, 0xf6,
-	0x39, 0xa6, 0x2c, 0x33, 0x15, 0x9e, 0x8d, 0xe4, 0xdf, 0x4b, 0x32, 0xff, 0x3a, 0xe1, 0x0a, 0x0e,
-	0x57, 0x05, 0xfc, 0xce, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x52, 0x45, 0x13, 0x9c, 0xc2, 0x22,
-	0x00, 0x00,
+	// 3055 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47,
+	0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed,
+	0xf5, 0xda, 0xc6, 0x92, 0xbd, 0x76, 0xb0, 0x7e, 0x84, 0x59, 0x24, 0xad, 0x2c, 0xc9, 0xb6, 0xbc,
+	0x43, 0xcb, 0xc2, 0x04, 0x04, 0x74, 0xb4, 0xba, 0x6b, 0x47, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab,
+	0xb5, 0x92, 0x89, 0x20, 0x38, 0xc1, 0x8d, 0x23, 0x37, 0x7e, 0x80, 0xe0, 0xc6, 0x91, 0x0b, 0x07,
+	0x0e, 0x0e, 0x4e, 0x1c, 0x39, 0x11, 0x78, 0x3f, 0x81, 0x2f, 0x20, 0xea, 0xd5, 0x5d, 0x3d, 0x0f,
+	0x19, 0x14, 0x1b, 0xc1, 0x65, 0xa2, 0x33, 0x2b, 0x2b, 0x5f, 0x55, 0x99, 0x95, 0x59, 0x35, 0x30,
+	0x1c, 0x61, 0x76, 0x92, 0x1e, 0xaf, 0xfb, 0x24, 0xdc, 0x38, 0xf5, 0x98, 0xf7, 0xba, 0x4f, 0x22,
+	0xe6, 0xe1, 0x08, 0xd1, 0x64, 0x0a, 0x4e, 0xa8, 0xbf, 0x31, 0xc6, 0xc7, 0xc9, 0x46, 0x4c, 0x09,
+	0x23, 0x3e, 0x19, 0xab, 0xaf, 0x64, 0xc3, 0x1b, 0xa1, 0x88, 0xad, 0x0b, 0xc0, 0xaa, 0x8e, 0x68,
+	0xec, 0x0f, 0x1a, 0xc4, 0xc7, 0x12, 0x31, 0x68, 0xf8, 0x89, 0xfe, 0x6c, 0xb2, 0x8b, 0x18, 0x25,
+	0x0a, 0x78, 0x76, 0x44, 0xc8, 0x68, 0x8c, 0x24, 0x8f, 0xe3, 0xf4, 0xd1, 0x06, 0x0a, 0x63, 0x76,
+	0x21, 0x07, 0xed, 0xdf, 0x97, 0x61, 0x75, 0x9b, 0x22, 0x8f, 0xa1, 0x6d, 0xad, 0x80, 0x83, 0xbe,
+	0x4c, 0x51, 0xc2, 0xac, 0x17, 0xa0, 0x95, 0x29, 0xe5, 0xe2, 0xa0, 0x5f, 0xba, 0x59, 0xba, 0xd3,
+	0x70, 0x9a, 0x19, 0x6e, 0x3f, 0xb0, 0xae, 0x43, 0x0d, 0x9d, 0x23, 0x9f, 0x8f, 0x96, 0xc5, 0xe8,
+	0x22, 0x07, 0xf7, 0x03, 0xeb, 0x4d, 0x68, 0x26, 0x8c, 0xe2, 0x68, 0xe4, 0xa6, 0x09, 0xa2, 0xfd,
+	0xca, 0xcd, 0xd2, 0x9d, 0xe6, 0xdd, 0xa5, 0x75, 0xae, 0xf2, 0xfa, 0xa1, 0x18, 0x38, 0x4a, 0x10,
+	0x75, 0x20, 0xc9, 0xbe, 0xad, 0xdb, 0x50, 0x0b, 0xd0, 0x19, 0xf6, 0x51, 0xd2, 0xaf, 0xde, 0xac,
+	0xdc, 0x69, 0xde, 0x6d, 0x49, 0xf2, 0x07, 0x02, 0xe9, 0xe8, 0x41, 0xeb, 0x15, 0xa8, 0x27, 0x8c,
+	0x50, 0x6f, 0x84, 0x92, 0xfe, 0x82, 0x20, 0x6c, 0x6b, 0xbe, 0x02, 0xeb, 0x64, 0xc3, 0xd6, 0x73,
+	0x50, 0x79, 0xb8, 0xbd, 0xdf, 0x5f, 0x14, 0xd2, 0x41, 0x51, 0xc5, 0xc8, 0x77, 0x38, 0xda, 0xba,
+	0x05, 0xed, 0xc4, 0x8b, 0x82, 0x63, 0x72, 0xee, 0xc6, 0x38, 0x88, 0x92, 0x7e, 0xed, 0x66, 0xe9,
+	0x4e, 0xdd, 0x69, 0x29, 0xe4, 0x90, 0xe3, 0xec, 0xf7, 0xe0, 0xda, 0x21, 0xf3, 0x28, 0xbb, 0x82,
+	0x77, 0xec, 0x23, 0x58, 0x75, 0x50, 0x48, 0xce, 0xae, 0xe4, 0xda, 0x3e, 0xd4, 0x18, 0x0e, 0x11,
+	0x49, 0x99, 0x70, 0x6d, 0xdb, 0xd1, 0xa0, 0xfd, 0xc7, 0x12, 0x58, 0x3b, 0xe7, 0xc8, 0x1f, 0x52,
+	0xe2, 0xa3, 0x24, 0xf9, 0x3f, 0x2d, 0xd7, 0xcb, 0x50, 0x8b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57,
+	0xab, 0xa0, 0xb5, 0xd2, 0xa3, 0xf6, 0x17, 0xb0, 0x72, 0x88, 0x47, 0x91, 0x37, 0x7e, 0x8a, 0xfa,
+	0xae, 0xc2, 0x62, 0x22, 0x78, 0x0a, 0x55, 0xdb, 0x8e, 0x82, 0xec, 0x21, 0x58, 0x9f, 0x7b, 0x98,
+	0x3d, 0x3d, 0x49, 0xf6, 0xeb, 0xb0, 0x5c, 0xe0, 0x98, 0xc4, 0x24, 0x4a, 0x90, 0x50, 0x80, 0x79,
+	0x2c, 0x4d, 0x04, 0xb3, 0x05, 0x47, 0x41, 0x36, 0x81, 0xd5, 0xa3, 0x38, 0xb8, 0x62, 0x34, 0xdd,
+	0x85, 0x06, 0x45, 0x09, 0x49, 0x29, 0x8f, 0x81, 0xb2, 0x70, 0xea, 0x8a, 0x74, 0xea, 0x27, 0x38,
+	0x4a, 0xcf, 0x1d, 0x3d, 0xe6, 0xe4, 0x64, 0x6a, 0x7f, 0xb2, 0xe4, 0x2a, 0xfb, 0xf3, 0x3d, 0xb8,
+	0x36, 0xf4, 0xd2, 0xe4, 0x2a, 0xba, 0xda, 0xef, 0xf3, 0xbd, 0x9d, 0xa4, 0xe1, 0x95, 0x26, 0xff,
+	0xa1, 0x04, 0xf5, 0xed, 0x38, 0x3d, 0x4a, 0xbc, 0x11, 0xb2, 0xbe, 0x03, 0x4d, 0x46, 0x98, 0x37,
+	0x76, 0x53, 0x0e, 0x0a, 0xf2, 0xaa, 0x03, 0x02, 0x25, 0x09, 0x5e, 0x80, 0x56, 0x8c, 0xa8, 0x1f,
+	0xa7, 0x8a, 0xa2, 0x7c, 0xb3, 0x72, 0xa7, 0xea, 0x34, 0x25, 0x4e, 0x92, 0xac, 0xc3, 0xb2, 0x18,
+	0x73, 0x71, 0xe4, 0x9e, 0x22, 0x1a, 0xa1, 0x71, 0x48, 0x02, 0x24, 0x36, 0x47, 0xd5, 0xe9, 0x89,
+	0xa1, 0xfd, 0xe8, 0xe3, 0x6c, 0xc0, 0x7a, 0x15, 0x7a, 0x19, 0x3d, 0xdf, 0xf1, 0x82, 0xba, 0x2a,
+	0xa8, 0xbb, 0x8a, 0xfa, 0x48, 0xa1, 0xed, 0x5f, 0x42, 0xe7, 0xb3, 0x13, 0x4a, 0x18, 0x1b, 0xe3,
+	0x68, 0xf4, 0xc0, 0x63, 0x1e, 0x0f, 0xcd, 0x18, 0x51, 0x4c, 0x82, 0x44, 0x69, 0xab, 0x41, 0xeb,
+	0x35, 0xe8, 0x31, 0x49, 0x8b, 0x02, 0x57, 0xd3, 0x94, 0x05, 0xcd, 0x52, 0x36, 0x30, 0x54, 0xc4,
+	0x2f, 0x41, 0x27, 0x27, 0xe6, 0xc1, 0xad, 0xf4, 0x6d, 0x67, 0xd8, 0xcf, 0x70, 0x88, 0xec, 0x33,
+	0xe1, 0x2b, 0xb1, 0xc8, 0xd6, 0x6b, 0xd0, 0xc8, 0xfd, 0x50, 0x12, 0x3b, 0xa4, 0x23, 0x77, 0x88,
+	0x76, 0xa7, 0x53, 0xcf, 0x9c, 0xf2, 0x01, 0x74, 0x59, 0xa6, 0xb8, 0x1b, 0x78, 0xcc, 0x2b, 0x6e,
+	0xaa, 0xa2, 0x55, 0x4e, 0x87, 0x15, 0x60, 0xfb, 0x7d, 0x68, 0x0c, 0x71, 0x90, 0x48, 0xc1, 0x7d,
+	0xa8, 0xf9, 0x29, 0xa5, 0x28, 0x62, 0xda, 0x64, 0x05, 0x5a, 0x2b, 0xb0, 0x30, 0xc6, 0x21, 0x66,
+	0xca, 0x4c, 0x09, 0xd8, 0x04, 0xe0, 0x00, 0x85, 0x84, 0x5e, 0x08, 0x87, 0xad, 0xc0, 0x82, 0xb9,
+	0xb8, 0x12, 0xb0, 0x9e, 0x85, 0x46, 0xe8, 0x9d, 0x67, 0x8b, 0xca, 0x47, 0xea, 0xa1, 0x77, 0x2e,
+	0x95, 0xef, 0x43, 0xed, 0x91, 0x87, 0xc7, 0x7e, 0xc4, 0x94, 0x57, 0x34, 0x98, 0x0b, 0xac, 0x9a,
+	0x02, 0xff, 0x5a, 0x86, 0xa6, 0x94, 0x28, 0x15, 0x5e, 0x81, 0x05, 0xdf, 0xf3, 0x4f, 0x32, 0x91,
+	0x02, 0xb0, 0x6e, 0x6b, 0x45, 0xca, 0x66, 0x86, 0xcb, 0x35, 0xd5, 0xaa, 0x6d, 0x00, 0x24, 0x8f,
+	0xbd, 0x58, 0xe9, 0x56, 0x99, 0x43, 0xdc, 0xe0, 0x34, 0x52, 0xdd, 0xb7, 0xa0, 0x25, 0xf7, 0x9d,
+	0x9a, 0x52, 0x9d, 0x33, 0xa5, 0x29, 0xa9, 0xe4, 0xa4, 0x5b, 0xd0, 0x4e, 0x13, 0xe4, 0x9e, 0x60,
+	0x44, 0x3d, 0xea, 0x9f, 0x5c, 0xf4, 0x17, 0xe4, 0x01, 0x94, 0x26, 0x68, 0x4f, 0xe3, 0xac, 0xbb,
+	0xb0, 0xc0, 0x73, 0x4b, 0xd2, 0x5f, 0x14, 0x67, 0xdd, 0x73, 0x26, 0x4b, 0x61, 0xea, 0xba, 0xf8,
+	0xdd, 0x89, 0x18, 0xbd, 0x70, 0x24, 0xe9, 0xe0, 0x1d, 0x80, 0x1c, 0x69, 0x2d, 0x41, 0xe5, 0x14,
+	0x5d, 0xa8, 0x38, 0xe4, 0x9f, 0xdc, 0x39, 0x67, 0xde, 0x38, 0xd5, 0x5e, 0x97, 0xc0, 0x7b, 0xe5,
+	0x77, 0x4a, 0xb6, 0x0f, 0xdd, 0xad, 0xf1, 0x29, 0x26, 0xc6, 0xf4, 0x15, 0x58, 0x08, 0xbd, 0x2f,
+	0x08, 0xd5, 0x9e, 0x14, 0x80, 0xc0, 0xe2, 0x88, 0x50, 0xcd, 0x42, 0x00, 0x56, 0x07, 0xca, 0x24,
+	0x16, 0xfe, 0x6a, 0x38, 0x65, 0x12, 0xe7, 0x82, 0xaa, 0x86, 0x20, 0xfb, 0x9f, 0x55, 0x80, 0x5c,
+	0x8a, 0xe5, 0xc0, 0x00, 0x13, 0x37, 0x41, 0x94, 0x9f, 0xef, 0xee, 0xf1, 0x05, 0x43, 0x89, 0x4b,
+	0x91, 0x9f, 0xd2, 0x04, 0x9f, 0xf1, 0xf5, 0xe3, 0x66, 0x5f, 0x93, 0x66, 0x4f, 0xe8, 0xe6, 0x5c,
+	0xc7, 0xe4, 0x50, 0xce, 0xdb, 0xe2, 0xd3, 0x1c, 0x3d, 0xcb, 0xda, 0x87, 0x6b, 0x39, 0xcf, 0xc0,
+	0x60, 0x57, 0xbe, 0x8c, 0xdd, 0x72, 0xc6, 0x2e, 0xc8, 0x59, 0xed, 0xc0, 0x32, 0x26, 0xee, 0x97,
+	0x29, 0x4a, 0x0b, 0x8c, 0x2a, 0x97, 0x31, 0xea, 0x61, 0xf2, 0x43, 0x31, 0x21, 0x67, 0x33, 0x84,
+	0x1b, 0x86, 0x95, 0x3c, 0xdc, 0x0d, 0x66, 0xd5, 0xcb, 0x98, 0xad, 0x66, 0x5a, 0xf1, 0x7c, 0x90,
+	0x73, 0xfc, 0x08, 0x56, 0x31, 0x71, 0x1f, 0x7b, 0x98, 0x4d, 0xb2, 0x5b, 0xf8, 0x16, 0x23, 0xf9,
+	0x89, 0x56, 0xe4, 0x25, 0x8d, 0x0c, 0x11, 0x1d, 0x15, 0x8c, 0x5c, 0xfc, 0x16, 0x23, 0x0f, 0xc4,
+	0x84, 0x9c, 0xcd, 0x26, 0xf4, 0x30, 0x99, 0xd4, 0xa6, 0x76, 0x19, 0x93, 0x2e, 0x26, 0x45, 0x4d,
+	0xb6, 0xa0, 0x97, 0x20, 0x9f, 0x11, 0x6a, 0x6e, 0x82, 0xfa, 0x65, 0x2c, 0x96, 0x14, 0x7d, 0xc6,
+	0xc3, 0xfe, 0x29, 0xb4, 0xf6, 0xd2, 0x11, 0x62, 0xe3, 0xe3, 0x2c, 0x19, 0x3c, 0xb5, 0xfc, 0x63,
+	0xff, 0xbb, 0x0c, 0xcd, 0xed, 0x11, 0x25, 0x69, 0x5c, 0xc8, 0xc9, 0x32, 0x48, 0x27, 0x73, 0xb2,
+	0x20, 0x11, 0x39, 0x59, 0x12, 0xbf, 0x0d, 0xad, 0x50, 0x84, 0xae, 0xa2, 0x97, 0x79, 0xa8, 0x37,
+	0x15, 0xd4, 0x4e, 0x33, 0x34, 0x92, 0xd9, 0x3a, 0x40, 0x8c, 0x83, 0x44, 0xcd, 0x91, 0xe9, 0xa8,
+	0xab, 0xca, 0x2d, 0x9d, 0xa2, 0x9d, 0x46, 0x9c, 0x65, 0xeb, 0x37, 0xa1, 0x79, 0xcc, 0x9d, 0xa4,
+	0x26, 0x14, 0x92, 0x51, 0xee, 0x3d, 0x07, 0x8e, 0xf3, 0x20, 0xdc, 0x83, 0xf6, 0x89, 0x74, 0x99,
+	0x9a, 0x24, 0xf7, 0xd0, 0x2d, 0x65, 0x49, 0x6e, 0xef, 0xba, 0xe9, 0x59, 0xb9, 0x00, 0xad, 0x13,
+	0x03, 0x35, 0x38, 0x84, 0xde, 0x14, 0xc9, 0x8c, 0x1c, 0x74, 0xc7, 0xcc, 0x41, 0xcd, 0xbb, 0x96,
+	0x14, 0x64, 0xce, 0x34, 0xf3, 0xd2, 0x6f, 0xcb, 0xd0, 0xfa, 0x14, 0xb1, 0xc7, 0x84, 0x9e, 0x4a,
+	0x7d, 0x2d, 0xa8, 0x46, 0x5e, 0x88, 0x14, 0x47, 0xf1, 0x6d, 0xdd, 0x80, 0x3a, 0x3d, 0x97, 0x09,
+	0x44, 0xad, 0x67, 0x8d, 0x9e, 0x8b, 0xc4, 0x60, 0x3d, 0x0f, 0x40, 0xcf, 0xdd, 0xd8, 0xf3, 0x4f,
+	0x91, 0xf2, 0x60, 0xd5, 0x69, 0xd0, 0xf3, 0xa1, 0x44, 0xf0, 0xad, 0x40, 0xcf, 0x5d, 0x44, 0x29,
+	0xa1, 0x89, 0xca, 0x55, 0x75, 0x7a, 0xbe, 0x23, 0x60, 0x35, 0x37, 0xa0, 0x24, 0x8e, 0x51, 0x20,
+	0x72, 0xb4, 0x98, 0xfb, 0x40, 0x22, 0xb8, 0x54, 0xa6, 0xa5, 0x2e, 0x4a, 0xa9, 0x2c, 0x97, 0xca,
+	0x72, 0xa9, 0x35, 0x39, 0x93, 0x99, 0x52, 0x59, 0x26, 0xb5, 0x2e, 0xa5, 0x32, 0x43, 0x2a, 0xcb,
+	0xa5, 0x36, 0xf4, 0x5c, 0x25, 0xd5, 0xfe, 0x4d, 0x09, 0x56, 0x27, 0x0b, 0x3f, 0x55, 0x9b, 0xbe,
+	0x0d, 0x2d, 0x5f, 0xac, 0x57, 0x61, 0x4f, 0xf6, 0xa6, 0x56, 0xd2, 0x69, 0xfa, 0xc6, 0x36, 0xbe,
+	0x07, 0xed, 0x48, 0x3a, 0x38, 0xdb, 0x9a, 0x95, 0x7c, 0x5d, 0x4c, 0xdf, 0x3b, 0xad, 0xc8, 0x80,
+	0xec, 0x00, 0xac, 0xcf, 0x29, 0x66, 0xe8, 0x90, 0x51, 0xe4, 0x85, 0x4f, 0xa3, 0xba, 0xb7, 0xa0,
+	0x2a, 0xaa, 0x15, 0xbe, 0x4c, 0x2d, 0x47, 0x7c, 0xdb, 0x2f, 0xc3, 0x72, 0x41, 0x8a, 0xb2, 0x75,
+	0x09, 0x2a, 0x63, 0x14, 0x09, 0xee, 0x6d, 0x87, 0x7f, 0xda, 0x1e, 0xf4, 0x1c, 0xe4, 0x05, 0x4f,
+	0x4f, 0x1b, 0x25, 0xa2, 0x92, 0x8b, 0xb8, 0x03, 0x96, 0x29, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x19,
+	0x5a, 0x3f, 0x84, 0xde, 0xf6, 0x98, 0x24, 0xe8, 0x90, 0x05, 0x38, 0x7a, 0x1a, 0xed, 0xc8, 0x2f,
+	0x60, 0xf9, 0x33, 0x76, 0xf1, 0x39, 0x67, 0x96, 0xe0, 0xaf, 0xd0, 0x53, 0xb2, 0x8f, 0x92, 0xc7,
+	0xda, 0x3e, 0x4a, 0x1e, 0xf3, 0xe6, 0xc6, 0x27, 0xe3, 0x34, 0x8c, 0x44, 0x28, 0xb4, 0x1d, 0x05,
+	0xd9, 0x5b, 0xd0, 0x92, 0x35, 0xf4, 0x01, 0x09, 0xd2, 0x31, 0x9a, 0x19, 0x83, 0x6b, 0x00, 0xb1,
+	0x47, 0xbd, 0x10, 0x31, 0x44, 0xe5, 0x1e, 0x6a, 0x38, 0x06, 0xc6, 0xfe, 0x5d, 0x19, 0x56, 0xe4,
+	0x7d, 0xc3, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x03, 0xa8, 0x9f, 0x90, 0x84, 0x19, 0x0c, 0x33, 0x98,
+	0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0x16, 0x2e, 0x01, 0x2a, 0x97, 0x5f, 0x02, 0x4c, 0xb5,
+	0xf9, 0xd5, 0xe9, 0x36, 0x9f, 0x47, 0x9b, 0x26, 0xc2, 0x32, 0xc6, 0x1b, 0x4e, 0x43, 0x61, 0xf6,
+	0x03, 0xeb, 0x36, 0x74, 0x47, 0x5c, 0x4b, 0xf7, 0x84, 0x90, 0x53, 0x37, 0xf6, 0xd8, 0x89, 0x08,
+	0xf5, 0x86, 0xd3, 0x16, 0xe8, 0x3d, 0x42, 0x4e, 0x87, 0x1e, 0x3b, 0xb1, 0xde, 0x85, 0x8e, 0x2a,
+	0x03, 0x43, 0xe1, 0xa2, 0x44, 0x1d, 0x7e, 0x2a, 0x8a, 0x4c, 0xef, 0x39, 0xed, 0x53, 0x03, 0x4a,
+	0xec, 0xeb, 0x70, 0xed, 0x01, 0x4a, 0x18, 0x25, 0x17, 0x45, 0xc7, 0xd8, 0xdf, 0x07, 0xd8, 0x8f,
+	0x18, 0xa2, 0x8f, 0x3c, 0x1f, 0x25, 0xd6, 0x1b, 0x26, 0xa4, 0x8a, 0xa3, 0xa5, 0x75, 0x79, 0xdd,
+	0x93, 0x0d, 0x38, 0x06, 0x8d, 0xbd, 0x0e, 0x8b, 0x0e, 0x49, 0x79, 0x3a, 0x7a, 0x51, 0x7f, 0xa9,
+	0x79, 0x2d, 0x35, 0x4f, 0x20, 0x1d, 0x35, 0x66, 0xef, 0xe9, 0x16, 0x36, 0x67, 0xa7, 0x96, 0x68,
+	0x1d, 0x1a, 0x58, 0xe3, 0x54, 0x56, 0x99, 0x16, 0x9d, 0x93, 0xd8, 0xef, 0xc3, 0xb2, 0xe4, 0x24,
+	0x39, 0x6b, 0x36, 0x2f, 0xc2, 0x22, 0xd5, 0x6a, 0x94, 0xf2, 0x7b, 0x1e, 0x45, 0xa4, 0xc6, 0xb8,
+	0x3f, 0x3e, 0xc1, 0x09, 0xcb, 0x0d, 0xd1, 0xfe, 0x58, 0x86, 0x1e, 0x1f, 0x28, 0xf0, 0xb4, 0x3f,
+	0x84, 0xd6, 0xa6, 0x33, 0xfc, 0x14, 0xe1, 0xd1, 0xc9, 0x31, 0xcf, 0x9e, 0xdf, 0x2b, 0xc2, 0xca,
+	0x60, 0x4b, 0x69, 0x6b, 0x0c, 0x39, 0x05, 0x3a, 0xfb, 0x23, 0x58, 0xdd, 0x0c, 0x02, 0x13, 0xa5,
+	0xb5, 0x7e, 0x03, 0x1a, 0x91, 0xc1, 0xce, 0x38, 0xb3, 0x0a, 0xd4, 0x39, 0x91, 0xfd, 0x33, 0x58,
+	0x7e, 0x18, 0x8d, 0x71, 0x84, 0xb6, 0x87, 0x47, 0x07, 0x28, 0xcb, 0x45, 0x16, 0x54, 0x79, 0xcd,
+	0x26, 0x78, 0xd4, 0x1d, 0xf1, 0xcd, 0x83, 0x33, 0x3a, 0x76, 0xfd, 0x38, 0x4d, 0xd4, 0x65, 0xcf,
+	0x62, 0x74, 0xbc, 0x1d, 0xa7, 0x09, 0x3f, 0x5c, 0x78, 0x71, 0x41, 0xa2, 0xf1, 0x85, 0x88, 0xd0,
+	0xba, 0x53, 0xf3, 0xe3, 0xf4, 0x61, 0x34, 0xbe, 0xb0, 0xbf, 0x2b, 0x3a, 0x70, 0x84, 0x02, 0xc7,
+	0x8b, 0x02, 0x12, 0x3e, 0x40, 0x67, 0x86, 0x84, 0xac, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x12, 0xb4,
+	0x36, 0x47, 0x28, 0x62, 0x0f, 0x10, 0xf3, 0xf0, 0x58, 0x74, 0x74, 0x67, 0x88, 0x26, 0x98, 0x44,
+	0x2a, 0xdc, 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x61, 0xe6, 0x06, 0x1e, 0x0a, 0x49, 0x24, 0xb8, 0xd4,
+	0x1d, 0xe0, 0xa8, 0x07, 0x02, 0x63, 0xbd, 0x0c, 0x5d, 0x79, 0x19, 0xe7, 0x9e, 0x78, 0x51, 0x30,
+	0xe6, 0x81, 0x5e, 0x11, 0xa1, 0xd9, 0x91, 0xe8, 0x3d, 0x85, 0xb5, 0x5e, 0x81, 0x25, 0x15, 0x86,
+	0x39, 0x65, 0x55, 0x50, 0x76, 0x15, 0xbe, 0x40, 0x9a, 0xc6, 0x31, 0xa1, 0x2c, 0x71, 0x13, 0xe4,
+	0xfb, 0x24, 0x8c, 0x55, 0x3b, 0xd4, 0xd5, 0xf8, 0x43, 0x89, 0xb6, 0x47, 0xb0, 0xbc, 0xcb, 0xed,
+	0x54, 0x96, 0xe4, 0xdb, 0xaa, 0x13, 0xa2, 0xd0, 0x3d, 0x1e, 0x13, 0xff, 0xd4, 0xe5, 0xc9, 0x51,
+	0x79, 0x98, 0x17, 0x5c, 0x5b, 0x1c, 0x79, 0x88, 0xbf, 0x12, 0x9d, 0x3f, 0xa7, 0x3a, 0x21, 0x2c,
+	0x1e, 0xa7, 0x23, 0x37, 0xa6, 0xe4, 0x18, 0x29, 0x13, 0xbb, 0x21, 0x0a, 0xf7, 0x24, 0x7e, 0xc8,
+	0xd1, 0xf6, 0x9f, 0x4b, 0xb0, 0x52, 0x94, 0xa4, 0x52, 0xfd, 0x06, 0xac, 0x14, 0x45, 0xa9, 0xe3,
+	0x5f, 0x96, 0x97, 0x3d, 0x53, 0xa0, 0x2c, 0x04, 0xee, 0x41, 0x5b, 0x5c, 0xdd, 0xba, 0x81, 0xe4,
+	0x54, 0x2c, 0x7a, 0xcc, 0x75, 0x71, 0x5a, 0x9e, 0xb9, 0x4a, 0xef, 0xc2, 0x0d, 0x65, 0xbe, 0x3b,
+	0xad, 0xb6, 0xdc, 0x10, 0xab, 0x8a, 0xe0, 0x60, 0x42, 0xfb, 0x4f, 0xa0, 0x9f, 0xa3, 0xb6, 0x2e,
+	0x04, 0x32, 0xdf, 0xcc, 0xcb, 0x13, 0xc6, 0x6e, 0x06, 0x01, 0x15, 0x51, 0x52, 0x75, 0x66, 0x0d,
+	0xd9, 0xf7, 0xe1, 0xfa, 0x21, 0x62, 0xd2, 0x1b, 0x1e, 0x53, 0x9d, 0x88, 0x64, 0xb6, 0x04, 0x95,
+	0x43, 0xe4, 0x0b, 0xe3, 0x2b, 0x0e, 0xff, 0xe4, 0x1b, 0xf0, 0x28, 0x41, 0xbe, 0xb0, 0xb2, 0xe2,
+	0x88, 0x6f, 0xfb, 0x4f, 0x25, 0xa8, 0xa9, 0xe4, 0xcc, 0x0f, 0x98, 0x80, 0xe2, 0x33, 0x44, 0xd5,
+	0xd6, 0x53, 0x90, 0xf5, 0x12, 0x74, 0xe4, 0x97, 0x4b, 0x62, 0x86, 0x49, 0x96, 0xf2, 0xdb, 0x12,
+	0xfb, 0x50, 0x22, 0xc5, 0xe5, 0x9b, 0xb8, 0xfe, 0x52, 0x9d, 0xa6, 0x82, 0x38, 0xfe, 0x51, 0xc2,
+	0x23, 0x5c, 0xa4, 0xf8, 0x86, 0xa3, 0x20, 0xbe, 0xd5, 0x35, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c,
+	0xab, 0x87, 0x24, 0x8d, 0x98, 0x1b, 0x13, 0x1c, 0x31, 0x95, 0xd3, 0x41, 0xa0, 0x86, 0x1c, 0x63,
+	0xff, 0xba, 0x04, 0x8b, 0xf2, 0x02, 0x9a, 0xf7, 0xb6, 0xd9, 0xc9, 0x5a, 0xc6, 0xa2, 0x4a, 0x11,
+	0xb2, 0xe4, 0x69, 0x2a, 0xbe, 0x79, 0x1c, 0x9f, 0x85, 0xf2, 0x7c, 0x50, 0xaa, 0x9d, 0x85, 0xe2,
+	0x60, 0x78, 0x09, 0x3a, 0xf9, 0x01, 0x2d, 0xc6, 0xa5, 0x8a, 0xed, 0x0c, 0x2b, 0xc8, 0xe6, 0x6a,
+	0x6a, 0xff, 0x98, 0xb7, 0xf4, 0xd9, 0xe5, 0xeb, 0x12, 0x54, 0xd2, 0x4c, 0x19, 0xfe, 0xc9, 0x31,
+	0xa3, 0xec, 0x68, 0xe7, 0x9f, 0xd6, 0x6d, 0xe8, 0x78, 0x41, 0x80, 0xf9, 0x74, 0x6f, 0xbc, 0x8b,
+	0x83, 0x2c, 0x48, 0x8b, 0x58, 0xfb, 0x6f, 0x25, 0xe8, 0x6e, 0x93, 0xf8, 0xe2, 0x43, 0x3c, 0x46,
+	0x46, 0x06, 0x11, 0x4a, 0xaa, 0x93, 0x9d, 0x7f, 0xf3, 0x6a, 0xf5, 0x11, 0x1e, 0x23, 0x19, 0x5a,
+	0x72, 0x65, 0xeb, 0x1c, 0x21, 0xc2, 0x4a, 0x0f, 0x66, 0xd7, 0x6e, 0x6d, 0x39, 0x78, 0x40, 0x02,
+	0x51, 0x97, 0x07, 0x98, 0xba, 0xd9, 0x25, 0x5b, 0xdb, 0xa9, 0x05, 0x98, 0x8a, 0x21, 0x65, 0xc8,
+	0x82, 0xb8, 0x44, 0x35, 0x0d, 0x59, 0x94, 0x18, 0x6e, 0xc8, 0x2a, 0x2c, 0x92, 0x47, 0x8f, 0x12,
+	0xc4, 0x44, 0x05, 0x5d, 0x71, 0x14, 0x94, 0xa5, 0xb9, 0xba, 0x91, 0xe6, 0x56, 0xc0, 0xda, 0x45,
+	0xec, 0xe1, 0xc3, 0x83, 0x9d, 0x33, 0x14, 0x31, 0x7d, 0x3a, 0xbc, 0x0e, 0x75, 0x8d, 0xfa, 0x6f,
+	0xae, 0x27, 0x5f, 0x85, 0xce, 0x66, 0x10, 0x1c, 0x3e, 0xf6, 0x62, 0xed, 0x8f, 0x3e, 0xd4, 0x86,
+	0xdb, 0xfb, 0x43, 0xe9, 0x92, 0x0a, 0x37, 0x40, 0x81, 0xfc, 0x34, 0xda, 0x45, 0xec, 0x00, 0x31,
+	0x8a, 0xfd, 0xec, 0x34, 0xba, 0x05, 0x35, 0x85, 0xe1, 0x33, 0x43, 0xf9, 0xa9, 0xd3, 0xac, 0x02,
+	0xed, 0x1f, 0x80, 0xf5, 0x23, 0x5e, 0x57, 0x21, 0x59, 0x54, 0x2b, 0x49, 0xaf, 0x42, 0xef, 0x4c,
+	0x60, 0x5d, 0x59, 0x70, 0x18, 0xcb, 0xd0, 0x95, 0x03, 0x22, 0x06, 0x85, 0xec, 0x23, 0x58, 0x96,
+	0x65, 0xa0, 0xe4, 0x73, 0x05, 0x16, 0xdc, 0x87, 0xd9, 0x7a, 0x56, 0x1d, 0xf1, 0x7d, 0xf7, 0x2f,
+	0x3d, 0x75, 0x54, 0xa8, 0x5b, 0x07, 0x6b, 0x17, 0xba, 0x13, 0x4f, 0x44, 0x96, 0xba, 0x86, 0x9a,
+	0xfd, 0x72, 0x34, 0x58, 0x5d, 0x97, 0x4f, 0x4e, 0xeb, 0xfa, 0xc9, 0x69, 0x7d, 0x27, 0x8c, 0xd9,
+	0x85, 0xb5, 0x03, 0x9d, 0xe2, 0x63, 0x8a, 0xf5, 0xac, 0xae, 0xda, 0x66, 0x3c, 0xb1, 0xcc, 0x65,
+	0xb3, 0x0b, 0xdd, 0x89, 0x77, 0x15, 0xad, 0xcf, 0xec, 0xe7, 0x96, 0xb9, 0x8c, 0xee, 0x43, 0xd3,
+	0x78, 0x48, 0xb1, 0xfa, 0x92, 0xc9, 0xf4, 0xdb, 0xca, 0x5c, 0x06, 0xdb, 0xd0, 0x2e, 0xbc, 0x6d,
+	0x58, 0x03, 0x65, 0xcf, 0x8c, 0x07, 0x8f, 0xb9, 0x4c, 0xb6, 0xa0, 0x69, 0x3c, 0x31, 0x68, 0x2d,
+	0xa6, 0xdf, 0x31, 0x06, 0x37, 0x66, 0x8c, 0xa8, 0x13, 0x69, 0x17, 0xba, 0x13, 0xef, 0x0e, 0xda,
+	0x25, 0xb3, 0x9f, 0x23, 0xe6, 0x2a, 0xf3, 0xb1, 0x58, 0x22, 0xa3, 0xad, 0x34, 0x96, 0x68, 0xfa,
+	0x95, 0x61, 0xf0, 0xdc, 0xec, 0x41, 0xa5, 0xd5, 0x0e, 0x74, 0x8a, 0x0f, 0x0c, 0x9a, 0xd9, 0xcc,
+	0x67, 0x87, 0xcb, 0xd7, 0xbb, 0xf0, 0xd6, 0x90, 0xaf, 0xf7, 0xac, 0x27, 0x88, 0xb9, 0x8c, 0x36,
+	0x01, 0x54, 0x13, 0x19, 0xe0, 0x28, 0x73, 0xf4, 0x54, 0xf3, 0x9a, 0x39, 0x7a, 0x46, 0xc3, 0x79,
+	0x1f, 0x40, 0xf6, 0x7e, 0x01, 0x49, 0x99, 0x75, 0x5d, 0xab, 0x31, 0xd1, 0x70, 0x0e, 0xfa, 0xd3,
+	0x03, 0x53, 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x01, 0x40, 0xde, 0x53, 0x6a, 0x06, 0x53, 0x5d,
+	0xe6, 0x25, 0x3e, 0x68, 0x99, 0x1d, 0xa4, 0xa5, 0x6c, 0x9d, 0xd1, 0x55, 0x5e, 0xc2, 0xa2, 0x3b,
+	0xd1, 0x21, 0x14, 0x37, 0xdb, 0x64, 0xe3, 0x30, 0x98, 0xea, 0x12, 0xac, 0x7b, 0xd0, 0x32, 0x5b,
+	0x03, 0xad, 0xc5, 0x8c, 0x76, 0x61, 0x50, 0x68, 0x0f, 0xac, 0xfb, 0xd0, 0x29, 0xb6, 0x05, 0x7a,
+	0x4b, 0xcd, 0x6c, 0x16, 0x06, 0xea, 0xd2, 0xcb, 0x20, 0x7f, 0x0b, 0x20, 0x6f, 0x1f, 0xb4, 0xfb,
+	0xa6, 0x1a, 0x8a, 0x09, 0xa9, 0xbb, 0xd0, 0x9d, 0x68, 0x0b, 0xb4, 0xc5, 0xb3, 0xbb, 0x85, 0xb9,
+	0xae, 0x7b, 0x1b, 0x20, 0x3f, 0x2e, 0xb4, 0xf4, 0xa9, 0x03, 0x64, 0xd0, 0xd6, 0x17, 0x82, 0x92,
+	0x6e, 0x1b, 0xda, 0x85, 0x9e, 0x59, 0xa7, 0x99, 0x59, 0x8d, 0xf4, 0x65, 0xc9, 0xb7, 0xd8, 0x60,
+	0x6a, 0xcf, 0xcd, 0x6c, 0x3b, 0x2f, 0xdb, 0x3f, 0x66, 0x57, 0xa3, 0x57, 0x6e, 0x46, 0xa7, 0xf3,
+	0x2d, 0xf1, 0x6c, 0x76, 0x2e, 0x46, 0x3c, 0xcf, 0x68, 0x68, 0xe6, 0x32, 0xda, 0x83, 0xee, 0xae,
+	0x2e, 0x4a, 0x55, 0xc1, 0xac, 0xd4, 0x99, 0xd1, 0x20, 0x0c, 0x06, 0xb3, 0x86, 0x54, 0x50, 0x7d,
+	0x0c, 0xbd, 0xa9, 0x62, 0xd9, 0x5a, 0xcb, 0xae, 0x65, 0x67, 0x56, 0xd1, 0x73, 0xd5, 0xda, 0x87,
+	0xa5, 0xc9, 0x5a, 0xd9, 0x7a, 0x5e, 0x25, 0xca, 0xd9, 0x35, 0xf4, 0x5c, 0x56, 0xef, 0x42, 0x5d,
+	0xd7, 0x66, 0x96, 0xba, 0xfe, 0x9e, 0xa8, 0xd5, 0xe6, 0x4e, 0xbd, 0x07, 0x4d, 0xa3, 0x14, 0xd2,
+	0xd9, 0x6e, 0xba, 0x3a, 0x1a, 0xa8, 0xdb, 0xea, 0x8c, 0xf2, 0x1e, 0xd4, 0x54, 0xf9, 0x63, 0xad,
+	0x64, 0x9b, 0xdc, 0xa8, 0x86, 0x2e, 0xdb, 0x61, 0xbb, 0x88, 0x19, 0x45, 0x8d, 0x16, 0x3a, 0x5d,
+	0xe7, 0xe8, 0x14, 0x5b, 0x18, 0x51, 0x6b, 0xb1, 0x09, 0x2d, 0xb3, 0xac, 0xd1, 0x4b, 0x3a, 0xa3,
+	0xd4, 0x99, 0xa7, 0xc9, 0xd6, 0xf9, 0xd7, 0xdf, 0xac, 0x3d, 0xf3, 0x8f, 0x6f, 0xd6, 0x9e, 0xf9,
+	0xd5, 0x93, 0xb5, 0xd2, 0xd7, 0x4f, 0xd6, 0x4a, 0x7f, 0x7f, 0xb2, 0x56, 0xfa, 0xd7, 0x93, 0xb5,
+	0xd2, 0x4f, 0x7e, 0xfe, 0x3f, 0xfe, 0x0f, 0x87, 0xa6, 0x11, 0xc3, 0x21, 0xda, 0x38, 0xc3, 0x94,
+	0x19, 0x43, 0xf1, 0xe9, 0x48, 0xfe, 0x19, 0xc7, 0xf8, 0x8f, 0x0e, 0xd7, 0xf2, 0x78, 0x51, 0xc0,
+	0x6f, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x4e, 0xa6, 0xdc, 0xf0, 0x23, 0x00, 0x00,
 }
 
 func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) {
@@ -5446,6 +5533,79 @@ func (m *Metrics) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *VolumeStatsRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *VolumeStatsRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *VolumeStatsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.VolumeGuestPath) > 0 {
+		i -= len(m.VolumeGuestPath)
+		copy(dAtA[i:], m.VolumeGuestPath)
+		i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ResizeVolumeRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ResizeVolumeRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ResizeVolumeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if m.Size_ != 0 {
+		i = encodeVarintAgent(dAtA, i, uint64(m.Size_))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.VolumeGuestPath) > 0 {
+		i -= len(m.VolumeGuestPath)
+		copy(dAtA[i:], m.VolumeGuestPath)
+		i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
 	offset -= sovAgent(v)
 	base := offset
@@ -6737,6 +6897,41 @@ func (m *Metrics) Size() (n int) {
 	return n
 }
 
+func (m *VolumeStatsRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.VolumeGuestPath)
+	if l > 0 {
+		n += 1 + l + sovAgent(uint64(l))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *ResizeVolumeRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.VolumeGuestPath)
+	if l > 0 {
+		n += 1 + l + sovAgent(uint64(l))
+	}
+	if m.Size_ != 0 {
+		n += 1 + sovAgent(uint64(m.Size_))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
 func sovAgent(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -7551,6 +7746,29 @@ func (this *Metrics) String() string {
 	}, "")
 	return s
 }
+func (this *VolumeStatsRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&VolumeStatsRequest{`,
+		`VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`,
+		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func (this *ResizeVolumeRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&ResizeVolumeRequest{`,
+		`VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`,
+		`Size_:` + fmt.Sprintf("%v", this.Size_) + `,`,
+		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func valueToStringAgent(v interface{}) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -7592,6 +7810,8 @@ type AgentServiceService interface {
 	CopyFile(ctx context.Context, req *CopyFileRequest) (*types.Empty, error)
 	GetOOMEvent(ctx context.Context, req *GetOOMEventRequest) (*OOMEvent, error)
 	AddSwap(ctx context.Context, req *AddSwapRequest) (*types.Empty, error)
+	GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error)
+	ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error)
 }
 
 func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc AgentServiceService) {
@@ -7813,6 +8033,20 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag
 			}
 			return svc.AddSwap(ctx, &req)
 		},
+		"GetVolumeStats": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req VolumeStatsRequest
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.GetVolumeStats(ctx, &req)
+		},
+		"ResizeVolume": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req ResizeVolumeRequest
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.ResizeVolume(ctx, &req)
+		},
 	})
 }
 
@@ -8073,6 +8307,22 @@ func (c *agentServiceClient) AddSwap(ctx context.Context, req *AddSwapRequest) (
 	}
 	return &resp, nil
 }
+
+func (c *agentServiceClient) GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error) {
+	var resp VolumeStatsResponse
+	if err := c.client.Call(ctx, "grpc.AgentService", "GetVolumeStats", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *agentServiceClient) ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error) {
+	var resp types.Empty
+	if err := c.client.Call(ctx, "grpc.AgentService", "ResizeVolume", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
 func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
@@ -15399,6 +15649,191 @@ func (m *Metrics) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *VolumeStatsRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowAgent
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: VolumeStatsRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: VolumeStatsRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowAgent
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthAgent
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.VolumeGuestPath = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipAgent(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ResizeVolumeRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowAgent
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ResizeVolumeRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ResizeVolumeRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowAgent
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthAgent
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.VolumeGuestPath = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
+			}
+			m.Size_ = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowAgent
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Size_ |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipAgent(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipAgent(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0

src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go

@@ -0,0 +1,585 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto
+
+package grpc
+
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb"
+	github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
+	proto "github.com/gogo/protobuf/proto"
+	math "math"
+	math_rand "math/rand"
+	testing "testing"
+	time "time"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+func TestVolumeStatsResponseProto(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeStatsResponse(popr, false)
+	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeStatsResponse{}
+	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	littlefuzz := make([]byte, len(dAtA))
+	copy(littlefuzz, dAtA)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+	if len(littlefuzz) > 0 {
+		fuzzamount := 100
+		for i := 0; i < fuzzamount; i++ {
+			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+		}
+		// shouldn't panic
+		_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
+	}
+}
+
+func TestVolumeStatsResponseMarshalTo(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeStatsResponse(popr, false)
+	size := p.Size()
+	dAtA := make([]byte, size)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	_, err := p.MarshalTo(dAtA)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeStatsResponse{}
+	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func BenchmarkVolumeStatsResponseProtoMarshal(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	pops := make([]*VolumeStatsResponse, 10000)
+	for i := 0; i < 10000; i++ {
+		pops[i] = NewPopulatedVolumeStatsResponse(popr, false)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
+		if err != nil {
+			panic(err)
+		}
+		total += len(dAtA)
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func BenchmarkVolumeStatsResponseProtoUnmarshal(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	datas := make([][]byte, 10000)
+	for i := 0; i < 10000; i++ {
+		dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeStatsResponse(popr, false))
+		if err != nil {
+			panic(err)
+		}
+		datas[i] = dAtA
+	}
+	msg := &VolumeStatsResponse{}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		total += len(datas[i%10000])
+		if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
+			panic(err)
+		}
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func TestVolumeUsageProto(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeUsage(popr, false)
+	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeUsage{}
+	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	littlefuzz := make([]byte, len(dAtA))
+	copy(littlefuzz, dAtA)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+	if len(littlefuzz) > 0 {
+		fuzzamount := 100
+		for i := 0; i < fuzzamount; i++ {
+			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+		}
+		// shouldn't panic
+		_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
+	}
+}
+
+func TestVolumeUsageMarshalTo(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeUsage(popr, false)
+	size := p.Size()
+	dAtA := make([]byte, size)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	_, err := p.MarshalTo(dAtA)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeUsage{}
+	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func BenchmarkVolumeUsageProtoMarshal(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	pops := make([]*VolumeUsage, 10000)
+	for i := 0; i < 10000; i++ {
+		pops[i] = NewPopulatedVolumeUsage(popr, false)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
+		if err != nil {
+			panic(err)
+		}
+		total += len(dAtA)
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func BenchmarkVolumeUsageProtoUnmarshal(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	datas := make([][]byte, 10000)
+	for i := 0; i < 10000; i++ {
+		dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeUsage(popr, false))
+		if err != nil {
+			panic(err)
+		}
+		datas[i] = dAtA
+	}
+	msg := &VolumeUsage{}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		total += len(datas[i%10000])
+		if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
+			panic(err)
+		}
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func TestVolumeConditionProto(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeCondition(popr, false)
+	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeCondition{}
+	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	littlefuzz := make([]byte, len(dAtA))
+	copy(littlefuzz, dAtA)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+	if len(littlefuzz) > 0 {
+		fuzzamount := 100
+		for i := 0; i < fuzzamount; i++ {
+			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+		}
+		// shouldn't panic
+		_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
+	}
+}
+
+func TestVolumeConditionMarshalTo(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeCondition(popr, false)
+	size := p.Size()
+	dAtA := make([]byte, size)
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	_, err := p.MarshalTo(dAtA)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeCondition{}
+	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	for i := range dAtA {
+		dAtA[i] = byte(popr.Intn(256))
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func BenchmarkVolumeConditionProtoMarshal(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	pops := make([]*VolumeCondition, 10000)
+	for i := 0; i < 10000; i++ {
+		pops[i] = NewPopulatedVolumeCondition(popr, false)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
+		if err != nil {
+			panic(err)
+		}
+		total += len(dAtA)
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func BenchmarkVolumeConditionProtoUnmarshal(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	datas := make([][]byte, 10000)
+	for i := 0; i < 10000; i++ {
+		dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeCondition(popr, false))
+		if err != nil {
+			panic(err)
+		}
+		datas[i] = dAtA
+	}
+	msg := &VolumeCondition{}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		total += len(datas[i%10000])
+		if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
+			panic(err)
+		}
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func TestVolumeStatsResponseJSON(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeStatsResponse(popr, true)
+	marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
+	jsondata, err := marshaler.MarshalToString(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeStatsResponse{}
+	err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+	}
+}
+func TestVolumeUsageJSON(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeUsage(popr, true)
+	marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
+	jsondata, err := marshaler.MarshalToString(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeUsage{}
+	err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+	}
+}
+func TestVolumeConditionJSON(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeCondition(popr, true)
+	marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
+	jsondata, err := marshaler.MarshalToString(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	msg := &VolumeCondition{}
+	err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+	}
+}
+func TestVolumeStatsResponseProtoText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeStatsResponse(popr, true)
+	dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
+	msg := &VolumeStatsResponse{}
+	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestVolumeStatsResponseProtoCompactText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeStatsResponse(popr, true)
+	dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
+	msg := &VolumeStatsResponse{}
+	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestVolumeUsageProtoText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeUsage(popr, true)
+	dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
+	msg := &VolumeUsage{}
+	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestVolumeUsageProtoCompactText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeUsage(popr, true)
+	dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
+	msg := &VolumeUsage{}
+	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestVolumeConditionProtoText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeCondition(popr, true)
+	dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
+	msg := &VolumeCondition{}
+	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestVolumeConditionProtoCompactText(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeCondition(popr, true)
+	dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
+	msg := &VolumeCondition{}
+	if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	if !p.Equal(msg) {
+		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+	}
+}
+
+func TestVolumeStatsResponseSize(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeStatsResponse(popr, true)
+	size2 := github_com_gogo_protobuf_proto.Size(p)
+	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	size := p.Size()
+	if len(dAtA) != size {
+		t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+	}
+	if size2 != size {
+		t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+	}
+	size3 := github_com_gogo_protobuf_proto.Size(p)
+	if size3 != size {
+		t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+	}
+}
+
+func BenchmarkVolumeStatsResponseSize(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	pops := make([]*VolumeStatsResponse, 1000)
+	for i := 0; i < 1000; i++ {
+		pops[i] = NewPopulatedVolumeStatsResponse(popr, false)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		total += pops[i%1000].Size()
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func TestVolumeUsageSize(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeUsage(popr, true)
+	size2 := github_com_gogo_protobuf_proto.Size(p)
+	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	size := p.Size()
+	if len(dAtA) != size {
+		t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+	}
+	if size2 != size {
+		t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+	}
+	size3 := github_com_gogo_protobuf_proto.Size(p)
+	if size3 != size {
+		t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+	}
+}
+
+func BenchmarkVolumeUsageSize(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	pops := make([]*VolumeUsage, 1000)
+	for i := 0; i < 1000; i++ {
+		pops[i] = NewPopulatedVolumeUsage(popr, false)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		total += pops[i%1000].Size()
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func TestVolumeConditionSize(t *testing.T) {
+	seed := time.Now().UnixNano()
+	popr := math_rand.New(math_rand.NewSource(seed))
+	p := NewPopulatedVolumeCondition(popr, true)
+	size2 := github_com_gogo_protobuf_proto.Size(p)
+	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+	if err != nil {
+		t.Fatalf("seed = %d, err = %v", seed, err)
+	}
+	size := p.Size()
+	if len(dAtA) != size {
+		t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+	}
+	if size2 != size {
+		t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+	}
+	size3 := github_com_gogo_protobuf_proto.Size(p)
+	if size3 != size {
+		t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+	}
+}
+
+func BenchmarkVolumeConditionSize(b *testing.B) {
+	popr := math_rand.New(math_rand.NewSource(616))
+	total := 0
+	pops := make([]*VolumeCondition, 1000)
+	for i := 0; i < 1000; i++ {
+		pops[i] = NewPopulatedVolumeCondition(popr, false)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		total += pops[i%1000].Size()
+	}
+	b.SetBytes(int64(total / b.N))
+}
+
+func TestVolumeStatsResponseStringer(t *testing.T) {
+	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+	p := NewPopulatedVolumeStatsResponse(popr, false)
+	s1 := p.String()
+	s2 := fmt.Sprintf("%v", p)
+	if s1 != s2 {
+		t.Fatalf("String want %v got %v", s1, s2)
+	}
+}
+func TestVolumeUsageStringer(t *testing.T) {
+	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+	p := NewPopulatedVolumeUsage(popr, false)
+	s1 := p.String()
+	s2 := fmt.Sprintf("%v", p)
+	if s1 != s2 {
+		t.Fatalf("String want %v got %v", s1, s2)
+	}
+}
+func TestVolumeConditionStringer(t *testing.T) {
+	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+	p := NewPopulatedVolumeCondition(popr, false)
+	s1 := p.String()
+	s2 := fmt.Sprintf("%v", p)
+	if s1 != s2 {
+		t.Fatalf("String want %v got %v", s1, s2)
+	}
+}
+
+//These tests are generated by github.com/gogo/protobuf/plugin/testgen

src/runtime/virtcontainers/pkg/mock/mock.go

@@ -232,6 +232,14 @@ func (p *HybridVSockTTRPCMockImp) AddSwap(ctx context.Context, req *pb.AddSwapRe
 	return &gpb.Empty{}, nil
 }
 
-func (p *HybridVSockTTRPCMockImp) PullImage(ctx context.Context, req *pb.PullImageRequest) (*gpb.Empty, error) {
+func (p *HybridVSockTTRPCMockImp) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequest) (*pb.VolumeStatsResponse, error) {
+	return &pb.VolumeStatsResponse{}, nil
+}
+
+func (p *HybridVSockTTRPCMockImp) ResizeVolume(ctx context.Context, req *pb.ResizeVolumeRequest) (*gpb.Empty, error) {
+	return &gpb.Empty{}, nil
+}
+
+func (p *HybridVSockTTRPCMockImp) PullImage(ctx context.Context, req *pb.PullImageRequest) (*gpb.Empty
 	return &gpb.Empty{}, nil
 }

src/runtime/virtcontainers/pkg/vcmock/sandbox.go

@@ -256,6 +256,13 @@ func (s *Sandbox) GetHypervisorPid() (int, error) {
 	return 0, nil
 }
 
+func (s *Sandbox) GuestVolumeStats(ctx context.Context, path string) ([]byte, error) {
+	return nil, nil
+}
+func (s *Sandbox) ResizeGuestVolume(ctx context.Context, path string, size uint64) error {
+	return nil
+}
+
 func (s *Sandbox) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) {
 	return nil, nil
 }

src/runtime/virtcontainers/qemu.go

@@ -1840,8 +1840,8 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) {
 		coreID := fmt.Sprintf("%d", hc.Properties.Core)
 		threadID := fmt.Sprintf("%d", hc.Properties.Thread)
 
-		// If CPU type is IBM pSeries or Z, we do not set socketID and threadID
-		if machine.Type == "pseries" || machine.Type == "s390-ccw-virtio" {
+		// If CPU type is IBM pSeries, Z or arm virt, we do not set socketID and threadID
+		if machine.Type == "pseries" || machine.Type == "s390-ccw-virtio" || machine.Type == "virt" {
 			socketID = ""
 			threadID = ""
 			dieID = ""

src/runtime/virtcontainers/qemu_amd64.go

@@ -158,9 +158,8 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) {
 func (q *qemuAmd64) capabilities() types.Capabilities {
 	var caps types.Capabilities
 
-	if (q.qemuMachine.Type == QemuQ35 ||
-		q.qemuMachine.Type == QemuVirt) &&
-		q.protection == noneProtection {
+	if q.qemuMachine.Type == QemuQ35 ||
+		q.qemuMachine.Type == QemuVirt {
 		caps.SetBlockDeviceHotplugSupport()
 	}
 

src/runtime/virtcontainers/qemu_arch_base.go

@@ -277,9 +277,7 @@ func (q *qemuArchBase) kernelParameters(debug bool) []Param {
 
 func (q *qemuArchBase) capabilities() types.Capabilities {
 	var caps types.Capabilities
-	if q.protection == noneProtection {
-		caps.SetBlockDeviceHotplugSupport()
-	}
+	caps.SetBlockDeviceHotplugSupport()
 	caps.SetMultiQueueSupport()
 	caps.SetFsSharingSupport()
 	return caps

src/runtime/virtcontainers/qemu_ppc64le.go

@@ -101,8 +101,7 @@ func (q *qemuPPC64le) capabilities() types.Capabilities {
 	var caps types.Capabilities
 
 	// pseries machine type supports hotplugging drives
-	if q.qemuMachine.Type == QemuPseries &&
-		q.protection == noneProtection {
+	if q.qemuMachine.Type == QemuPseries {
 		caps.SetBlockDeviceHotplugSupport()
 	}
 

src/runtime/virtcontainers/sandbox.go

@@ -2261,6 +2261,43 @@ func (s *Sandbox) GetAgentURL() (string, error) {
 	return s.agent.getAgentURL()
 }
 
+// GuestVolumeStats return the filesystem stat of a given volume in the guest.
+func (s *Sandbox) GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error) {
+	guestMountPath, err := s.guestMountPath(volumePath)
+	if err != nil {
+		return nil, err
+	}
+	return s.agent.getGuestVolumeStats(ctx, guestMountPath)
+}
+
+// ResizeGuestVolume resizes a volume in the guest.
+func (s *Sandbox) ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error {
+	// TODO: https://github.com/kata-containers/kata-containers/issues/3694.
+	guestMountPath, err := s.guestMountPath(volumePath)
+	if err != nil {
+		return err
+	}
+	return s.agent.resizeGuestVolume(ctx, guestMountPath, size)
+}
+
+func (s *Sandbox) guestMountPath(volumePath string) (string, error) {
+	// verify the device even exists
+	if _, err := os.Stat(volumePath); err != nil {
+		s.Logger().WithError(err).WithField("volume", volumePath).Error("Cannot get stats for volume that doesn't exist")
+		return "", err
+	}
+
+	// verify that we have a mount in this sandbox who's source maps to this
+	for _, c := range s.containers {
+		for _, m := range c.mounts {
+			if volumePath == m.Source {
+				return m.GuestDeviceMount, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("mount %s not found in sandbox", volumePath)
+}
+
 // getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets'
 // cpus and mems as a string in canonical linux CPU/mems list format
 func (s *Sandbox) getSandboxCPUSet() (string, string, error) {

src/tools/agent-ctl/src/client.rs

@@ -175,6 +175,11 @@ static AGENT_CMDS: &[AgentCmd] = &[
         st: ServiceType::Agent,
         fp: agent_cmd_sandbox_get_oom_event,
     },
+    AgentCmd {
+        name: "GetVolumeStats",
+        st: ServiceType::Agent,
+        fp: agent_cmd_sandbox_get_volume_stats,
+    },
     AgentCmd {
         name: "ListInterfaces",
         st: ServiceType::Agent,
@@ -1703,6 +1708,29 @@ fn agent_cmd_sandbox_get_oom_event(
     Ok(())
 }
 
+fn agent_cmd_sandbox_get_volume_stats(
+    ctx: &Context,
+    client: &AgentServiceClient,
+    _health: &HealthClient,
+    _options: &mut Options,
+    args: &str,
+) -> Result<()> {
+    let req: VolumeStatsRequest = utils::make_request(args)?;
+
+    let ctx = clone_context(ctx);
+
+    debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
+
+    let reply = client
+        .get_volume_stats(ctx, &req)
+        .map_err(|e| anyhow!(e).context(ERR_API_FAILED))?;
+
+    info!(sl!(), "response received";
+        "response" => format!("{:?}", reply));
+
+    Ok(())
+}
+
 fn agent_cmd_sandbox_copy_file(
     ctx: &Context,
     client: &AgentServiceClient,

tools/packaging/kernel/build-kernel.sh

@@ -89,6 +89,7 @@ Options:
 	-c <path>   	: Path to config file to build the kernel.
 	-d          	: Enable bash debug.
 	-e          	: Enable experimental kernel.
+	-E          	: Enable arch-specific experimental kernel, arch info offered by "-a".
 	-f          	: Enable force generate config when setup.
 	-g <vendor> 	: GPU vendor, intel or nvidia.
 	-h          	: Display this help.
@@ -462,7 +463,7 @@ install_kata() {
 }
 
 main() {
-	while getopts "a:b:c:defg:hk:p:st:v:x:" opt; do
+	while getopts "a:b:c:deEfg:hk:p:t:v:x:" opt; do	
 		case "$opt" in
 			a)
 				arch_target="${OPTARG}"
@@ -480,6 +481,9 @@ main() {
 			e)
 				build_type="experimental"
 				;;
+			E)
+				build_type="arch-experimental"
+				;;
 			f)
 				force_setup_generate_config="true"
 				;;
@@ -525,6 +529,17 @@ main() {
 	if [ -z "$kernel_version" ]; then
 		if [[ ${build_type} == "experimental" ]]; then
 			kernel_version=$(get_from_kata_deps "assets.kernel-experimental.tag")
+		elif [[ ${build_type} == "arch-experimental" ]]; then
+			case "${arch_target}" in
+			"aarch64")
+				build_type="arm-experimental"
+				kernel_version=$(get_from_kata_deps "assets.arm-kernel-experimental.version")
+			;;
+			*)
+				info "No arch-specific experimental kernel supported, using experimental one instead"
+				kernel_version=$(get_from_kata_deps "assets.kernel-experimental.tag")
+			;;
+			esac
 		elif [[ "${conf_guest}" == "tdx" ]]; then
 			 kernel_version=$(get_from_kata_deps "assets.kernel.tdx.tag")
 		else

tools/packaging/kernel/configs/fragments/arm64/base.conf

@@ -37,7 +37,6 @@ CONFIG_ARM64_PAN=y
 CONFIG_ARM64_CNP=y
 CONFIG_ARM64_PMEM=y
 CONFIG_ARM64_RAS_EXTN=y
-CONFIG_ARM64_UAO=y
 # end of ARMv8.2 architectural feature
 
 CONFIG_NO_HZ_FULL=y

tools/packaging/kernel/configs/fragments/common/fs.conf

@@ -26,7 +26,6 @@ CONFIG_FS_POSIX_ACL=y
 CONFIG_EXPORTFS=y
 CONFIG_EXPORTFS_BLOCK_OPS=y
 CONFIG_FILE_LOCKING=y
-CONFIG_MANDATORY_FILE_LOCKING=y
 # A bunch of these are required for systemd at least.
 CONFIG_FSNOTIFY=y
 CONFIG_DNOTIFY=y

tools/packaging/kernel/patches/5.15.x/arm-experimental/0001-arm64-kernel-Handle-disabled-present-cpus-in-MADT-GI.patch

@@ -0,0 +1,145 @@
+From 790af0565140c9df7394c195c22960d92f117c30 Mon Sep 17 00:00:00 2001
+From: Salil Mehta <salil.mehta@huawei.com>
+Date: Wed, 1 Dec 2021 14:58:33 +0800
+Subject: [PATCH 1/7] arm64: kernel: Handle disabled[(+)present] cpus in
+ MADT/GICC during init
+
+With ACPI enabled, cpus get identified by the presence of the GICC
+entry in the MADT Table. Each GICC entry part of MADT presents cpu as
+enabled or disabled. As of now, the disabled cpus are skipped as
+physical cpu hotplug is not supported. These remain disabled even after
+the kernel has booted.
+
+To support virtual cpu hotplug(in which case disabled vcpus could be
+hotplugged even after kernel has booted), QEMU will populate MADT Table
+with appropriate details of GICC entry for each possible(present+disabled)
+vcpu. Now, during the init time vcpus will be identified as present or
+disabled. To achieve this, below changes have been made with respect to
+the present/possible vcpu handling along with the mentioned reasoning:
+
+1. Identify all possible(present+disabled) vcpus at boot/init time
+   and set their present mask and possible mask. In the existing code,
+   cpus are being marked present quite late within smp_prepare_cpus()
+   function, which gets called in context to the kernel thread. Since
+   the cpu hotplug is not supported, present cpus are always equal to
+   the possible cpus. But with cpu hotplug enabled, this assumption is
+   not true. Hence, present cpus should be marked while MADT GICC entries
+   are bring parsed for each vcpu.
+2. Set possible cpus to include disabled. This needs to be done now
+   while parsing MADT GICC entries corresponding to each vcpu as the
+   disabled vcpu info is available only at this point as for hotplug
+   case possible vcpus is not equal to present vcpus.
+3. We will store the parsed madt/gicc entry even for the disabled vcpus
+   during init time. This is needed as some modules like PMU registers
+   IRQs for each possible vcpus during init time. Therefore, a valid
+   entry of the MADT GICC should be present for all possible vcpus.
+4. Refactoring related to DT/OF is also done to align it with the init
+   changes to support vcpu hotplug.
+
+Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
+Signed-off-by: Xiongfeng Wang <wangxiongfeng2@huawei.com>
+---
+ arch/arm64/kernel/smp.c | 28 +++++++++++++++++++++-------
+ 1 file changed, 21 insertions(+), 7 deletions(-)
+
+diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
+index 6f6ff072acbd..4b317e71b1c4 100644
+--- a/arch/arm64/kernel/smp.c
++++ b/arch/arm64/kernel/smp.c
+@@ -524,13 +524,12 @@ static int __init smp_cpu_setup(int cpu)
+ 	if (ops->cpu_init(cpu))
+ 		return -ENODEV;
+ 
+-	set_cpu_possible(cpu, true);
+-
+ 	return 0;
+ }
+ 
+ static bool bootcpu_valid __initdata;
+ static unsigned int cpu_count = 1;
++static unsigned int disabled_cpu_count;
+ 
+ #ifdef CONFIG_ACPI
+ static struct acpi_madt_generic_interrupt cpu_madt_gicc[NR_CPUS];
+@@ -549,10 +548,17 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)
+ static void __init
+ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ {
++	unsigned int total_cpu_count = disabled_cpu_count + cpu_count;
+ 	u64 hwid = processor->arm_mpidr;
+ 
+ 	if (!(processor->flags & ACPI_MADT_ENABLED)) {
++#ifndef CONFIG_ACPI_HOTPLUG_CPU
+ 		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
++#else
++		cpu_madt_gicc[total_cpu_count] = *processor;
++		set_cpu_possible(total_cpu_count, true);
++		disabled_cpu_count++;
++#endif
+ 		return;
+ 	}
+ 
+@@ -561,7 +567,7 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 		return;
+ 	}
+ 
+-	if (is_mpidr_duplicate(cpu_count, hwid)) {
++	if (is_mpidr_duplicate(total_cpu_count, hwid)) {
+ 		pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
+ 		return;
+ 	}
+@@ -582,9 +588,9 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 		return;
+ 
+ 	/* map the logical cpu id to cpu MPIDR */
+-	set_cpu_logical_map(cpu_count, hwid);
++	set_cpu_logical_map(total_cpu_count, hwid);
+ 
+-	cpu_madt_gicc[cpu_count] = *processor;
++	cpu_madt_gicc[total_cpu_count] = *processor;
+ 
+ 	/*
+ 	 * Set-up the ACPI parking protocol cpu entries
+@@ -595,7 +601,10 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 	 * initialize the cpu if the parking protocol is
+ 	 * the only available enable method).
+ 	 */
+-	acpi_set_mailbox_entry(cpu_count, processor);
++	acpi_set_mailbox_entry(total_cpu_count, processor);
++
++	set_cpu_possible(total_cpu_count, true);
++	set_cpu_present(total_cpu_count, true);
+ 
+ 	cpu_count++;
+ }
+@@ -629,6 +638,9 @@ static void __init acpi_parse_and_init_cpus(void)
+ 	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
+ 				      acpi_parse_gic_cpu_interface, 0);
+ 
++	pr_debug("possible cpus(%u) present cpus(%u) disabled cpus(%u)\n",
++		 cpu_count+disabled_cpu_count, cpu_count, disabled_cpu_count);
++
+ 	/*
+ 	 * In ACPI, SMP and CPU NUMA information is provided in separate
+ 	 * static tables, namely the MADT and the SRAT.
+@@ -699,6 +711,9 @@ static void __init of_parse_and_init_cpus(void)
+ 		set_cpu_logical_map(cpu_count, hwid);
+ 
+ 		early_map_cpu_to_node(cpu_count, of_node_to_nid(dn));
++
++		set_cpu_possible(cpu_count, true);
++		set_cpu_present(cpu_count, true);
+ next:
+ 		cpu_count++;
+ 	}
+@@ -783,7 +798,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
+ 		if (err)
+ 			continue;
+ 
+-		set_cpu_present(cpu, true);
+ 		numa_store_cpu_info(cpu);
+ 	}
+ }
+-- 
+2.17.1
+

tools/packaging/kernel/patches/5.15.x/arm-experimental/0002-arm64-kernel-Bound-the-total-present-disabled-cpus-w.patch

@@ -0,0 +1,86 @@
+From 2bd0439913fde8598113cc3959764a877c0bd1ad Mon Sep 17 00:00:00 2001
+From: Salil Mehta <salil.mehta@huawei.com>
+Date: Wed, 1 Dec 2021 16:01:17 +0800
+Subject: [PATCH 2/7] arm64: kernel: Bound the total(present+disabled) cpus
+ with nr_cpu_ids
+
+Bound the total number of identified cpus(including disabled cpus) by
+maximum allowed limit by the kernel. Max value is either specified as
+part of the kernel parameters 'nr_cpus' or specified during compile
+time using CONFIG_NR_CPUS.
+
+Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
+Signed-off-by: Xiongfeng Wang <wangxiongfeng2@huawei.com>
+---
+ arch/arm64/kernel/smp.c | 18 ++++++++++++------
+ 1 file changed, 12 insertions(+), 6 deletions(-)
+
+diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
+index 4b317e71b1c4..18a0576f2721 100644
+--- a/arch/arm64/kernel/smp.c
++++ b/arch/arm64/kernel/smp.c
+@@ -528,6 +528,7 @@ static int __init smp_cpu_setup(int cpu)
+ }
+ 
+ static bool bootcpu_valid __initdata;
++static bool cpus_clipped __initdata = false;
+ static unsigned int cpu_count = 1;
+ static unsigned int disabled_cpu_count;
+ 
+@@ -551,6 +552,11 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 	unsigned int total_cpu_count = disabled_cpu_count + cpu_count;
+ 	u64 hwid = processor->arm_mpidr;
+ 
++	if (total_cpu_count > nr_cpu_ids) {
++		cpus_clipped = true;
++		return;
++	}
++
+ 	if (!(processor->flags & ACPI_MADT_ENABLED)) {
+ #ifndef CONFIG_ACPI_HOTPLUG_CPU
+ 		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
+@@ -584,9 +590,6 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 		return;
+ 	}
+ 
+-	if (cpu_count >= NR_CPUS)
+-		return;
+-
+ 	/* map the logical cpu id to cpu MPIDR */
+ 	set_cpu_logical_map(total_cpu_count, hwid);
+ 
+@@ -704,8 +707,10 @@ static void __init of_parse_and_init_cpus(void)
+ 			continue;
+ 		}
+ 
+-		if (cpu_count >= NR_CPUS)
++		if (cpu_count >= NR_CPUS) {
++			cpus_clipped = true;
+ 			goto next;
++		}
+ 
+ 		pr_debug("cpu logical map 0x%llx\n", hwid);
+ 		set_cpu_logical_map(cpu_count, hwid);
+@@ -726,6 +731,7 @@ static void __init of_parse_and_init_cpus(void)
+  */
+ void __init smp_init_cpus(void)
+ {
++	unsigned int total_cpu_count = disabled_cpu_count + cpu_count;
+ 	int i;
+ 
+ 	if (acpi_disabled)
+@@ -733,9 +739,9 @@ void __init smp_init_cpus(void)
+ 	else
+ 		acpi_parse_and_init_cpus();
+ 
+-	if (cpu_count > nr_cpu_ids)
++	if (cpus_clipped)
+ 		pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
+-			cpu_count, nr_cpu_ids);
++			total_cpu_count, nr_cpu_ids);
+ 
+ 	if (!bootcpu_valid) {
+ 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
+-- 
+2.17.1
+

tools/packaging/kernel/patches/5.15.x/arm-experimental/0003-arm64-kernel-Init-cpu-operations-for-all-possible-vc.patch

@@ -0,0 +1,116 @@
+From 58ceaa003bab7d2613f01ec58925a75e1f731240 Mon Sep 17 00:00:00 2001
+From: Salil Mehta <salil.mehta@huawei.com>
+Date: Thu, 2 Dec 2021 13:57:51 +0800
+Subject: [PATCH 3/7] arm64: kernel: Init cpu operations for all possible vcpus
+
+Currently, cpu-operations are only initialized for the cpus which
+already have logical cpuid to hwid assoication established. And this
+only happens for the cpus which are present during boot time.
+
+To support virtual cpu hotplug, we shall initialze the cpu-operations
+for all possible(present+disabled) vcpus. This means logical cpuid to
+hwid/mpidr association might not exists(i.e. might be INVALID_HWID)
+during init. Later, when the vcpu is actually hotplugged logical cpuid
+is allocated and associated with the hwid/mpidr.
+
+This patch does some refactoring to support above change.
+
+Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
+Signed-off-by: Xiongfeng Wang <wangxiongfeng2@huawei.com>
+---
+ arch/arm64/kernel/smp.c | 39 +++++++++++++++------------------------
+ 1 file changed, 15 insertions(+), 24 deletions(-)
+
+diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
+index 18a0576f2721..fed4415e8cfe 100644
+--- a/arch/arm64/kernel/smp.c
++++ b/arch/arm64/kernel/smp.c
+@@ -518,13 +518,16 @@ static int __init smp_cpu_setup(int cpu)
+ 	const struct cpu_operations *ops;
+ 
+ 	if (init_cpu_ops(cpu))
+-		return -ENODEV;
++		goto out;
+ 
+ 	ops = get_cpu_ops(cpu);
+ 	if (ops->cpu_init(cpu))
+-		return -ENODEV;
++		goto out;
+ 
+ 	return 0;
++out:
++	__cpu_logical_map[cpu] = INVALID_HWID;
++	return -ENODEV;
+ }
+ 
+ static bool bootcpu_valid __initdata;
+@@ -562,7 +565,8 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
+ #else
+ 		cpu_madt_gicc[total_cpu_count] = *processor;
+-		set_cpu_possible(total_cpu_count, true);
++		if (!smp_cpu_setup(total_cpu_count))
++			set_cpu_possible(total_cpu_count, true);
+ 		disabled_cpu_count++;
+ #endif
+ 		return;
+@@ -606,9 +610,10 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
+ 	 */
+ 	acpi_set_mailbox_entry(total_cpu_count, processor);
+ 
+-	set_cpu_possible(total_cpu_count, true);
+-	set_cpu_present(total_cpu_count, true);
+-
++	if (!smp_cpu_setup(total_cpu_count)) {
++		set_cpu_possible(total_cpu_count, true);
++		set_cpu_present(total_cpu_count, true);
++	}
+ 	cpu_count++;
+ }
+ 
+@@ -716,9 +721,10 @@ static void __init of_parse_and_init_cpus(void)
+ 		set_cpu_logical_map(cpu_count, hwid);
+ 
+ 		early_map_cpu_to_node(cpu_count, of_node_to_nid(dn));
+-
+-		set_cpu_possible(cpu_count, true);
+-		set_cpu_present(cpu_count, true);
++		if (!smp_cpu_setup(cpu_count)) {
++			set_cpu_possible(cpu_count, true);
++			set_cpu_present(cpu_count, true);
++		}
+ next:
+ 		cpu_count++;
+ 	}
+@@ -732,7 +738,6 @@ static void __init of_parse_and_init_cpus(void)
+ void __init smp_init_cpus(void)
+ {
+ 	unsigned int total_cpu_count = disabled_cpu_count + cpu_count;
+-	int i;
+ 
+ 	if (acpi_disabled)
+ 		of_parse_and_init_cpus();
+@@ -747,20 +752,6 @@ void __init smp_init_cpus(void)
+ 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
+ 		return;
+ 	}
+-
+-	/*
+-	 * We need to set the cpu_logical_map entries before enabling
+-	 * the cpus so that cpu processor description entries (DT cpu nodes
+-	 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
+-	 * with entries in cpu_logical_map while initializing the cpus.
+-	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
+-	 */
+-	for (i = 1; i < nr_cpu_ids; i++) {
+-		if (cpu_logical_map(i) != INVALID_HWID) {
+-			if (smp_cpu_setup(i))
+-				set_cpu_logical_map(i, INVALID_HWID);
+-		}
+-	}
+ }
+ 
+ void __init smp_prepare_cpus(unsigned int max_cpus)
+-- 
+2.17.1
+

tools/packaging/kernel/patches/5.15.x/arm-experimental/0004-arm64-kernel-Arch-specific-ACPI-hooks-like-logical-c.patch

@@ -0,0 +1,125 @@
+From 6b7b492fc89e97e5ee51f9d033000fb6483a5298 Mon Sep 17 00:00:00 2001
+From: Salil Mehta <salil.mehta@huawei.com>
+Date: Wed, 1 Dec 2021 16:21:50 +0800
+Subject: [PATCH 4/7] arm64: kernel: Arch specific ACPI hooks(like logical
+ cpuid<->hwid etc.)
+
+To support virtual cpu hotplug, some arch specifc hooks must be
+facilitated. These hooks are called by the generic ACPI cpu hotplug
+framework during a vcpu hot-(un)plug event handling. The changes
+required involve:
+
+1. Allocation of the logical cpuid corresponding to the hwid/mpidr
+2. Mapping of logical cpuid to hwid/mpidr and marking present
+3. Removing vcpu from present mask during hot-unplug
+4. For arm64, all possible cpus are registered within topology_init()
+   Hence, we need to override the weak ACPI call of arch_register_cpu()
+   (which returns -ENODEV) and return success.
+5. NUMA node mapping set for this vcpu using SRAT Table info during init
+   time will be discarded as the logical cpu-ids used at that time
+   might not be correct. This mapping will be set again using the
+   proximity/node info obtained by evaluating _PXM ACPI method.
+
+Note, during hot unplug of vcpu, we do not unmap the association between
+the logical cpuid and hwid/mpidr. This remains persistent.
+
+Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
+Signed-off-by: Xiongfeng Wang <wangxiongfeng2@huawei.com>
+---
+ arch/arm64/kernel/smp.c | 80 +++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 80 insertions(+)
+
+diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
+index fed4415e8cfe..8ab68ec01090 100644
+--- a/arch/arm64/kernel/smp.c
++++ b/arch/arm64/kernel/smp.c
+@@ -543,6 +543,86 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)
+ 	return &cpu_madt_gicc[cpu];
+ }
+ 
++#ifdef CONFIG_ACPI_HOTPLUG_CPU
++int arch_register_cpu(int num)
++{
++	return 0;
++}
++
++static int set_numa_node_for_cpu(acpi_handle handle, int cpu)
++{
++#ifdef CONFIG_ACPI_NUMA
++	int node_id;
++
++	/* will evaluate _PXM */
++	node_id = acpi_get_node(handle);
++	if (node_id != NUMA_NO_NODE)
++		set_cpu_numa_node(cpu, node_id);
++#endif
++	return 0;
++}
++
++static void unset_numa_node_for_cpu(int cpu)
++{
++#ifdef CONFIG_ACPI_NUMA
++	set_cpu_numa_node(cpu, NUMA_NO_NODE);
++#endif
++}
++
++static int allocate_logical_cpuid(u64 physid)
++{
++	int first_invalid_idx = -1;
++	bool first = true;
++	int i;
++
++	for_each_possible_cpu(i) {
++		/*
++		 * logical cpuid<->hwid association remains persistent once
++		 * established
++		 */
++		if (cpu_logical_map(i) == physid)
++			return i;
++
++		if ((cpu_logical_map(i) == INVALID_HWID) && first) {
++			first_invalid_idx = i;
++			first = false;
++		}
++	}
++
++	return first_invalid_idx;
++}
++
++int acpi_unmap_cpu(int cpu)
++{
++	set_cpu_present(cpu, false);
++	unset_numa_node_for_cpu(cpu);
++
++	return 0;
++}
++
++int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id,
++		 int *cpuid)
++{
++	int cpu;
++
++	cpu = allocate_logical_cpuid(physid);
++	if (cpu < 0) {
++		pr_warn("Unable to map logical cpuid to physid 0x%llx\n",
++			physid);
++		return -ENOSPC;
++	}
++
++	/* map the logical cpu id to cpu MPIDR */
++	__cpu_logical_map[cpu] = physid;
++	set_numa_node_for_cpu(handle, cpu);
++
++	set_cpu_present(cpu, true);
++	*cpuid = cpu;
++
++	return 0;
++}
++#endif
++
+ /*
+  * acpi_map_gic_cpu_interface - parse processor MADT entry
+  *
+-- 
+2.17.1
+

tools/packaging/kernel/patches/5.15.x/arm-experimental/0005-cpu-numa-fix-failure-when-hot-remove-cpu.patch

@@ -0,0 +1,82 @@
+From 5c979f026c1319c712e7fa4882ec3a4ef3e2101b Mon Sep 17 00:00:00 2001
+From: Jianyong Wu <jianyong.wu@arm.com>
+Date: Fri, 3 Dec 2021 17:11:39 +0800
+Subject: [PATCH 5/7] cpu/numa: fix failure when hot-remove cpu
+
+when hot-remove cpu, the map from cpu to numa will set to NUMA_NO_NODE
+which will lead to failure as the map is used by others. thus we need a
+specific map to descrip the unpluged cpu.
+Here we introduce a new map to descrip the unpluged cpu map.
+
+Singed-off-by: Jianyong Wu <jianyong.wu@arm.com>
+---
+ arch/arm64/include/asm/smp.h |  2 ++
+ arch/arm64/kernel/setup.c    | 14 ++++++++++++++
+ arch/arm64/kernel/smp.c      |  5 ++++-
+ 3 files changed, 20 insertions(+), 1 deletion(-)
+
+diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
+index fc55f5a57a06..7949f6090eed 100644
+--- a/arch/arm64/include/asm/smp.h
++++ b/arch/arm64/include/asm/smp.h
+@@ -47,6 +47,8 @@ DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
+  */
+ extern u64 __cpu_logical_map[NR_CPUS];
+ extern u64 cpu_logical_map(unsigned int cpu);
++extern u64 get_acpicpu_numa_node(unsigned int cpu);
++extern int set_acpicpu_numa_node(unsigned int cpu, unsigned int node);
+ 
+ static inline void set_cpu_logical_map(unsigned int cpu, u64 hwid)
+ {
+diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
+index be5f85b0a24d..68d7a7894e10 100644
+--- a/arch/arm64/kernel/setup.c
++++ b/arch/arm64/kernel/setup.c
+@@ -284,6 +284,20 @@ static int __init reserve_memblock_reserved_regions(void)
+ }
+ arch_initcall(reserve_memblock_reserved_regions);
+ 
++u64 __acpicpu_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
++
++u64 get_acpicpu_numa_node(unsigned int cpu)
++{
++	return __acpicpu_node_map[cpu];
++}
++
++int set_acpicpu_numa_node(unsigned int cpu, unsigned int node)
++{
++	__acpicpu_node_map[cpu] = node;
++
++	return 0;
++}
++
+ u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
+ 
+ u64 cpu_logical_map(unsigned int cpu)
+diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
+index 8ab68ec01090..0c07921b0b61 100644
+--- a/arch/arm64/kernel/smp.c
++++ b/arch/arm64/kernel/smp.c
+@@ -557,7 +557,10 @@ static int set_numa_node_for_cpu(acpi_handle handle, int cpu)
+ 	/* will evaluate _PXM */
+ 	node_id = acpi_get_node(handle);
+ 	if (node_id != NUMA_NO_NODE)
++	{
++		set_acpicpu_numa_node(cpu, node_id);
+ 		set_cpu_numa_node(cpu, node_id);
++	}
+ #endif
+ 	return 0;
+ }
+@@ -565,7 +568,7 @@ static int set_numa_node_for_cpu(acpi_handle handle, int cpu)
+ static void unset_numa_node_for_cpu(int cpu)
+ {
+ #ifdef CONFIG_ACPI_NUMA
+-	set_cpu_numa_node(cpu, NUMA_NO_NODE);
++	set_acpicpu_numa_node(cpu, NUMA_NO_NODE);
+ #endif
+ }
+ 
+-- 
+2.17.1
+

tools/packaging/kernel/patches/5.15.x/arm-experimental/0006-arm64-mm-avoid-fixmap-race-condition-when-create-pud.patch

@@ -0,0 +1,69 @@
+From e3a11f2f7ccb0dbbb8cf95944e89b34fd928107a Mon Sep 17 00:00:00 2001
+From: Jianyong Wu <jianyong.wu@arm.com>
+Date: Mon, 6 Dec 2021 10:52:37 +0800
+Subject: [PATCH 6/7] arm64/mm: avoid fixmap race condition when create pud
+ mapping
+
+The 'fixmap' is a global resource and is used recursively by
+create pud mapping(), leading to a potential race condition in the
+presence of a concurrent call to alloc_init_pud():
+
+kernel_init thread                          virtio-mem workqueue thread
+==================                          ===========================
+
+  alloc_init_pud(...)                       alloc_init_pud(...)
+  pudp = pud_set_fixmap_offset(...)         pudp = pud_set_fixmap_offset(...)
+  READ_ONCE(*pudp)
+  pud_clear_fixmap(...)
+                                            READ_ONCE(*pudp) // CRASH!
+
+As kernel may sleep during creating pud mapping, introduce a mutex lock to
+serialise use of the fixmap entries by alloc_init_pud(). However, there is
+no need for locking in early boot stage and it doesn't work well with
+KASLR enabled when early boot. So, enable lock when system_state doesn't
+equal to "SYSTEM_BOOTING".
+
+Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
+Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
+Fixes: f4710445458c ("arm64: mm: use fixmap when creating page tables")
+Link: https://lore.kernel.org/r/20220201114400.56885-1-jianyong.wu@arm.com
+Signed-off-by: Will Deacon <will@kernel.org>
+---
+ arch/arm64/mm/mmu.c | 7 +++++++
+ 1 file changed, 7 insertions(+)
+
+diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
+index cfd9deb347c3..432fab4ce2b4 100644
+--- a/arch/arm64/mm/mmu.c
++++ b/arch/arm64/mm/mmu.c
+@@ -63,6 +63,7 @@ static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
+ static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+ 
+ static DEFINE_SPINLOCK(swapper_pgdir_lock);
++static DEFINE_SPINLOCK(fixmap_lock);
+ 
+ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+ {
+@@ -328,6 +329,11 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ 	}
+ 	BUG_ON(p4d_bad(p4d));
+ 
++	/*
++	 * We only have one fixmap entry per page-table level, so take
++	 * the fixmap lock until we're done.
++	 */
++	spin_lock(&fixmap_lock);
+ 	pudp = pud_set_fixmap_offset(p4dp, addr);
+ 	do {
+ 		pud_t old_pud = READ_ONCE(*pudp);
+@@ -358,6 +364,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ 	} while (pudp++, addr = next, addr != end);
+ 
+ 	pud_clear_fixmap();
++	spin_unlock(&fixmap_lock);
+ }
+ 
+ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
+-- 
+2.17.1
+

tools/packaging/kernel/patches/5.15.x/arm-experimental/0007-virtio-mem-enable-virtio-mem-on-arm64.patch

@@ -0,0 +1,67 @@
+From b1a3d86afbccb5485d2a53cc7e4e097a40f9d443 Mon Sep 17 00:00:00 2001
+From: Jianyong Wu <jianyong.wu@arm.com>
+Date: Tue, 14 Dec 2021 14:18:39 +0800
+Subject: [PATCH 7/7] virtio-mem: enable virtio-mem on arm64
+
+It seems that virtio-mem works on arm64 now and can be enabled.
+
+Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
+---
+ arch/arm64/mm/mmu.c    | 12 +++++++-----
+ drivers/virtio/Kconfig |  2 +-
+ 2 files changed, 8 insertions(+), 6 deletions(-)
+
+diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
+index 432fab4ce2b4..809fe52d3035 100644
+--- a/arch/arm64/mm/mmu.c
++++ b/arch/arm64/mm/mmu.c
+@@ -63,7 +63,7 @@ static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
+ static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+ 
+ static DEFINE_SPINLOCK(swapper_pgdir_lock);
+-static DEFINE_SPINLOCK(fixmap_lock);
++static DEFINE_MUTEX(fixmap_lock);
+ 
+ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+ {
+@@ -330,10 +330,11 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ 	BUG_ON(p4d_bad(p4d));
+ 
+ 	/*
+-	 * We only have one fixmap entry per page-table level, so take
+-	 * the fixmap lock until we're done.
++	 * No need for locking during early boot. And it doesn't work as
++	 * expected with KASLR enabled.
+ 	 */
+-	spin_lock(&fixmap_lock);
++	if (system_state != SYSTEM_BOOTING)
++		mutex_lock(&fixmap_lock);
+ 	pudp = pud_set_fixmap_offset(p4dp, addr);
+ 	do {
+ 		pud_t old_pud = READ_ONCE(*pudp);
+@@ -364,7 +365,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ 	} while (pudp++, addr = next, addr != end);
+ 
+ 	pud_clear_fixmap();
+-	spin_unlock(&fixmap_lock);
++	if (system_state != SYSTEM_BOOTING)
++		mutex_unlock(&fixmap_lock);
+ }
+ 
+ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
+diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
+index ce1b3f6ec325..ebabff45935c 100644
+--- a/drivers/virtio/Kconfig
++++ b/drivers/virtio/Kconfig
+@@ -96,7 +96,7 @@ config VIRTIO_BALLOON
+ config VIRTIO_MEM
+ 	tristate "Virtio mem driver"
+ 	default m
+-	depends on X86_64
++	depends on X86_64 || ARM64
+ 	depends on VIRTIO
+ 	depends on MEMORY_HOTPLUG_SPARSE
+ 	depends on MEMORY_HOTREMOVE
+-- 
+2.17.1
+

versions.yaml

@@ -75,7 +75,7 @@ assets:
       url: "https://github.com/cloud-hypervisor/cloud-hypervisor"
       uscan-url: >-
         https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz
-      version: "b0324f85571c441f840e9bdeb25410514a00bb74"
+      version: "v22.0"
 
     firecracker:
       description: "Firecracker micro-VMM"
@@ -162,6 +162,11 @@ assets:
     url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/"
     tag: "v5.13.10"
 
+  arm-kernel-experimental:
+    description: "Linux kernel with cpu/mem hotplug support on arm64"
+    url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/"
+    version: "v5.15.7"
+
 externals:
   description: "Third-party projects used by the system"