CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #6504
Signed-off-by: Megan Wright <megan.wright@ibm.com>

src/agent/Cargo.lock

@@ -641,9 +641,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",
@@ -2079,6 +2079,7 @@ dependencies = [
  "async-recursion",
  "async-trait",
  "capctl",
+ "cfg-if 1.0.0",
  "cgroups-rs",
  "clap",
  "futures",

src/agent/Cargo.toml

@@ -49,10 +49,11 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
+cfg-if = "1.0.0"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 
 # Tracing
 tracing = "0.1.26"

src/agent/rustjail/Cargo.toml

@@ -25,7 +25,7 @@ scan_fmt = "0.2.6"
 regex = "1.5.6"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 

src/agent/src/ap.rs

@@ -0,0 +1,79 @@
+// Copyright (c) IBM Corp. 2023
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+use std::fmt;
+use std::str::FromStr;
+
+use anyhow::{anyhow, Context};
+
+// IBM Adjunct Processor (AP) is used for cryptographic operations
+// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x).
+// In Linux, virtual cryptographic devices are called AP queues.
+// The name of an AP queue respects a format <xx>.<xxxx> in hexadecimal notation [1, p.467]:
+//   - <xx> is an adapter ID
+//   - <xxxx> is an adapter domain ID
+// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf
+
+#[derive(Debug)]
+pub struct Address {
+    pub adapter_id: u8,
+    pub adapter_domain: u16,
+}
+
+impl Address {
+    pub fn new(adapter_id: u8, adapter_domain: u16) -> Address {
+        Address {
+            adapter_id,
+            adapter_domain,
+        }
+    }
+}
+
+impl FromStr for Address {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> anyhow::Result<Self> {
+        let split: Vec<&str> = s.split('.').collect();
+        if split.len() != 2 {
+            return Err(anyhow!(
+                "Wrong AP bus format. It needs to be in the form <xx>.<xxxx> (e.g. 0a.003f), got {:?}",
+                s
+            ));
+        }
+
+        let adapter_id = u8::from_str_radix(split[0], 16).context(format!(
+            "Wrong AP bus format. AP ID needs to be in the form <xx> (e.g. 0a), got {:?}",
+            split[0]
+        ))?;
+        let adapter_domain = u16::from_str_radix(split[1], 16).context(format!(
+            "Wrong AP bus format. AP domain needs to be in the form <xxxx> (e.g. 003f), got {:?}",
+            split[1]
+        ))?;
+
+        Ok(Address::new(adapter_id, adapter_domain))
+    }
+}
+
+impl fmt::Display for Address {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_from_str() {
+        let device = Address::from_str("a.1").unwrap();
+        assert_eq!(format!("{}", device), "0a.0001");
+
+        assert!(Address::from_str("").is_err());
+        assert!(Address::from_str(".").is_err());
+        assert!(Address::from_str("0.0.0").is_err());
+        assert!(Address::from_str("0g.0000").is_err());
+        assert!(Address::from_str("0a.10000").is_err());
+    }
+}

src/agent/src/device.rs

@@ -16,13 +16,12 @@ use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 
-#[cfg(target_arch = "s390x")]
-use crate::ccw;
 use crate::linux_abi::*;
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
 use anyhow::{anyhow, Context, Result};
+use cfg_if::cfg_if;
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
@@ -46,14 +45,22 @@ pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm";
 pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral";
 pub const DRIVER_LOCAL_TYPE: &str = "local";
 pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind";
-// VFIO device to be bound to a guest kernel driver
-pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
-// VFIO device to be bound to vfio-pci and made available inside the
+// VFIO PCI device to be bound to a guest kernel driver
+pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk";
+// VFIO PCI device to be bound to vfio-pci and made available inside the
 // container as a VFIO device node
-pub const DRIVER_VFIO_TYPE: &str = "vfio";
+pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci";
+pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap";
 pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs";
 pub const FS_TYPE_HUGETLB: &str = "hugetlbfs";
 
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        use crate::ap;
+        use crate::ccw;
+    }
+}
+
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
@@ -280,7 +287,7 @@ pub async fn get_virtio_blk_ccw_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     device: &ccw::Device,
 ) -> Result<String> {
-    let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device);
+    let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device);
     let uev = wait_for_uevent(sandbox, matcher).await?;
     let devname = uev.devname;
     return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() {
@@ -401,6 +408,39 @@ async fn get_vfio_device_name(sandbox: &Arc<Mutex<Sandbox>>, grp: IommuGroup) ->
     Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
+#[cfg(target_arch = "s390x")]
+#[derive(Debug)]
+struct ApMatcher {
+    syspath: String,
+}
+
+#[cfg(target_arch = "s390x")]
+impl ApMatcher {
+    fn new(address: ap::Address) -> ApMatcher {
+        ApMatcher {
+            syspath: format!(
+                "{}/card{:02x}/{}",
+                AP_ROOT_BUS_PATH, address.adapter_id, address
+            ),
+        }
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+impl UeventMatcher for ApMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.action == "add" && uev.devpath == self.syspath
+    }
+}
+
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address) -> Result<()> {
+    let matcher = ApMatcher::new(address);
+    wait_for_uevent(sandbox, matcher).await?;
+    Ok(())
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -701,7 +741,7 @@ async fn virtio_nvdimm_device_handler(
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
-fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
+fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> {
     let mut tokens = opt.split('=');
     let hostbdf = tokens.next()?;
     let path = tokens.next()?;
@@ -716,14 +756,18 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
-    let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
+#[instrument]
+async fn vfio_pci_device_handler(
+    device: &Device,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<SpecUpdate> {
+    let vfio_in_guest = device.field_type != DRIVER_VFIO_PCI_GK_TYPE;
     let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
     let mut group = None;
 
     for opt in device.options.iter() {
-        let (host, pcipath) =
-            split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
+        let (host, pcipath) = split_vfio_pci_option(opt)
+            .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?;
         let host =
             pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
         let pcipath = pci::Path::from_str(pcipath)?;
@@ -765,6 +809,28 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) ->
     })
 }
 
+// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options
+// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that
+// amount to the hotplug timeout.
+#[cfg(target_arch = "s390x")]
+#[instrument]
+async fn vfio_ap_device_handler(
+    device: &Device,
+    sandbox: &Arc<Mutex<Sandbox>>,
+) -> Result<SpecUpdate> {
+    // Force AP bus rescan
+    fs::write(AP_SCANS_PATH, "1")?;
+    for apqn in device.options.iter() {
+        wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?;
+    }
+    Ok(Default::default())
+}
+
+#[cfg(not(target_arch = "s390x"))]
+async fn vfio_ap_device_handler(_: &Device, _: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+    Err(anyhow!("AP is only supported on s390x"))
+}
+
 #[instrument]
 pub async fn add_devices(
     devices: &[Device],
@@ -830,7 +896,10 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
         DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
         DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
         DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
-        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await,
+        DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => {
+            vfio_pci_device_handler(device, sandbox).await
+        }
+        DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await,
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -1380,7 +1449,7 @@ mod tests {
     #[cfg(target_arch = "s390x")]
     #[tokio::test]
     async fn test_virtio_blk_ccw_matcher() {
-        let root_bus = create_ccw_root_bus_path();
+        let root_bus = CCW_ROOT_BUS_PATH;
         let subsystem = "block";
         let devname = "vda";
         let relpath = "0.0.0002";
@@ -1489,13 +1558,13 @@ mod tests {
     }
 
     #[test]
-    fn test_split_vfio_option() {
+    fn test_split_vfio_pci_option() {
         assert_eq!(
-            split_vfio_option("0000:01:00.0=02/01"),
+            split_vfio_pci_option("0000:01:00.0=02/01"),
             Some(("0000:01:00.0", "02/01"))
         );
-        assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None);
-        assert_eq!(split_vfio_option("0000:01:00.0"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None);
+        assert_eq!(split_vfio_pci_option("0000:01:00.0"), None);
     }
 
     #[test]
@@ -1574,4 +1643,35 @@ mod tests {
         // Test dev2
         assert!(pci_iommu_group(&syspci, dev2).is_err());
     }
+
+    #[cfg(target_arch = "s390x")]
+    #[tokio::test]
+    async fn test_vfio_ap_matcher() {
+        let subsystem = "ap";
+        let card = "0a";
+        let relpath = format!("{}.0001", card);
+
+        let mut uev = Uevent::default();
+        uev.action = U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = subsystem.to_string();
+        uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath);
+
+        let ap_address = ap::Address::from_str(&relpath).unwrap();
+        let matcher = ApMatcher::new(ap_address);
+
+        assert!(matcher.is_match(&uev));
+
+        let mut uev_remove = uev.clone();
+        uev_remove.action = U_EVENT_ACTION_REMOVE.to_string();
+        assert!(!matcher.is_match(&uev_remove));
+
+        let mut uev_other_device = uev.clone();
+        uev_other_device.devpath = format!(
+            "{}/card{}/{}",
+            AP_ROOT_BUS_PATH,
+            card,
+            format!("{}.0002", card)
+        );
+        assert!(!matcher.is_match(&uev_other_device));
+    }
 }

src/agent/src/linux_abi.rs

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use cfg_if::cfg_if;
+
 /// Linux ABI related constants.
 
 #[cfg(target_arch = "aarch64")]
@@ -64,10 +66,14 @@ pub fn create_pci_root_bus_path() -> String {
     ret
 }
 
-#[cfg(target_arch = "s390x")]
-pub fn create_ccw_root_bus_path() -> String {
-    String::from("/devices/css0")
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0";
+        pub const AP_ROOT_BUS_PATH: &str = "/devices/ap";
+        pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans";
+    }
 }
+
 // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 // The Linux kernel's core ACPI subsystem creates struct acpi_device
 // objects for ACPI namespace objects representing devices, power resources

src/agent/src/main.rs

@@ -20,6 +20,7 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
+use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
@@ -34,8 +35,6 @@ use std::process::exit;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
-#[cfg(target_arch = "s390x")]
-mod ccw;
 mod config;
 mod console;
 mod device;
@@ -75,6 +74,13 @@ mod image_rpc;
 mod rpc;
 mod tracer;
 
+cfg_if! {
+    if #[cfg(target_arch = "s390x")] {
+        mod ap;
+        mod ccw;
+    }
+}
+
 const NAME: &str = "kata-agent";
 
 lazy_static! {

src/agent/src/mount.rs

@@ -240,6 +240,70 @@ async fn ephemeral_storage_handler(
     Ok("".to_string())
 }
 
+// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them
+// with mount options passed by the caller
+#[instrument]
+pub async fn update_ephemeral_mounts(
+    logger: Logger,
+    storages: Vec<Storage>,
+    sandbox: Arc<Mutex<Sandbox>>,
+) -> Result<()> {
+    for (_, storage) in storages.iter().enumerate() {
+        let handler_name = storage.driver.clone();
+        let logger = logger.new(o!(
+	    "msg" => "updating tmpfs storage",
+            "subsystem" => "storage",
+            "storage-type" => handler_name.to_owned()));
+
+        match handler_name.as_str() {
+            DRIVER_EPHEMERAL_TYPE => {
+                fs::create_dir_all(Path::new(&storage.mount_point))?;
+
+                if storage.options.is_empty() {
+                    continue;
+                } else {
+                    // assume that fsGid has already been set
+                    let mut opts = Vec::<&str>::new();
+                    for (_, opt) in storage.options.iter().enumerate() {
+                        if opt.starts_with(FS_GID) {
+                            continue;
+                        }
+                        opts.push(opt)
+                    }
+                    let mount_path = Path::new(&storage.mount_point);
+                    let src_path = Path::new(&storage.source);
+
+                    let (flags, options) = parse_mount_flags_and_options(opts);
+
+                    info!(logger, "mounting storage";
+                    "mount-source" => src_path.display(),
+                    "mount-destination" => mount_path.display(),
+                    "mount-fstype"  => storage.fstype.as_str(),
+                    "mount-options" => options.as_str(),
+                    );
+
+                    return baremount(
+                        src_path,
+                        mount_path,
+                        storage.fstype.as_str(),
+                        flags,
+                        options.as_str(),
+                        &logger,
+                    );
+                }
+            }
+            _ => {
+                return Err(anyhow!(
+                    "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported",
+                    storage.driver.to_owned()
+                ));
+            }
+        };
+    }
+
+    Ok(())
+}
+
 #[instrument]
 async fn overlayfs_storage_handler(
     logger: &Logger,

src/agent/src/rpc.rs

@@ -55,7 +55,7 @@ use crate::device::{
 use crate::image_rpc;
 use crate::linux_abi::*;
 use crate::metrics::get_metrics;
-use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST};
+use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST};
 use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
 use crate::network::setup_guest_dns;
 use crate::pci;
@@ -1115,6 +1115,23 @@ impl agent_ttrpc::AgentService for AgentService {
         })
     }
 
+    async fn update_ephemeral_mounts(
+        &self,
+        ctx: &TtrpcContext,
+        req: protocols::agent::UpdateEphemeralMountsRequest,
+    ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "update_mounts", req);
+        is_allowed!(req);
+
+        match update_ephemeral_mounts(sl!(), req.storages.to_vec(), self.sandbox.clone()).await {
+            Ok(_) => Ok(Empty::new()),
+            Err(e) => Err(ttrpc_error!(
+                ttrpc::Code::INTERNAL,
+                format!("Failed to update mounts: {:?}", e),
+            )),
+        }
+    }
+
     async fn get_ip_tables(
         &self,
         ctx: &TtrpcContext,

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -776,3 +776,587 @@ impl Default for BlockDeviceMgr {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use test_utils::skip_if_not_root;
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+    use crate::test_utils::tests::create_vm_for_test;
+
+    #[test]
+    fn test_block_device_type() {
+        let dev_type = BlockDeviceType::get_type("spool:/device1");
+        assert_eq!(dev_type, BlockDeviceType::Spool);
+        let dev_type = BlockDeviceType::get_type("/device1");
+        assert_eq!(dev_type, BlockDeviceType::RawBlock);
+    }
+
+    #[test]
+    fn test_create_block_devices_configs() {
+        let mgr = BlockDeviceMgr::default();
+        assert!(!mgr.has_root_block_device());
+        assert!(!mgr.has_part_uuid_root());
+        assert!(!mgr.is_read_only_root());
+        assert_eq!(mgr.get_index_of_drive_id(""), None);
+        assert_eq!(mgr.info_list.len(), 0);
+    }
+
+    #[test]
+    fn test_add_non_root_block_device() {
+        skip_if_not_root!();
+        let dummy_file = TempFile::new().unwrap();
+        let dummy_path = dummy_file.as_path().to_owned();
+        let dummy_id = String::from("1");
+        let dummy_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path.clone(),
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: dummy_id.clone(),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(BlockDeviceMgr::insert_device(
+            vm.device_manager_mut(),
+            ctx,
+            dummy_block_device.clone(),
+        )
+        .is_ok());
+
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+        assert!(!vm.device_manager().block_manager.has_root_block_device());
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
+        assert!(!vm.device_manager().block_manager.is_read_only_root());
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+        assert_eq!(
+            vm.device_manager().block_manager.info_list[0]
+                .config
+                .device_type(),
+            BlockDeviceType::RawBlock
+        );
+        assert_eq!(
+            vm.device_manager().block_manager.info_list[0]
+                .config
+                .queue_sizes(),
+            [128u16]
+        );
+
+        let dev_config = vm.device_manager().block_manager.iter().next().unwrap();
+        assert_eq!(dev_config.config, dummy_block_device);
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_path(&dummy_path)
+            .is_some());
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&dummy_id)
+            .is_some());
+    }
+
+    #[test]
+    fn test_update_blk_device_ratelimiters() {
+        skip_if_not_root!();
+        //Init vm for test.
+        let mut vm = create_vm_for_test();
+        let device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+        );
+
+        let dummy_file = TempFile::new().unwrap();
+        let dummy_path = dummy_file.as_path().to_owned();
+
+        let dummy_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: true,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device)
+            .unwrap();
+
+        let cfg = BlockDeviceConfigUpdateInfo {
+            drive_id: String::from("1"),
+            rate_limiter: None,
+        };
+
+        let mut device_op_ctx = DeviceOpContext::new(
+            Some(vm.epoll_manager().clone()),
+            vm.device_manager(),
+            Some(vm.vm_as().unwrap().clone()),
+            None,
+            false,
+        );
+
+        vm.device_manager_mut()
+            .block_manager
+            .attach_devices(&mut device_op_ctx)
+            .unwrap();
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+
+        //Patch while the epoll handler is invalid.
+        let expected_error = "could not send patch message to the block epoll handler".to_string();
+
+        assert_eq!(
+            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg)
+                .unwrap_err()
+                .to_string(),
+            expected_error
+        );
+
+        //Invalid drive id
+        let cfg2 = BlockDeviceConfigUpdateInfo {
+            drive_id: String::from("2"),
+            rate_limiter: None,
+        };
+
+        let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id);
+
+        assert_eq!(
+            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2)
+                .unwrap_err()
+                .to_string(),
+            expected_error
+        );
+    }
+
+    #[test]
+    fn test_add_one_root_block_device() {
+        skip_if_not_root!();
+        let dummy_file = TempFile::new().unwrap();
+        let dummy_path = dummy_file.as_path().to_owned();
+        let dummy_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: true,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(BlockDeviceMgr::insert_device(
+            vm.device_manager_mut(),
+            ctx,
+            dummy_block_device.clone(),
+        )
+        .is_ok());
+
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+        assert!(vm.device_manager().block_manager.has_root_block);
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root);
+        assert!(vm.device_manager().block_manager.read_only_root);
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
+
+        let dev_config = vm.device_manager().block_manager.iter().next().unwrap();
+        assert_eq!(dev_config.config, dummy_block_device);
+        assert!(vm.device_manager().block_manager.is_read_only_root());
+    }
+
+    #[test]
+    fn test_add_two_root_block_devices_configs() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device_1 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let root_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(
+            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2)
+                .is_err()
+        );
+    }
+
+    #[test]
+    // Test BlockDevicesConfigs::add when you first add the root device and then the other devices.
+    fn test_add_root_block_device_first() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let dummy_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_3 = TempFile::new().unwrap();
+        let dummy_path_3 = dummy_file_3.as_path().to_owned();
+        let dummy_block_device_3 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_3,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("3"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+        vm.device_manager_mut()
+            .block_manager
+            .create(root_block_device.clone())
+            .unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .create(dummy_block_device_2.clone())
+            .unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .create(dummy_block_device_3.clone())
+            .unwrap();
+
+        assert!(vm.device_manager().block_manager.has_root_block_device(),);
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap();
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap();
+    }
+
+    #[test]
+    // Test BlockDevicesConfigs::add when you add other devices first and then the root device.
+    fn test_root_block_device_add_last() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let dummy_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_3 = TempFile::new().unwrap();
+        let dummy_path_3 = dummy_file_3.as_path().to_owned();
+        let dummy_block_device_3 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_3,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("3"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+            .unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone())
+            .unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone())
+            .unwrap();
+
+        assert!(vm.device_manager().block_manager.has_root_block_device(),);
+        assert!(!vm.device_manager().block_manager.has_part_uuid_root());
+        assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
+
+        let mut block_dev_iter = vm.device_manager().block_manager.iter();
+        // The root device should be first in the list no matter of the order in
+        // which the devices were added.
+        assert_eq!(
+            block_dev_iter.next().unwrap().config.drive_id,
+            root_block_device.drive_id
+        );
+        assert_eq!(
+            block_dev_iter.next().unwrap().config.drive_id,
+            dummy_block_device_2.drive_id
+        );
+        assert_eq!(
+            block_dev_iter.next().unwrap().config.drive_id,
+            dummy_block_device_3.drive_id
+        );
+    }
+
+    #[test]
+    fn test_block_device_update() {
+        skip_if_not_root!();
+        let dummy_file_1 = TempFile::new().unwrap();
+        let dummy_path_1 = dummy_file_1.as_path().to_owned();
+        let root_block_device = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1.clone(),
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let dummy_file_2 = TempFile::new().unwrap();
+        let dummy_path_2 = dummy_file_2.as_path().to_owned();
+        let mut dummy_block_device_2 = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2.clone(),
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+
+        let mut vm = crate::vm::tests::create_vm_instance();
+
+        // Add 2 block devices.
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+            .unwrap();
+
+        // Get index zero.
+        assert_eq!(
+            vm.device_manager()
+                .block_manager
+                .get_index_of_drive_id(&String::from("1"))
+                .unwrap(),
+            0
+        );
+
+        // Get None.
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&String::from("foo"))
+            .is_none());
+
+        // Test several update cases using dummy_block_device_2.
+        // Validate `dummy_block_device_2` is already in the list
+        assert!(vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&dummy_block_device_2.drive_id)
+            .is_some());
+        // Update OK.
+        dummy_block_device_2.is_read_only = true;
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+            .unwrap();
+
+        let index = vm
+            .device_manager()
+            .block_manager
+            .get_index_of_drive_id(&dummy_block_device_2.drive_id)
+            .unwrap();
+        // Validate update was successful.
+        assert!(
+            vm.device_manager().block_manager.info_list[index]
+                .config
+                .is_read_only
+        );
+
+        // Update with invalid path.
+        let dummy_filename_3 = String::from("test_update_3");
+        let dummy_path_3 = PathBuf::from(dummy_filename_3);
+        dummy_block_device_2.path_on_host = dummy_path_3;
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(BlockDeviceMgr::insert_device(
+            vm.device_manager_mut(),
+            ctx,
+            dummy_block_device_2.clone(),
+        )
+        .is_err());
+
+        // Update with 2 root block devices.
+        dummy_block_device_2.path_on_host = dummy_path_2.clone();
+        dummy_block_device_2.is_root_device = true;
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        assert!(
+            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,)
+                .is_err(),
+        );
+
+        // Switch roots and add a PARTUUID for the new one.
+        let root_block_device_old = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_1,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: false,
+            part_uuid: None,
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("1"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        let root_block_device_new = BlockDeviceConfigInfo {
+            path_on_host: dummy_path_2,
+            device_type: BlockDeviceType::RawBlock,
+            is_root_device: true,
+            part_uuid: Some("0eaa91a0-01".to_string()),
+            is_read_only: false,
+            is_direct: false,
+            no_drop: false,
+            drive_id: String::from("2"),
+            rate_limiter: None,
+            num_queues: BlockDeviceConfigInfo::default_num_queues(),
+            queue_size: 128,
+            use_shared_irq: None,
+            use_generic_irq: None,
+        };
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap();
+        let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
+        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap();
+        assert!(vm.device_manager().block_manager.has_part_uuid_root);
+    }
+}

src/libs/Cargo.lock

@@ -110,9 +110,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/libs/kata-sys-util/Cargo.toml

@@ -12,7 +12,7 @@ edition = "2018"
 
 [dependencies]
 byteorder = "1.4.3"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 chrono = "0.4.0"
 common-path = "=1.0.0"
 fail = "0.5.0"

src/libs/kata-types/src/config/agent.rs

@@ -9,14 +9,16 @@ use crate::config::{ConfigOps, TomlConfig};
 
 pub use vendor::AgentVendor;
 
-use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT};
+use super::default::{
+    DEFAULT_AGENT_DIAL_TIMEOUT_MS, DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT,
+};
 use crate::eother;
 
 /// agent name of Kata agent.
 pub const AGENT_NAME_KATA: &str = "kata";
 
 /// Kata agent configuration information.
-#[derive(Debug, Default, Deserialize, Serialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct Agent {
     /// If enabled, the agent will log additional debug messages to the system log.
     #[serde(default, rename = "enable_debug")]
@@ -81,6 +83,24 @@ pub struct Agent {
     pub container_pipe_size: u32,
 }
 
+impl std::default::Default for Agent {
+    fn default() -> Self {
+        Self {
+            debug: true,
+            enable_tracing: false,
+            debug_console_enabled: false,
+            server_port: DEFAULT_AGENT_VSOCK_PORT,
+            log_port: DEFAULT_AGENT_LOG_PORT,
+            dial_timeout_ms: DEFAULT_AGENT_DIAL_TIMEOUT_MS,
+            reconnect_timeout_ms: 3_000,
+            request_timeout_ms: 30_000,
+            health_check_request_timeout_ms: 90_000,
+            kernel_modules: Default::default(),
+            container_pipe_size: 0,
+        }
+    }
+}
+
 fn default_server_port() -> u32 {
     DEFAULT_AGENT_VSOCK_PORT
 }

src/libs/kata-types/src/config/default.rs

@@ -24,6 +24,7 @@ pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024;
 pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025;
 pub const DEFAULT_AGENT_DBG_CONSOLE_PORT: u32 = 1026;
 pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA;
+pub const DEFAULT_AGENT_DIAL_TIMEOUT_MS: u32 = 10;
 
 pub const DEFAULT_RUNTIME_NAME: &str = RUNTIME_NAME_VIRTCONTAINER;
 pub const DEFAULT_HYPERVISOR: &str = HYPERVISOR_NAME_DRAGONBALL;

src/libs/protocols/protos/agent.proto

@@ -34,6 +34,7 @@ service AgentService {
 	rpc SignalProcess(SignalProcessRequest) returns (google.protobuf.Empty);
 	rpc WaitProcess(WaitProcessRequest) returns (WaitProcessResponse); // wait & reap like waitpid(2)
 	rpc UpdateContainer(UpdateContainerRequest) returns (google.protobuf.Empty);
+	rpc UpdateEphemeralMounts(UpdateEphemeralMountsRequest) returns (google.protobuf.Empty);
 	rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse);
 	rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty);
 	rpc ResumeContainer(ResumeContainerRequest) returns (google.protobuf.Empty);
@@ -316,6 +317,10 @@ message UpdateRoutesRequest {
 	Routes routes = 1;
 }
 
+message UpdateEphemeralMountsRequest {
+	repeated Storage storages = 1;
+}
+
 message ListInterfacesRequest {
 }
 

src/libs/shim-interface/src/shim_mgmt/client.rs

@@ -53,7 +53,7 @@ impl MgmtClient {
             .method(Method::GET)
             .uri(url)
             .body(Body::empty())?;
-        return self.send_request(req).await;
+        self.send_request(req).await
     }
 
     /// The HTTP Post method for client
@@ -72,7 +72,7 @@ impl MgmtClient {
             .uri(url)
             .header("content-type", content_type)
             .body(body)?;
-        return self.send_request(req).await;
+        self.send_request(req).await
     }
 
     /// The http PUT method for client
@@ -82,7 +82,7 @@ impl MgmtClient {
             .method(Method::PUT)
             .uri(url)
             .body(Body::from(data))?;
-        return self.send_request(req).await;
+        self.send_request(req).await
     }
 
     async fn send_request(&self, req: Request<Body>) -> Result<Response<Body>> {

src/runtime-rs/Cargo.lock

@@ -402,9 +402,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",
@@ -2452,6 +2452,7 @@ dependencies = [
 name = "runtimes"
 version = "0.1.0"
 dependencies = [
+ "agent",
  "anyhow",
  "common",
  "hyper",
@@ -2470,6 +2471,7 @@ dependencies = [
  "slog",
  "slog-scope",
  "tokio",
+ "url",
  "virt_container",
  "wasm_container",
 ]

src/runtime-rs/Makefile

@@ -203,7 +203,7 @@ ifneq (,$(DBCMD))
     DEFNETWORKMODEL_DB := tcfilter
     KERNELPARAMS = console=ttyS1 agent.log_vport=1025
 	KERNELTYPE_DB = uncompressed
-    KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_DB))
+    KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME_DB,$(KERNELTYPE_DB))
     KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB)
     DEFSANDBOXCGROUPONLY = true
     RUNTIMENAME := virt_container
@@ -371,8 +371,8 @@ endef
 
 # Returns the name of the kernel file to use based on the provided KERNELTYPE.
 # $1 : KERNELTYPE (compressed or uncompressed)
-define MAKE_KERNEL_NAME
-$(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container)
+define MAKE_KERNEL_NAME_DB
+$(if $(findstring uncompressed,$1),vmlinux-dragonball-experimental.container,vmlinuz-dragonball-experimental.container)
 endef
 
 .DEFAULT_GOAL := default

src/runtime-rs/crates/agent/src/kata/agent.rs

@@ -115,5 +115,7 @@ impl_agent!(
     copy_file | crate::CopyFileRequest | crate::Empty | None,
     get_oom_event | crate::Empty | crate::OomEventResponse | Some(0),
     get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None,
-    set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None
+    set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None,
+    get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None,
+    resize_volume | crate::ResizeVolumeRequest | crate::Empty | None
 );

src/runtime-rs/crates/agent/src/kata/trans.rs

@@ -8,7 +8,7 @@ use std::convert::Into;
 
 use protocols::{
     agent::{self, OOMEvent},
-    empty, health, types,
+    csi, empty, health, types,
 };
 
 use crate::{
@@ -20,11 +20,12 @@ use crate::{
         GetIPTablesResponse, GuestDetailsResponse, HealthCheckResponse, HugetlbStats, IPAddress,
         IPFamily, Interface, Interfaces, KernelModule, MemHotplugByProbeRequest, MemoryData,
         MemoryStats, NetworkStats, OnlineCPUMemRequest, PidsStats, ReadStreamRequest,
-        ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes,
-        SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest,
-        StatsContainerResponse, Storage, StringUser, ThrottlingData, TtyWinResizeRequest,
-        UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse,
-        WaitProcessRequest, WriteStreamRequest,
+        ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, ResizeVolumeRequest,
+        Route, Routes, SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse,
+        SignalProcessRequest, StatsContainerResponse, Storage, StringUser, ThrottlingData,
+        TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest,
+        VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest,
+        WriteStreamRequest,
     },
     OomEventResponse, WaitProcessResponse, WriteStreamResponse,
 };
@@ -846,3 +847,35 @@ impl From<agent::OOMEvent> for OomEventResponse {
         }
     }
 }
+
+impl From<VolumeStatsRequest> for agent::VolumeStatsRequest {
+    fn from(from: VolumeStatsRequest) -> Self {
+        Self {
+            volume_guest_path: from.volume_guest_path,
+            unknown_fields: Default::default(),
+            cached_size: Default::default(),
+        }
+    }
+}
+
+impl From<csi::VolumeStatsResponse> for VolumeStatsResponse {
+    fn from(from: csi::VolumeStatsResponse) -> Self {
+        let result: String = format!(
+            "Usage: {:?} Volume Condition: {:?}",
+            from.get_usage(),
+            from.get_volume_condition()
+        );
+        Self { data: result }
+    }
+}
+
+impl From<ResizeVolumeRequest> for agent::ResizeVolumeRequest {
+    fn from(from: ResizeVolumeRequest) -> Self {
+        Self {
+            volume_guest_path: from.volume_guest_path,
+            size: from.size,
+            unknown_fields: Default::default(),
+            cached_size: Default::default(),
+        }
+    }
+}

src/runtime-rs/crates/agent/src/lib.rs

@@ -23,8 +23,8 @@ pub use types::{
     ReseedRandomDevRequest, ResizeVolumeRequest, Route, Routes, SetGuestDateTimeRequest,
     SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, StatsContainerResponse, Storage,
     TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest,
-    VersionCheckResponse, WaitProcessRequest, WaitProcessResponse, WriteStreamRequest,
-    WriteStreamResponse,
+    VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest,
+    WaitProcessResponse, WriteStreamRequest, WriteStreamResponse,
 };
 
 use anyhow::Result;
@@ -88,4 +88,6 @@ pub trait Agent: AgentManager + HealthService + Send + Sync {
     async fn get_oom_event(&self, req: Empty) -> Result<OomEventResponse>;
     async fn get_ip_tables(&self, req: GetIPTablesRequest) -> Result<GetIPTablesResponse>;
     async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result<SetIPTablesResponse>;
+    async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result<VolumeStatsResponse>;
+    async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result<Empty>;
 }

src/runtime-rs/crates/agent/src/types.rs

@@ -9,6 +9,8 @@ use std::convert::TryFrom;
 
 use serde::{Deserialize, Serialize};
 
+pub const DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT: u32 = 10;
+
 #[derive(PartialEq, Clone, Default)]
 pub struct Empty {}
 
@@ -164,7 +166,7 @@ impl ContainerProcessID {
     }
 }
 
-#[derive(PartialEq, Clone, Debug, Default)]
+#[derive(PartialEq, Clone, Debug)]
 pub struct RemoveContainerRequest {
     pub container_id: String,
     pub timeout: u32,
@@ -179,6 +181,15 @@ impl RemoveContainerRequest {
     }
 }
 
+impl std::default::Default for RemoveContainerRequest {
+    fn default() -> Self {
+        Self {
+            container_id: "".to_string(),
+            timeout: DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT,
+        }
+    }
+}
+
 #[derive(PartialEq, Clone, Default)]
 pub struct SignalProcessRequest {
     pub process_id: ContainerProcessID,
@@ -568,6 +579,16 @@ pub struct ResizeVolumeRequest {
     pub size: u64,
 }
 
+#[derive(PartialEq, Clone, Default, Debug)]
+pub struct VolumeStatsRequest {
+    pub volume_guest_path: String,
+}
+
+#[derive(PartialEq, Clone, Default, Debug)]
+pub struct VolumeStatsResponse {
+    pub data: String,
+}
+
 #[cfg(test)]
 mod test {
     use std::convert::TryFrom;

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -473,7 +473,20 @@ impl CloudHypervisorInner {
     }
 
     pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
-        todo!()
+        if let Some(pid) = self.pid {
+            Ok(pid)
+        } else {
+            Err(anyhow!("could not get vmm master tid"))
+        }
+    }
+
+    pub(crate) async fn get_ns_path(&self) -> Result<String> {
+        if let Some(pid) = self.pid {
+            let ns_path = format!("/proc/{}/ns", pid);
+            Ok(ns_path)
+        } else {
+            Err(anyhow!("could not get ns path"))
+        }
     }
 
     pub(crate) async fn check(&self) -> Result<()> {

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -123,6 +123,11 @@ impl Hypervisor for CloudHypervisor {
         inner.get_vmm_master_tid().await
     }
 
+    async fn get_ns_path(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_ns_path().await
+    }
+
     async fn check(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.check().await

src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs

@@ -132,6 +132,11 @@ impl DragonballInner {
         Ok(master_tid)
     }
 
+    pub(crate) async fn get_ns_path(&self) -> Result<String> {
+        let ns_path = self.vmm_instance.get_ns_path();
+        Ok(ns_path)
+    }
+
     pub(crate) async fn check(&self) -> Result<()> {
         Ok(())
     }

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -122,6 +122,11 @@ impl Hypervisor for Dragonball {
         inner.get_vmm_master_tid().await
     }
 
+    async fn get_ns_path(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_ns_path().await
+    }
+
     async fn check(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.check().await

src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs

@@ -81,6 +81,13 @@ impl VmmInstance {
         result
     }
 
+    pub fn get_ns_path(&self) -> String {
+        let info_binding = self.vmm_shared_info.clone();
+        let info = info_binding.read().unwrap();
+        let result = format!("/proc/{}/task/{}/ns", info.pid, info.master_tid);
+        result
+    }
+
     pub fn get_vcpu_tids(&self) -> Vec<(u8, u32)> {
         let info = self.vmm_shared_info.clone();
         let result = info.read().unwrap().tids.clone();

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -88,6 +88,7 @@ pub trait Hypervisor: Send + Sync {
     async fn get_thread_ids(&self) -> Result<VcpuThreadIds>;
     async fn get_pids(&self) -> Result<Vec<u32>>;
     async fn get_vmm_master_tid(&self) -> Result<u32>;
+    async fn get_ns_path(&self) -> Result<String>;
     async fn cleanup(&self) -> Result<()>;
     async fn check(&self) -> Result<()>;
     async fn get_jailer_root(&self) -> Result<String>;

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -94,6 +94,11 @@ impl QemuInner {
         todo!()
     }
 
+    pub(crate) async fn get_ns_path(&self) -> Result<String> {
+        info!(sl!(), "QemuInner::get_ns_path()");
+        todo!()
+    }
+
     pub(crate) async fn cleanup(&self) -> Result<()> {
         info!(sl!(), "QemuInner::cleanup()");
         todo!()

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -108,6 +108,11 @@ impl Hypervisor for Qemu {
         inner.get_vmm_master_tid().await
     }
 
+    async fn get_ns_path(&self) -> Result<String> {
+        let inner = self.inner.read().await;
+        inner.get_ns_path().await
+    }
+
     async fn cleanup(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.cleanup().await

src/runtime-rs/crates/resource/Cargo.toml

@@ -14,7 +14,7 @@ anyhow = "^1.0"
 async-trait = "0.1.48"
 bitflags = "1.2.1"
 byte-unit = "4.0.14"
-cgroups-rs = "0.3.1"
+cgroups-rs = "0.3.2"
 futures = "0.3.11"
 lazy_static = "1.4.0"
 libc = ">=0.2.39"

src/runtime-rs/crates/runtimes/Cargo.toml

@@ -15,7 +15,9 @@ hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
 hyperlocal = "0.8"
 serde_json = "1.0.88"
 nix = "0.25.0"
+url = "2.3.1"
 
+agent = { path = "../agent" }
 common = { path = "./common" }
 kata-types = { path = "../../../libs/kata-types" }
 kata-sys-util = { path = "../../../libs/kata-sys-util" }

src/runtime-rs/crates/runtimes/common/src/sandbox.rs

@@ -26,4 +26,6 @@ pub trait Sandbox: Send + Sync {
     // utils
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>>;
     async fn get_iptables(&self, is_ipv6: bool) -> Result<Vec<u8>>;
+    async fn direct_volume_stats(&self, volume_path: &str) -> Result<String>;
+    async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()>;
 }

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -353,9 +353,10 @@ impl RuntimeHandlerManager {
 
 /// Config override ordering(high to low):
 /// 1. podsandbox annotation
-/// 2. shimv2 create task option
-/// TODO: https://github.com/kata-containers/kata-containers/issues/3961
-/// 3. environment
+/// 2. environment variable
+/// 3. shimv2 create task option
+/// 4. If above three are not set, then get default path from DEFAULT_RUNTIME_CONFIGURATIONS
+/// in kata-containers/src/libs/kata-types/src/config/default.rs, in array order.
 fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig> {
     const KATA_CONF_FILE: &str = "KATA_CONF_FILE";
     let annotation = Annotation::new(spec.annotations.clone());

src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs

@@ -7,12 +7,17 @@
 // This defines the handlers corresponding to the url when a request is sent to destined url,
 // the handler function should be invoked, and the corresponding data will be in the response
 
-use anyhow::{anyhow, Result};
+use agent::ResizeVolumeRequest;
+use anyhow::{anyhow, Context, Result};
 use common::Sandbox;
 use hyper::{Body, Method, Request, Response, StatusCode};
 use std::sync::Arc;
+use url::Url;
 
-use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL};
+use shim_interface::shim_mgmt::{
+    AGENT_URL, DIRECT_VOLUME_PATH_KEY, DIRECT_VOLUME_RESIZE_URL, DIRECT_VOLUME_STATS_URL,
+    IP6_TABLE_URL, IP_TABLE_URL,
+};
 
 // main router for response, this works as a multiplexer on
 // http arrival which invokes the corresponding handler function
@@ -34,6 +39,10 @@ pub(crate) async fn handler_mux(
         (&Method::PUT, IP6_TABLE_URL) | (&Method::GET, IP6_TABLE_URL) => {
             ipv6_table_handler(sandbox, req).await
         }
+        (&Method::POST, DIRECT_VOLUME_STATS_URL) => direct_volume_stats_handler(sandbox, req).await,
+        (&Method::POST, DIRECT_VOLUME_RESIZE_URL) => {
+            direct_volume_resize_handler(sandbox, req).await
+        }
         _ => Ok(not_found(req).await),
     }
 }
@@ -101,3 +110,39 @@ async fn generic_ip_table_handler(
         _ => Err(anyhow!("IP Tables only takes PUT and GET")),
     }
 }
+
+async fn direct_volume_stats_handler(
+    sandbox: Arc<dyn Sandbox>,
+    req: Request<Body>,
+) -> Result<Response<Body>> {
+    let params = Url::parse(&req.uri().to_string())
+        .map_err(|e| anyhow!(e))?
+        .query_pairs()
+        .into_owned()
+        .collect::<std::collections::HashMap<String, String>>();
+    let volume_path = params
+        .get(DIRECT_VOLUME_PATH_KEY)
+        .context("shim-mgmt: volume path key not found in request params")?;
+    let result = sandbox.direct_volume_stats(volume_path).await;
+    match result {
+        Ok(stats) => Ok(Response::new(Body::from(stats))),
+        _ => Err(anyhow!("handler: Failed to get volume stats")),
+    }
+}
+
+async fn direct_volume_resize_handler(
+    sandbox: Arc<dyn Sandbox>,
+    req: Request<Body>,
+) -> Result<Response<Body>> {
+    let body = hyper::body::to_bytes(req.into_body()).await?;
+
+    // unserialize json body into resizeRequest struct
+    let resize_req: ResizeVolumeRequest =
+        serde_json::from_slice(&body).context("shim-mgmt: deserialize resizeRequest failed")?;
+    let result = sandbox.direct_volume_resize(resize_req).await;
+
+    match result {
+        Ok(_) => Ok(Response::new(Body::from(""))),
+        _ => Err(anyhow!("handler: Failed to resize volume")),
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs

@@ -344,20 +344,33 @@ impl Container {
         height: u32,
     ) -> Result<()> {
         let logger = logger_with_process(process);
-        let inner = self.inner.read().await;
+        let mut inner = self.inner.write().await;
         if inner.init_process.get_status().await != ProcessStatus::Running {
             warn!(logger, "container is not running");
             return Ok(());
         }
-        self.agent
-            .tty_win_resize(agent::TtyWinResizeRequest {
-                process_id: process.clone().into(),
-                row: height,
-                column: width,
-            })
-            .await
-            .context("resize pty")?;
-        Ok(())
+
+        if process.exec_id.is_empty() {
+            inner.init_process.height = height;
+            inner.init_process.width = width;
+        } else if let Some(exec) = inner.exec_processes.get_mut(&process.exec_id) {
+            exec.process.height = height;
+            exec.process.width = width;
+
+            // for some case, resize_pty request should be handled while the process has not been started in agent
+            // just return here, and truly resize_pty will happen in start_process
+            if exec.process.get_status().await != ProcessStatus::Running {
+                return Ok(());
+            }
+        } else {
+            return Err(anyhow!(
+                "could not find process {} in container {}",
+                process.exec_id(),
+                process.container_id()
+            ));
+        }
+
+        inner.win_resize_process(process, height, width).await
     }
 
     pub async fn stats(&self) -> Result<Option<agent::StatsContainerResponse>> {

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs

@@ -164,9 +164,12 @@ impl ContainerInner {
         let exit_status = self.get_exit_status().await;
         let _locked_exit_status = exit_status.read().await;
         info!(self.logger, "container terminated");
-        let timeout: u32 = 10;
+        let remove_request = agent::RemoveContainerRequest {
+            container_id: cid.to_string(),
+            ..Default::default()
+        };
         self.agent
-            .remove_container(agent::RemoveContainerRequest::new(cid, timeout))
+            .remove_container(remove_request)
             .await
             .or_else(|e| {
                 if force {

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs

@@ -73,7 +73,8 @@ impl ContainerManager for VirtContainerManager {
         // * should be run after the vm is started, before container is created, and after CreateRuntime Hooks
         // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createcontainer-hooks
         let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
-        let vmm_netns_path = format!("/proc/{}/task/{}/ns/{}", self.pid, vmm_master_tid, "net");
+        let vmm_ns_path = self.hypervisor.get_ns_path().await?;
+        let vmm_netns_path = format!("{}/{}", vmm_ns_path, "net");
         let state = oci::State {
             version: spec.version.clone(),
             id: config.container_id.clone(),

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -8,6 +8,7 @@ use std::sync::Arc;
 
 use agent::{
     self, kata::KataAgent, types::KernelModule, Agent, GetIPTablesRequest, SetIPTablesRequest,
+    VolumeStatsRequest,
 };
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
@@ -18,10 +19,7 @@ use common::{
 use containerd_shim_protos::events::task::TaskOOM;
 use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL};
 use kata_sys_util::hooks::HookStates;
-use kata_types::config::{
-    default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT},
-    TomlConfig,
-};
+use kata_types::config::TomlConfig;
 use resource::{
     manager::ManagerArgs,
     network::{NetworkConfig, NetworkWithNetNsConfig},
@@ -218,7 +216,7 @@ impl Sandbox for VirtSandbox {
         let agent_config = self.agent.agent_config().await;
         let kernel_modules = KernelModule::set_kernel_modules(agent_config.kernel_modules)?;
         let req = agent::CreateSandboxRequest {
-            hostname: "".to_string(),
+            hostname: spec.hostname.clone(),
             dns,
             storages: self
                 .resource_manager
@@ -329,6 +327,26 @@ impl Sandbox for VirtSandbox {
         self.agent.agent_sock().await
     }
 
+    async fn direct_volume_stats(&self, volume_guest_path: &str) -> Result<String> {
+        let req: agent::VolumeStatsRequest = VolumeStatsRequest {
+            volume_guest_path: volume_guest_path.to_string(),
+        };
+        let result = self
+            .agent
+            .get_volume_stats(req)
+            .await
+            .context("sandbox: failed to process direct volume stats query")?;
+        Ok(result.data)
+    }
+
+    async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()> {
+        self.agent
+            .resize_volume(resize_req)
+            .await
+            .context("sandbox: failed to resize direct-volume")?;
+        Ok(())
+    }
+
     async fn set_iptables(&self, is_ipv6: bool, data: Vec<u8>) -> Result<Vec<u8>> {
         info!(sl!(), "sb: set_iptables invoked");
         let req = SetIPTablesRequest { is_ipv6, data };
@@ -380,19 +398,7 @@ impl Persist for VirtSandbox {
             HYPERVISOR_DRAGONBALL => Ok(Arc::new(Dragonball::restore((), h).await?)),
             _ => Err(anyhow!("Unsupported hypervisor {}", &h.hypervisor_type)),
         }?;
-        let agent = Arc::new(KataAgent::new(kata_types::config::Agent {
-            debug: true,
-            enable_tracing: false,
-            server_port: DEFAULT_AGENT_VSOCK_PORT,
-            log_port: DEFAULT_AGENT_LOG_PORT,
-            dial_timeout_ms: 10,
-            reconnect_timeout_ms: 3_000,
-            request_timeout_ms: 30_000,
-            health_check_request_timeout_ms: 90_000,
-            kernel_modules: Default::default(),
-            container_pipe_size: 0,
-            debug_console_enabled: false,
-        }));
+        let agent = Arc::new(KataAgent::new(kata_types::config::Agent::default()));
         let sid = sandbox_args.sid;
         let args = ManagerArgs {
             sid: sid.clone(),

src/runtime-rs/crates/shim/src/args.rs

@@ -117,12 +117,11 @@ mod tests {
             result: Result<()>,
         }
 
-        let default_id = "1dfc0567".to_string();
-        let default_namespace = "ns1".to_string();
+        let default_id = "default_id".to_string();
+        let default_namespace = "default_namespace".to_string();
         let default_address = bind_address.to_string();
         let default_publish_binary = "containerd".to_string();
         let default_bundle = path.to_string();
-        let default_debug = false;
 
         let mut arg = Args {
             id: default_id.clone(),
@@ -130,7 +129,7 @@ mod tests {
             address: default_address.clone(),
             publish_binary: default_publish_binary.clone(),
             bundle: default_bundle.clone(),
-            debug: default_debug,
+            ..Default::default()
         };
 
         let tests = &[

src/runtime-rs/crates/shim/src/shim.rs

@@ -93,12 +93,12 @@ mod tests {
         std::env::set_current_dir(bundle_path).unwrap();
 
         let args = Args {
-            id: "1dfc0567".to_string(),
-            namespace: "test_namespace".into(),
-            address: "containerd_socket".into(),
+            id: "default_id".into(),
+            namespace: "default_namespace".into(),
+            address: "default_address".into(),
             publish_binary: "containerd".into(),
             bundle: bundle_path.to_str().unwrap().into(),
-            debug: false,
+            ..Default::default()
         };
 
         let executor = ShimExecutor::new(args);

src/runtime-rs/crates/shim/src/shim_start.rs

@@ -153,12 +153,12 @@ mod tests {
         std::env::set_current_dir(bundle_path).unwrap();
 
         let args = Args {
-            id: "sandbox1".into(),
-            namespace: "ns".into(),
-            address: "address".into(),
+            id: "default_id".into(),
+            namespace: "default_namespace".into(),
+            address: "default_address".into(),
             publish_binary: "containerd".into(),
             bundle: bundle_path.to_str().unwrap().into(),
-            debug: false,
+            ..Default::default()
         };
         let mut executor = ShimExecutor::new(args);
 
@@ -185,11 +185,11 @@ mod tests {
 
         let args = Args {
             id: sandbox_id.to_owned(),
-            namespace: "ns1".into(),
-            address: "containerd_socket".into(),
+            namespace: "default_namespace".into(),
+            address: "default_address".into(),
             publish_binary: "containerd".into(),
             bundle: bundle_path.to_str().unwrap().into(),
-            debug: false,
+            ..Default::default()
         };
         let executor = ShimExecutor::new(args);
 
@@ -204,11 +204,11 @@ mod tests {
 
         let args = Args {
             id: container_id,
-            namespace: "ns1".into(),
-            address: "containerd_socket".into(),
+            namespace: "default_namespace".into(),
+            address: "default_address".into(),
             publish_binary: "containerd".into(),
             bundle: bundle_path2.to_str().unwrap().into(),
-            debug: false,
+            ..Default::default()
         };
         let executor2 = ShimExecutor::new(args);
 

src/runtime/pkg/device/config/config.go

@@ -258,15 +258,24 @@ const (
 	// VFIODeviceErrorType is the error type of VFIO device
 	VFIODeviceErrorType VFIODeviceType = iota
 
-	// VFIODeviceNormalType is a normal VFIO device type
-	VFIODeviceNormalType
+	// VFIOPCIDeviceNormalType is a normal VFIO PCI device type
+	VFIOPCIDeviceNormalType
 
-	// VFIODeviceMediatedType is a VFIO mediated device type
-	VFIODeviceMediatedType
+	// VFIOPCIDeviceMediatedType is a VFIO PCI mediated device type
+	VFIOPCIDeviceMediatedType
+
+	// VFIOAPDeviceMediatedType is a VFIO AP mediated device type
+	VFIOAPDeviceMediatedType
 )
 
-// VFIODev represents a VFIO drive used for hotplugging
-type VFIODev struct {
+type VFIODev interface {
+	GetID() *string
+	GetType() VFIODeviceType
+	GetSysfsDev() *string
+}
+
+// VFIOPCIDev represents a VFIO PCI device used for hotplugging
+type VFIOPCIDev struct {
 	// ID is used to identify this drive in the hypervisor options.
 	ID string
 
@@ -298,6 +307,44 @@ type VFIODev struct {
 	IsPCIe bool
 }
 
+func (d VFIOPCIDev) GetID() *string {
+	return &d.ID
+}
+
+func (d VFIOPCIDev) GetType() VFIODeviceType {
+	return d.Type
+}
+
+func (d VFIOPCIDev) GetSysfsDev() *string {
+	return &d.SysfsDev
+}
+
+type VFIOAPDev struct {
+	// ID is used to identify this drive in the hypervisor options.
+	ID string
+
+	// sysfsdev of VFIO mediated device
+	SysfsDev string
+
+	// APDevices are the Adjunct Processor devices assigned to the mdev
+	APDevices []string
+
+	// Type of VFIO device
+	Type VFIODeviceType
+}
+
+func (d VFIOAPDev) GetID() *string {
+	return &d.ID
+}
+
+func (d VFIOAPDev) GetType() VFIODeviceType {
+	return d.Type
+}
+
+func (d VFIOAPDev) GetSysfsDev() *string {
+	return &d.SysfsDev
+}
+
 // RNGDev represents a random number generator device
 type RNGDev struct {
 	// ID is used to identify the device in the hypervisor options.

src/runtime/pkg/device/drivers/utils.go

@@ -89,18 +89,47 @@ func readPCIProperty(propertyPath string) (string, error) {
 	return strings.Split(string(buf), "\n")[0], nil
 }
 
-func GetVFIODeviceType(deviceFileName string) config.VFIODeviceType {
+func GetVFIODeviceType(deviceFilePath string) (config.VFIODeviceType, error) {
+	deviceFileName := filepath.Base(deviceFilePath)
+
 	//For example, 0000:04:00.0
 	tokens := strings.Split(deviceFileName, ":")
-	vfioDeviceType := config.VFIODeviceErrorType
 	if len(tokens) == 3 {
-		vfioDeviceType = config.VFIODeviceNormalType
-	} else {
-		//For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
-		tokens = strings.Split(deviceFileName, "-")
-		if len(tokens) == 5 {
-			vfioDeviceType = config.VFIODeviceMediatedType
-		}
+		return config.VFIOPCIDeviceNormalType, nil
 	}
-	return vfioDeviceType
+
+	//For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
+	tokens = strings.Split(deviceFileName, "-")
+	if len(tokens) != 5 {
+		return config.VFIODeviceErrorType, fmt.Errorf("Incorrect tokens found while parsing VFIO details: %s", deviceFileName)
+	}
+
+	deviceSysfsDev, err := GetSysfsDev(deviceFilePath)
+	if err != nil {
+		return config.VFIODeviceErrorType, err
+	}
+
+	if strings.HasPrefix(deviceSysfsDev, vfioAPSysfsDir) {
+		return config.VFIOAPDeviceMediatedType, nil
+	}
+
+	return config.VFIOPCIDeviceMediatedType, nil
+}
+
+// GetSysfsDev returns the sysfsdev of mediated device
+// Expected input string format is absolute path to the sysfs dev node
+// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4
+func GetSysfsDev(sysfsDevStr string) (string, error) {
+	return filepath.EvalSymlinks(sysfsDevStr)
+}
+
+// GetAPVFIODevices retrieves all APQNs associated with a mediated VFIO-AP
+// device
+func GetAPVFIODevices(sysfsdev string) ([]string, error) {
+	data, err := os.ReadFile(filepath.Join(sysfsdev, "matrix"))
+	if err != nil {
+		return []string{}, err
+	}
+	// Split by newlines, omitting final newline
+	return strings.Split(string(data[:len(data)-1]), "\n"), nil
 }

src/runtime/pkg/device/drivers/vfio.go

@@ -30,6 +30,7 @@ const (
 	iommuGroupPath      = "/sys/bus/pci/devices/%s/iommu_group"
 	vfioDevPath         = "/dev/vfio/%s"
 	pcieRootPortPrefix  = "rp"
+	vfioAPSysfsDir      = "/sys/devices/vfio_ap"
 )
 
 var (
@@ -85,19 +86,42 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		if err != nil {
 			return err
 		}
-		vfio := &config.VFIODev{
-			ID:       utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize),
-			Type:     vfioDeviceType,
-			BDF:      deviceBDF,
-			SysfsDev: deviceSysfsDev,
-			IsPCIe:   isPCIeDevice(deviceBDF),
-			Class:    getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass),
-		}
-		device.VfioDevs = append(device.VfioDevs, vfio)
-		if vfio.IsPCIe {
-			vfio.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
-			AllPCIeDevs[vfio.BDF] = true
+		id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize)
+
+		var vfio config.VFIODev
+
+		switch vfioDeviceType {
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			isPCIe := isPCIeDevice(deviceBDF)
+			// Do not directly assign to `vfio` -- need to access field still
+			vfioPCI := config.VFIOPCIDev{
+				ID:       id,
+				Type:     vfioDeviceType,
+				BDF:      deviceBDF,
+				SysfsDev: deviceSysfsDev,
+				IsPCIe:   isPCIe,
+				Class:    getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass),
+			}
+			if isPCIe {
+				vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
+				AllPCIeDevs[deviceBDF] = true
+			}
+			vfio = vfioPCI
+		case config.VFIOAPDeviceMediatedType:
+			devices, err := GetAPVFIODevices(deviceSysfsDev)
+			if err != nil {
+				return err
+			}
+			vfio = config.VFIOAPDev{
+				ID:        id,
+				SysfsDev:  deviceSysfsDev,
+				Type:      config.VFIOAPDeviceMediatedType,
+				APDevices: devices,
+			}
+		default:
+			return fmt.Errorf("Failed to append device: VFIO device type unrecognized")
 		}
+		device.VfioDevs = append(device.VfioDevs, &vfio)
 	}
 
 	coldPlug := device.DeviceInfo.ColdPlug
@@ -192,31 +216,60 @@ func (device *VFIODevice) Load(ds config.DeviceState) {
 	device.GenericDevice.Load(ds)
 
 	for _, dev := range ds.VFIODevs {
-		device.VfioDevs = append(device.VfioDevs, &config.VFIODev{
-			ID:       dev.ID,
-			Type:     config.VFIODeviceType(dev.Type),
-			BDF:      dev.BDF,
-			SysfsDev: dev.SysfsDev,
-		})
+		var vfio config.VFIODev
+
+		vfioDeviceType := (*device.VfioDevs[0]).GetType()
+		switch vfioDeviceType {
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			bdf := ""
+			if pciDev, ok := (*dev).(config.VFIOPCIDev); ok {
+				bdf = pciDev.BDF
+			}
+			vfio = config.VFIOPCIDev{
+				ID:       *(*dev).GetID(),
+				Type:     config.VFIODeviceType((*dev).GetType()),
+				BDF:      bdf,
+				SysfsDev: *(*dev).GetSysfsDev(),
+			}
+		case config.VFIOAPDeviceMediatedType:
+			vfio = config.VFIOAPDev{
+				ID:       *(*dev).GetID(),
+				SysfsDev: *(*dev).GetSysfsDev(),
+			}
+		default:
+			deviceLogger().WithError(
+				fmt.Errorf("VFIO device type unrecognized"),
+			).Error("Failed to append device")
+			return
+		}
+
+		device.VfioDevs = append(device.VfioDevs, &vfio)
 	}
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) {
-	vfioDeviceType = GetVFIODeviceType(deviceFileName)
+	sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
+	vfioDeviceType, err = GetVFIODeviceType(sysfsDevStr)
+	if err != nil {
+		return deviceBDF, deviceSysfsDev, vfioDeviceType, err
+	}
 
 	switch vfioDeviceType {
-	case config.VFIODeviceNormalType:
+	case config.VFIOPCIDeviceNormalType:
 		// Get bdf of device eg. 0000:00:1c.0
 		deviceBDF = getBDF(deviceFileName)
 		// Get sysfs path used by cloud-hypervisor
 		deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName)
-	case config.VFIODeviceMediatedType:
+	case config.VFIOPCIDeviceMediatedType:
 		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
 		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
-		deviceSysfsDev, err = getSysfsDev(sysfsDevStr)
+		deviceSysfsDev, err = GetSysfsDev(sysfsDevStr)
 		deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev))
+	case config.VFIOAPDeviceMediatedType:
+		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
+		deviceSysfsDev, err = GetSysfsDev(sysfsDevStr)
 	default:
 		err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName)
 	}
@@ -244,13 +297,6 @@ func getBDF(deviceSysStr string) string {
 	return tokens[1]
 }
 
-// getSysfsDev returns the sysfsdev of mediated device
-// Expected input string format is absolute path to the sysfs dev node
-// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4
-func getSysfsDev(sysfsDevStr string) (string, error) {
-	return filepath.EvalSymlinks(sysfsDevStr)
-}
-
 // BindDevicetoVFIO binds the device to vfio driver after unbinding from host.
 // Will be called by a network interface or a generic pcie device.
 func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) {

src/runtime/pkg/device/drivers/vfio_test.go

@@ -32,9 +32,9 @@ func TestGetVFIODetails(t *testing.T) {
 		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "")
 
 		switch vfioDeviceType {
-		case config.VFIODeviceNormalType:
+		case config.VFIOPCIDeviceNormalType:
 			assert.Equal(t, d.expectedStr, deviceBDF)
-		case config.VFIODeviceMediatedType:
+		case config.VFIOPCIDeviceMediatedType, config.VFIOAPDeviceMediatedType:
 			assert.Equal(t, d.expectedStr, deviceSysfsDev)
 		default:
 			assert.NotNil(t, err)

src/runtime/pkg/resourcecontrol/cgroups.go

@@ -67,7 +67,8 @@ func sandboxDevices() []specs.LinuxDeviceCgroup {
 	// In order to run Virtual Machines and create virtqueues, hypervisors
 	// need access to certain character devices in the host, like kvm and vhost-net.
 	hypervisorDevices := []string{
-		"/dev/kvm",         // To run virtual machines
+		"/dev/kvm",         // To run virtual machines with KVM
+		"/dev/mshv",        // To run virtual machines with Hyper-V
 		"/dev/vhost-net",   // To create virtqueues
 		"/dev/vfio/vfio",   // To access VFIO devices
 		"/dev/vhost-vsock", // To interact with vsock if

src/runtime/virtcontainers/agent.go

@@ -154,6 +154,9 @@ type agent interface {
 	// listInterfaces will tell the agent to list interfaces of an existed Sandbox
 	listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error)
 
+	// updateEphemeralMounts will tell the agent to update tmpfs mounts in the Sandbox.
+	updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error
+
 	// updateRoutes will tell the agent to update route table for an existed Sandbox.
 	updateRoutes(ctx context.Context, routes []*pbTypes.Route) ([]*pbTypes.Route, error)
 

src/runtime/virtcontainers/clh.go

@@ -857,12 +857,12 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error {
 	defer cancel()
 
 	// Create the clh device config via the constructor to ensure default values are properly assigned
-	clhDevice := *chclient.NewDeviceConfig(device.SysfsDev)
+	clhDevice := *chclient.NewDeviceConfig(*(*device).GetSysfsDev())
 	pciInfo, _, err := cl.VmAddDevicePut(ctx, clhDevice)
 	if err != nil {
 		return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err))
 	}
-	clh.devicesIds[device.ID] = pciInfo.GetId()
+	clh.devicesIds[*(*device).GetID()] = pciInfo.GetId()
 
 	// clh doesn't use bridges, so the PCI path is simply the slot
 	// number of the device.  This will break if clh starts using
@@ -879,7 +879,14 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error {
 		return fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf)
 	}
 
-	device.GuestPciPath, err = types.PciPathFromString(tokens[0])
+	guestPciPath, err := types.PciPathFromString(tokens[0])
+
+	pciDevice, ok := (*device).(config.VFIOPCIDev)
+	if !ok {
+		return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device)
+	}
+	pciDevice.GuestPciPath = guestPciPath
+	*device = pciDevice
 
 	return err
 }
@@ -923,7 +930,7 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int
 	case BlockDev:
 		deviceID = clhDriveIndexToID(devInfo.(*config.BlockDrive).Index)
 	case VfioDev:
-		deviceID = devInfo.(*config.VFIODev).ID
+		deviceID = *devInfo.(config.VFIODev).GetID()
 	default:
 		clh.Logger().WithFields(log.Fields{"devInfo": devInfo,
 			"deviceType": devType}).Error("HotplugRemoveDevice: unsupported device")

src/runtime/virtcontainers/clh_test.go

@@ -624,7 +624,7 @@ func TestCloudHypervisorHotplugRemoveDevice(t *testing.T) {
 	_, err = clh.HotplugRemoveDevice(context.Background(), &config.BlockDrive{}, BlockDev)
 	assert.NoError(err, "Hotplug remove block device expected no error")
 
-	_, err = clh.HotplugRemoveDevice(context.Background(), &config.VFIODev{}, VfioDev)
+	_, err = clh.HotplugRemoveDevice(context.Background(), &config.VFIOPCIDev{}, VfioDev)
 	assert.NoError(err, "Hotplug remove vfio block device expected no error")
 
 	_, err = clh.HotplugRemoveDevice(context.Background(), nil, NetDev)

src/runtime/virtcontainers/kata_agent.go

@@ -117,41 +117,42 @@ var (
 )
 
 const (
-	grpcCheckRequest             = "grpc.CheckRequest"
-	grpcExecProcessRequest       = "grpc.ExecProcessRequest"
-	grpcCreateSandboxRequest     = "grpc.CreateSandboxRequest"
-	grpcDestroySandboxRequest    = "grpc.DestroySandboxRequest"
-	grpcCreateContainerRequest   = "grpc.CreateContainerRequest"
-	grpcStartContainerRequest    = "grpc.StartContainerRequest"
-	grpcRemoveContainerRequest   = "grpc.RemoveContainerRequest"
-	grpcSignalProcessRequest     = "grpc.SignalProcessRequest"
-	grpcUpdateRoutesRequest      = "grpc.UpdateRoutesRequest"
-	grpcUpdateInterfaceRequest   = "grpc.UpdateInterfaceRequest"
-	grpcListInterfacesRequest    = "grpc.ListInterfacesRequest"
-	grpcListRoutesRequest        = "grpc.ListRoutesRequest"
-	grpcAddARPNeighborsRequest   = "grpc.AddARPNeighborsRequest"
-	grpcOnlineCPUMemRequest      = "grpc.OnlineCPUMemRequest"
-	grpcUpdateContainerRequest   = "grpc.UpdateContainerRequest"
-	grpcWaitProcessRequest       = "grpc.WaitProcessRequest"
-	grpcTtyWinResizeRequest      = "grpc.TtyWinResizeRequest"
-	grpcWriteStreamRequest       = "grpc.WriteStreamRequest"
-	grpcCloseStdinRequest        = "grpc.CloseStdinRequest"
-	grpcStatsContainerRequest    = "grpc.StatsContainerRequest"
-	grpcPauseContainerRequest    = "grpc.PauseContainerRequest"
-	grpcResumeContainerRequest   = "grpc.ResumeContainerRequest"
-	grpcPullImageRequest         = "grpc.PullImageRequest"
-	grpcReseedRandomDevRequest   = "grpc.ReseedRandomDevRequest"
-	grpcGuestDetailsRequest      = "grpc.GuestDetailsRequest"
-	grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest"
-	grpcCopyFileRequest          = "grpc.CopyFileRequest"
-	grpcSetGuestDateTimeRequest  = "grpc.SetGuestDateTimeRequest"
-	grpcGetOOMEventRequest       = "grpc.GetOOMEventRequest"
-	grpcGetMetricsRequest        = "grpc.GetMetricsRequest"
-	grpcAddSwapRequest           = "grpc.AddSwapRequest"
-	grpcVolumeStatsRequest       = "grpc.VolumeStatsRequest"
-	grpcResizeVolumeRequest      = "grpc.ResizeVolumeRequest"
-	grpcGetIPTablesRequest       = "grpc.GetIPTablesRequest"
-	grpcSetIPTablesRequest       = "grpc.SetIPTablesRequest"
+	grpcCheckRequest                 = "grpc.CheckRequest"
+	grpcExecProcessRequest           = "grpc.ExecProcessRequest"
+	grpcCreateSandboxRequest         = "grpc.CreateSandboxRequest"
+	grpcDestroySandboxRequest        = "grpc.DestroySandboxRequest"
+	grpcCreateContainerRequest       = "grpc.CreateContainerRequest"
+	grpcStartContainerRequest        = "grpc.StartContainerRequest"
+	grpcRemoveContainerRequest       = "grpc.RemoveContainerRequest"
+	grpcSignalProcessRequest         = "grpc.SignalProcessRequest"
+	grpcUpdateRoutesRequest          = "grpc.UpdateRoutesRequest"
+	grpcUpdateInterfaceRequest       = "grpc.UpdateInterfaceRequest"
+	grpcListInterfacesRequest        = "grpc.ListInterfacesRequest"
+	grpcListRoutesRequest            = "grpc.ListRoutesRequest"
+	grpcAddARPNeighborsRequest       = "grpc.AddARPNeighborsRequest"
+	grpcOnlineCPUMemRequest          = "grpc.OnlineCPUMemRequest"
+	grpcUpdateContainerRequest       = "grpc.UpdateContainerRequest"
+	grpcUpdateEphemeralMountsRequest = "grpc.UpdateEphemeralMountsRequest"
+	grpcWaitProcessRequest           = "grpc.WaitProcessRequest"
+	grpcTtyWinResizeRequest          = "grpc.TtyWinResizeRequest"
+	grpcWriteStreamRequest           = "grpc.WriteStreamRequest"
+	grpcCloseStdinRequest            = "grpc.CloseStdinRequest"
+	grpcStatsContainerRequest        = "grpc.StatsContainerRequest"
+	grpcPauseContainerRequest        = "grpc.PauseContainerRequest"
+	grpcResumeContainerRequest       = "grpc.ResumeContainerRequest"
+	grpcPullImageRequest             = "grpc.PullImageRequest"
+	grpcReseedRandomDevRequest       = "grpc.ReseedRandomDevRequest"
+	grpcGuestDetailsRequest          = "grpc.GuestDetailsRequest"
+	grpcMemHotplugByProbeRequest     = "grpc.MemHotplugByProbeRequest"
+	grpcCopyFileRequest              = "grpc.CopyFileRequest"
+	grpcSetGuestDateTimeRequest      = "grpc.SetGuestDateTimeRequest"
+	grpcGetOOMEventRequest           = "grpc.GetOOMEventRequest"
+	grpcGetMetricsRequest            = "grpc.GetMetricsRequest"
+	grpcAddSwapRequest               = "grpc.AddSwapRequest"
+	grpcVolumeStatsRequest           = "grpc.VolumeStatsRequest"
+	grpcResizeVolumeRequest          = "grpc.ResizeVolumeRequest"
+	grpcGetIPTablesRequest           = "grpc.GetIPTablesRequest"
+	grpcSetIPTablesRequest           = "grpc.SetIPTablesRequest"
 )
 
 // newKataAgent returns an agent from an agent type.
@@ -610,6 +611,21 @@ func (k *kataAgent) updateRoutes(ctx context.Context, routes []*pbTypes.Route) (
 	return nil, nil
 }
 
+func (k *kataAgent) updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error {
+	if storages != nil {
+		storagesReq := &grpc.UpdateEphemeralMountsRequest{
+			Storages: storages,
+		}
+
+		if _, err := k.sendReq(ctx, storagesReq); err != nil {
+			k.Logger().WithError(err).Error("update mounts request failed")
+			return err
+		}
+		return nil
+	}
+	return nil
+}
+
 func (k *kataAgent) addARPNeighbors(ctx context.Context, neighs []*pbTypes.ARPNeighbor) error {
 	if neighs != nil {
 		neighsReq := &grpc.AddARPNeighborsRequest{
@@ -1112,20 +1128,25 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c *
 
 	groupNum := filepath.Base(dev.ContainerPath)
 
-	// Each /dev/vfio/NN device represents a VFIO group, which
-	// could include several PCI devices.  So we give group
-	// information in the main structure, then list each
-	// individual PCI device in the Options array.
+	// For VFIO-PCI, each /dev/vfio/NN device represents a VFIO group,
+	// which could include several PCI devices. So we give group
+	// information in the main structure, then list each individual PCI
+	// device in the Options array.
 	//
 	// Each option is formatted as "DDDD:BB:DD.F=<pcipath>"
 	// DDDD:BB:DD.F is the device's PCI address on the
 	// *host*. <pcipath> is the device's PCI path in the guest
 	// (see qomGetPciPath() for details).
+	//
+	// For VFIO-AP, one VFIO group could include several queue devices. They are
+	// identified by APQNs (Adjunct Processor Queue Numbers), which do not differ
+	// between host and guest. They are passed as options so they can be awaited
+	// by the agent.
 	kataDevice := &grpc.Device{
 		ContainerPath: dev.ContainerPath,
-		Type:          kataVfioDevType,
+		Type:          kataVfioPciDevType,
 		Id:            groupNum,
-		Options:       make([]string, len(devList)),
+		Options:       nil,
 	}
 
 	// We always pass the device information to the agent, since
@@ -1133,11 +1154,18 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c *
 	// on the vfio_mode, we need to use a different device type so
 	// the agent can handle it properly
 	if c.sandbox.config.VfioMode == config.VFIOModeGuestKernel {
-		kataDevice.Type = kataVfioGuestKernelDevType
+		kataDevice.Type = kataVfioPciGuestKernelDevType
 	}
 
-	for i, pciDev := range devList {
-		kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDev.BDF, pciDev.GuestPciPath)
+	if (*devList[0]).GetType() == config.VFIOAPDeviceMediatedType {
+		kataDevice.Type = kataVfioApDevType
+		kataDevice.Options = (*devList[0]).(config.VFIOAPDev).APDevices
+	} else {
+		kataDevice.Options = make([]string, len(devList))
+		for i, device := range devList {
+			pciDevice := (*device).(config.VFIOPCIDev)
+			kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath)
+		}
 	}
 
 	return kataDevice
@@ -1963,6 +1991,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest))
 	}
+	k.reqHandlers[grpcUpdateEphemeralMountsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.UpdateEphemeralMounts(ctx, req.(*grpc.UpdateEphemeralMountsRequest))
+	}
 	k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest))
 	}

src/runtime/virtcontainers/mock_agent.go

@@ -122,6 +122,11 @@ func (n *mockAgent) listRoutes(ctx context.Context) ([]*pbTypes.Route, error) {
 	return nil, nil
 }
 
+// updateEphemeralMounts is the Noop agent updateEphemeralMounts implementation. It does nothing.
+func (n *mockAgent) updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error {
+	return nil
+}
+
 // check is the Noop agent health checker. It does nothing.
 func (n *mockAgent) check(ctx context.Context) error {
 	return nil

src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go

@@ -1527,6 +1527,45 @@ func (m *UpdateRoutesRequest) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_UpdateRoutesRequest proto.InternalMessageInfo
 
+type UpdateEphemeralMountsRequest struct {
+	Storages             []*Storage `protobuf:"bytes,1,rep,name=storages,proto3" json:"storages,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
+	XXX_unrecognized     []byte     `json:"-"`
+	XXX_sizecache        int32      `json:"-"`
+}
+
+func (m *UpdateEphemeralMountsRequest) Reset()      { *m = UpdateEphemeralMountsRequest{} }
+func (*UpdateEphemeralMountsRequest) ProtoMessage() {}
+func (*UpdateEphemeralMountsRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_712ce9a559fda969, []int{36}
+}
+func (m *UpdateEphemeralMountsRequest) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *UpdateEphemeralMountsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_UpdateEphemeralMountsRequest.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *UpdateEphemeralMountsRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_UpdateEphemeralMountsRequest.Merge(m, src)
+}
+func (m *UpdateEphemeralMountsRequest) XXX_Size() int {
+	return m.Size()
+}
+func (m *UpdateEphemeralMountsRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_UpdateEphemeralMountsRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UpdateEphemeralMountsRequest proto.InternalMessageInfo
+
 type ListInterfacesRequest struct {
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
@@ -1536,7 +1575,7 @@ type ListInterfacesRequest struct {
 func (m *ListInterfacesRequest) Reset()      { *m = ListInterfacesRequest{} }
 func (*ListInterfacesRequest) ProtoMessage() {}
 func (*ListInterfacesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{36}
+	return fileDescriptor_712ce9a559fda969, []int{37}
 }
 func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1574,7 +1613,7 @@ type ListRoutesRequest struct {
 func (m *ListRoutesRequest) Reset()      { *m = ListRoutesRequest{} }
 func (*ListRoutesRequest) ProtoMessage() {}
 func (*ListRoutesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{37}
+	return fileDescriptor_712ce9a559fda969, []int{38}
 }
 func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1613,7 +1652,7 @@ type ARPNeighbors struct {
 func (m *ARPNeighbors) Reset()      { *m = ARPNeighbors{} }
 func (*ARPNeighbors) ProtoMessage() {}
 func (*ARPNeighbors) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{38}
+	return fileDescriptor_712ce9a559fda969, []int{39}
 }
 func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1652,7 +1691,7 @@ type AddARPNeighborsRequest struct {
 func (m *AddARPNeighborsRequest) Reset()      { *m = AddARPNeighborsRequest{} }
 func (*AddARPNeighborsRequest) ProtoMessage() {}
 func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{39}
+	return fileDescriptor_712ce9a559fda969, []int{40}
 }
 func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1691,7 +1730,7 @@ type GetIPTablesRequest struct {
 func (m *GetIPTablesRequest) Reset()      { *m = GetIPTablesRequest{} }
 func (*GetIPTablesRequest) ProtoMessage() {}
 func (*GetIPTablesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{40}
+	return fileDescriptor_712ce9a559fda969, []int{41}
 }
 func (m *GetIPTablesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1731,7 +1770,7 @@ type GetIPTablesResponse struct {
 func (m *GetIPTablesResponse) Reset()      { *m = GetIPTablesResponse{} }
 func (*GetIPTablesResponse) ProtoMessage() {}
 func (*GetIPTablesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{41}
+	return fileDescriptor_712ce9a559fda969, []int{42}
 }
 func (m *GetIPTablesResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1773,7 +1812,7 @@ type SetIPTablesRequest struct {
 func (m *SetIPTablesRequest) Reset()      { *m = SetIPTablesRequest{} }
 func (*SetIPTablesRequest) ProtoMessage() {}
 func (*SetIPTablesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{42}
+	return fileDescriptor_712ce9a559fda969, []int{43}
 }
 func (m *SetIPTablesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1803,7 +1842,7 @@ func (m *SetIPTablesRequest) XXX_DiscardUnknown() {
 var xxx_messageInfo_SetIPTablesRequest proto.InternalMessageInfo
 
 type SetIPTablesResponse struct {
-	// raw stdout from iptables-restore or ip6tables-restore ?
+	// raw stdout from iptables-restore or ip6tables-restore
 	Data                 []byte   `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
@@ -1813,7 +1852,7 @@ type SetIPTablesResponse struct {
 func (m *SetIPTablesResponse) Reset()      { *m = SetIPTablesResponse{} }
 func (*SetIPTablesResponse) ProtoMessage() {}
 func (*SetIPTablesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{43}
+	return fileDescriptor_712ce9a559fda969, []int{44}
 }
 func (m *SetIPTablesResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1859,7 +1898,7 @@ type OnlineCPUMemRequest struct {
 func (m *OnlineCPUMemRequest) Reset()      { *m = OnlineCPUMemRequest{} }
 func (*OnlineCPUMemRequest) ProtoMessage() {}
 func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{44}
+	return fileDescriptor_712ce9a559fda969, []int{45}
 }
 func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1899,7 +1938,7 @@ type ReseedRandomDevRequest struct {
 func (m *ReseedRandomDevRequest) Reset()      { *m = ReseedRandomDevRequest{} }
 func (*ReseedRandomDevRequest) ProtoMessage() {}
 func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{45}
+	return fileDescriptor_712ce9a559fda969, []int{46}
 }
 func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1949,7 +1988,7 @@ type AgentDetails struct {
 func (m *AgentDetails) Reset()      { *m = AgentDetails{} }
 func (*AgentDetails) ProtoMessage() {}
 func (*AgentDetails) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{46}
+	return fileDescriptor_712ce9a559fda969, []int{47}
 }
 func (m *AgentDetails) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1995,7 +2034,7 @@ type GuestDetailsRequest struct {
 func (m *GuestDetailsRequest) Reset()      { *m = GuestDetailsRequest{} }
 func (*GuestDetailsRequest) ProtoMessage() {}
 func (*GuestDetailsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{47}
+	return fileDescriptor_712ce9a559fda969, []int{48}
 }
 func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2037,7 +2076,7 @@ type GuestDetailsResponse struct {
 func (m *GuestDetailsResponse) Reset()      { *m = GuestDetailsResponse{} }
 func (*GuestDetailsResponse) ProtoMessage() {}
 func (*GuestDetailsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{48}
+	return fileDescriptor_712ce9a559fda969, []int{49}
 }
 func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2078,7 +2117,7 @@ type MemHotplugByProbeRequest struct {
 func (m *MemHotplugByProbeRequest) Reset()      { *m = MemHotplugByProbeRequest{} }
 func (*MemHotplugByProbeRequest) ProtoMessage() {}
 func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{49}
+	return fileDescriptor_712ce9a559fda969, []int{50}
 }
 func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2120,7 +2159,7 @@ type SetGuestDateTimeRequest struct {
 func (m *SetGuestDateTimeRequest) Reset()      { *m = SetGuestDateTimeRequest{} }
 func (*SetGuestDateTimeRequest) ProtoMessage() {}
 func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{50}
+	return fileDescriptor_712ce9a559fda969, []int{51}
 }
 func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2166,7 +2205,7 @@ type FSGroup struct {
 func (m *FSGroup) Reset()      { *m = FSGroup{} }
 func (*FSGroup) ProtoMessage() {}
 func (*FSGroup) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{51}
+	return fileDescriptor_712ce9a559fda969, []int{52}
 }
 func (m *FSGroup) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2234,7 +2273,7 @@ type Storage struct {
 func (m *Storage) Reset()      { *m = Storage{} }
 func (*Storage) ProtoMessage() {}
 func (*Storage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{52}
+	return fileDescriptor_712ce9a559fda969, []int{53}
 }
 func (m *Storage) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2305,7 +2344,7 @@ type Device struct {
 func (m *Device) Reset()      { *m = Device{} }
 func (*Device) ProtoMessage() {}
 func (*Device) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{53}
+	return fileDescriptor_712ce9a559fda969, []int{54}
 }
 func (m *Device) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2346,7 +2385,7 @@ type StringUser struct {
 func (m *StringUser) Reset()      { *m = StringUser{} }
 func (*StringUser) ProtoMessage() {}
 func (*StringUser) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{54}
+	return fileDescriptor_712ce9a559fda969, []int{55}
 }
 func (m *StringUser) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2403,7 +2442,7 @@ type CopyFileRequest struct {
 func (m *CopyFileRequest) Reset()      { *m = CopyFileRequest{} }
 func (*CopyFileRequest) ProtoMessage() {}
 func (*CopyFileRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{55}
+	return fileDescriptor_712ce9a559fda969, []int{56}
 }
 func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2441,7 +2480,7 @@ type GetOOMEventRequest struct {
 func (m *GetOOMEventRequest) Reset()      { *m = GetOOMEventRequest{} }
 func (*GetOOMEventRequest) ProtoMessage() {}
 func (*GetOOMEventRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{56}
+	return fileDescriptor_712ce9a559fda969, []int{57}
 }
 func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2480,7 +2519,7 @@ type OOMEvent struct {
 func (m *OOMEvent) Reset()      { *m = OOMEvent{} }
 func (*OOMEvent) ProtoMessage() {}
 func (*OOMEvent) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{57}
+	return fileDescriptor_712ce9a559fda969, []int{58}
 }
 func (m *OOMEvent) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2519,7 +2558,7 @@ type AddSwapRequest struct {
 func (m *AddSwapRequest) Reset()      { *m = AddSwapRequest{} }
 func (*AddSwapRequest) ProtoMessage() {}
 func (*AddSwapRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{58}
+	return fileDescriptor_712ce9a559fda969, []int{59}
 }
 func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2557,7 +2596,7 @@ type GetMetricsRequest struct {
 func (m *GetMetricsRequest) Reset()      { *m = GetMetricsRequest{} }
 func (*GetMetricsRequest) ProtoMessage() {}
 func (*GetMetricsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{59}
+	return fileDescriptor_712ce9a559fda969, []int{60}
 }
 func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2596,7 +2635,7 @@ type Metrics struct {
 func (m *Metrics) Reset()      { *m = Metrics{} }
 func (*Metrics) ProtoMessage() {}
 func (*Metrics) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{60}
+	return fileDescriptor_712ce9a559fda969, []int{61}
 }
 func (m *Metrics) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2636,7 +2675,7 @@ type VolumeStatsRequest struct {
 func (m *VolumeStatsRequest) Reset()      { *m = VolumeStatsRequest{} }
 func (*VolumeStatsRequest) ProtoMessage() {}
 func (*VolumeStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{61}
+	return fileDescriptor_712ce9a559fda969, []int{62}
 }
 func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2677,7 +2716,7 @@ type ResizeVolumeRequest struct {
 func (m *ResizeVolumeRequest) Reset()      { *m = ResizeVolumeRequest{} }
 func (*ResizeVolumeRequest) ProtoMessage() {}
 func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{62}
+	return fileDescriptor_712ce9a559fda969, []int{63}
 }
 func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2745,6 +2784,7 @@ func init() {
 	proto.RegisterType((*Routes)(nil), "grpc.Routes")
 	proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest")
 	proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest")
+	proto.RegisterType((*UpdateEphemeralMountsRequest)(nil), "grpc.UpdateEphemeralMountsRequest")
 	proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest")
 	proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest")
 	proto.RegisterType((*ARPNeighbors)(nil), "grpc.ARPNeighbors")
@@ -2779,208 +2819,211 @@ func init() {
 }
 
 var fileDescriptor_712ce9a559fda969 = []byte{
-	// 3207 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x73, 0x1b, 0xc7,
-	0xd1, 0xc6, 0x83, 0x04, 0xd0, 0x78, 0x11, 0x4b, 0x8a, 0x82, 0x60, 0x9b, 0x96, 0x57, 0xb6, 0x4c,
-	0xd9, 0x9f, 0x48, 0x5b, 0x76, 0x59, 0x7e, 0x94, 0x3f, 0x7d, 0x24, 0x45, 0x93, 0xb4, 0x4d, 0x0b,
-	0xdf, 0x42, 0x8c, 0x53, 0x49, 0x25, 0x5b, 0xcb, 0xdd, 0x21, 0x38, 0x26, 0x76, 0x67, 0x3d, 0x3b,
-	0x0b, 0x91, 0x4e, 0x55, 0x2a, 0xa7, 0xe4, 0x96, 0x63, 0x6e, 0xf9, 0x03, 0xa9, 0xfc, 0x83, 0x5c,
-	0x73, 0x50, 0xe5, 0x94, 0x63, 0x2e, 0x49, 0xc5, 0xfa, 0x09, 0xf9, 0x05, 0xa9, 0x79, 0xed, 0x03,
-	0x0f, 0xda, 0x61, 0xa9, 0x2a, 0x17, 0xd4, 0x76, 0x4f, 0x4f, 0xbf, 0xa6, 0xa7, 0xa7, 0x7b, 0x06,
-	0xd0, 0x1f, 0x62, 0x76, 0x1a, 0x1f, 0x6f, 0xb8, 0xc4, 0xdf, 0x3c, 0x73, 0x98, 0x73, 0xd7, 0x25,
-	0x01, 0x73, 0x70, 0x80, 0x68, 0x34, 0x05, 0x47, 0xd4, 0xdd, 0x1c, 0xe1, 0xe3, 0x68, 0x33, 0xa4,
-	0x84, 0x11, 0x97, 0x8c, 0xd4, 0x57, 0xb4, 0xe9, 0x0c, 0x51, 0xc0, 0x36, 0x04, 0x60, 0x94, 0x87,
-	0x34, 0x74, 0x7b, 0x35, 0xe2, 0x62, 0x89, 0xe8, 0xd5, 0xdc, 0x48, 0x7f, 0xd6, 0xd9, 0x45, 0x88,
-	0x22, 0x05, 0xbc, 0x38, 0x24, 0x64, 0x38, 0x42, 0x92, 0xc7, 0x71, 0x7c, 0xb2, 0x89, 0xfc, 0x90,
-	0x5d, 0xc8, 0x41, 0xf3, 0xf7, 0x45, 0x58, 0xdd, 0xa1, 0xc8, 0x61, 0x68, 0x47, 0x2b, 0x60, 0xa1,
-	0x6f, 0x62, 0x14, 0x31, 0xe3, 0x55, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x16, 0x6e, 0x16, 0xd6,
-	0x6b, 0x56, 0x3d, 0xc1, 0x1d, 0x78, 0xc6, 0x75, 0xa8, 0xa0, 0x73, 0xe4, 0xf2, 0xd1, 0xa2, 0x18,
-	0x5d, 0xe4, 0xe0, 0x81, 0x67, 0xbc, 0x03, 0xf5, 0x88, 0x51, 0x1c, 0x0c, 0xed, 0x38, 0x42, 0xb4,
-	0x5b, 0xba, 0x59, 0x58, 0xaf, 0xdf, 0x5b, 0xda, 0xe0, 0x2a, 0x6f, 0x0c, 0xc4, 0xc0, 0x51, 0x84,
-	0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x86, 0x8a, 0x87, 0xc6, 0xd8, 0x45, 0x51, 0xb7, 0x7c, 0xb3,
-	0xb4, 0x5e, 0xbf, 0xd7, 0x90, 0xe4, 0x0f, 0x05, 0xd2, 0xd2, 0x83, 0xc6, 0x1d, 0xa8, 0x46, 0x8c,
-	0x50, 0x67, 0x88, 0xa2, 0xee, 0x82, 0x20, 0x6c, 0x6a, 0xbe, 0x02, 0x6b, 0x25, 0xc3, 0xc6, 0x4b,
-	0x50, 0x7a, 0xb4, 0x73, 0xd0, 0x5d, 0x14, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8,
-	0x05, 0xcd, 0xc8, 0x09, 0xbc, 0x63, 0x72, 0x6e, 0x87, 0xd8, 0x0b, 0xa2, 0x6e, 0xe5, 0x66, 0x61,
-	0xbd, 0x6a, 0x35, 0x14, 0xb2, 0xcf, 0x71, 0xe6, 0x47, 0x70, 0x6d, 0xc0, 0x1c, 0xca, 0xae, 0xe0,
-	0x1d, 0xf3, 0x08, 0x56, 0x2d, 0xe4, 0x93, 0xf1, 0x95, 0x5c, 0xdb, 0x85, 0x0a, 0xc3, 0x3e, 0x22,
-	0x31, 0x13, 0xae, 0x6d, 0x5a, 0x1a, 0x34, 0xff, 0x58, 0x00, 0x63, 0xf7, 0x1c, 0xb9, 0x7d, 0x4a,
-	0x5c, 0x14, 0x45, 0xff, 0xa5, 0xe5, 0x7a, 0x03, 0x2a, 0xa1, 0x54, 0xa0, 0x5b, 0x16, 0xe4, 0x6a,
-	0x15, 0xb4, 0x56, 0x7a, 0xd4, 0xfc, 0x1a, 0x56, 0x06, 0x78, 0x18, 0x38, 0xa3, 0xe7, 0xa8, 0xef,
-	0x2a, 0x2c, 0x46, 0x82, 0xa7, 0x50, 0xb5, 0x69, 0x29, 0xc8, 0xec, 0x83, 0xf1, 0x95, 0x83, 0xd9,
-	0xf3, 0x93, 0x64, 0xde, 0x85, 0xe5, 0x1c, 0xc7, 0x28, 0x24, 0x41, 0x84, 0x84, 0x02, 0xcc, 0x61,
-	0x71, 0x24, 0x98, 0x2d, 0x58, 0x0a, 0x32, 0x09, 0xac, 0x1e, 0x85, 0xde, 0x15, 0x77, 0xd3, 0x3d,
-	0xa8, 0x51, 0x14, 0x91, 0x98, 0xf2, 0x3d, 0x50, 0x14, 0x4e, 0x5d, 0x91, 0x4e, 0xfd, 0x02, 0x07,
-	0xf1, 0xb9, 0xa5, 0xc7, 0xac, 0x94, 0x4c, 0xc5, 0x27, 0x8b, 0xae, 0x12, 0x9f, 0x1f, 0xc1, 0xb5,
-	0xbe, 0x13, 0x47, 0x57, 0xd1, 0xd5, 0xfc, 0x98, 0xc7, 0x76, 0x14, 0xfb, 0x57, 0x9a, 0xfc, 0x87,
-	0x02, 0x54, 0x77, 0xc2, 0xf8, 0x28, 0x72, 0x86, 0xc8, 0x78, 0x05, 0xea, 0x8c, 0x30, 0x67, 0x64,
-	0xc7, 0x1c, 0x14, 0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x0a, 0x8d, 0x10, 0x51, 0x37, 0x8c,
-	0x15, 0x45, 0xf1, 0x66, 0x69, 0xbd, 0x6c, 0xd5, 0x25, 0x4e, 0x92, 0x6c, 0xc0, 0xb2, 0x18, 0xb3,
-	0x71, 0x60, 0x9f, 0x21, 0x1a, 0xa0, 0x91, 0x4f, 0x3c, 0x24, 0x82, 0xa3, 0x6c, 0x75, 0xc4, 0xd0,
-	0x41, 0xf0, 0x79, 0x32, 0x60, 0xbc, 0x09, 0x9d, 0x84, 0x9e, 0x47, 0xbc, 0xa0, 0x2e, 0x0b, 0xea,
-	0xb6, 0xa2, 0x3e, 0x52, 0x68, 0xf3, 0x97, 0xd0, 0x7a, 0x7c, 0x4a, 0x09, 0x63, 0x23, 0x1c, 0x0c,
-	0x1f, 0x3a, 0xcc, 0xe1, 0x5b, 0x33, 0x44, 0x14, 0x13, 0x2f, 0x52, 0xda, 0x6a, 0xd0, 0x78, 0x0b,
-	0x3a, 0x4c, 0xd2, 0x22, 0xcf, 0xd6, 0x34, 0x45, 0x41, 0xb3, 0x94, 0x0c, 0xf4, 0x15, 0xf1, 0xeb,
-	0xd0, 0x4a, 0x89, 0xf9, 0xe6, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x63, 0xec, 0x23, 0x73, 0x2c, 0x7c,
-	0x25, 0x16, 0xd9, 0x78, 0x0b, 0x6a, 0xa9, 0x1f, 0x0a, 0x22, 0x42, 0x5a, 0x32, 0x42, 0xb4, 0x3b,
-	0xad, 0x6a, 0xe2, 0x94, 0x4f, 0xa0, 0xcd, 0x12, 0xc5, 0x6d, 0xcf, 0x61, 0x4e, 0x3e, 0xa8, 0xf2,
-	0x56, 0x59, 0x2d, 0x96, 0x83, 0xcd, 0x8f, 0xa1, 0xd6, 0xc7, 0x5e, 0x24, 0x05, 0x77, 0xa1, 0xe2,
-	0xc6, 0x94, 0xa2, 0x80, 0x69, 0x93, 0x15, 0x68, 0xac, 0xc0, 0xc2, 0x08, 0xfb, 0x98, 0x29, 0x33,
-	0x25, 0x60, 0x12, 0x80, 0x43, 0xe4, 0x13, 0x7a, 0x21, 0x1c, 0xb6, 0x02, 0x0b, 0xd9, 0xc5, 0x95,
-	0x80, 0xf1, 0x22, 0xd4, 0x7c, 0xe7, 0x3c, 0x59, 0x54, 0x3e, 0x52, 0xf5, 0x9d, 0x73, 0xa9, 0x7c,
-	0x17, 0x2a, 0x27, 0x0e, 0x1e, 0xb9, 0x01, 0x53, 0x5e, 0xd1, 0x60, 0x2a, 0xb0, 0x9c, 0x15, 0xf8,
-	0xe7, 0x22, 0xd4, 0xa5, 0x44, 0xa9, 0xf0, 0x0a, 0x2c, 0xb8, 0x8e, 0x7b, 0x9a, 0x88, 0x14, 0x80,
-	0x71, 0x5b, 0x2b, 0x52, 0xcc, 0x66, 0xb8, 0x54, 0x53, 0xad, 0xda, 0x26, 0x40, 0xf4, 0xc4, 0x09,
-	0x95, 0x6e, 0xa5, 0x39, 0xc4, 0x35, 0x4e, 0x23, 0xd5, 0x7d, 0x17, 0x1a, 0x32, 0xee, 0xd4, 0x94,
-	0xf2, 0x9c, 0x29, 0x75, 0x49, 0x25, 0x27, 0xdd, 0x82, 0x66, 0x1c, 0x21, 0xfb, 0x14, 0x23, 0xea,
-	0x50, 0xf7, 0xf4, 0xa2, 0xbb, 0x20, 0x0f, 0xa0, 0x38, 0x42, 0xfb, 0x1a, 0x67, 0xdc, 0x83, 0x05,
-	0x9e, 0x5b, 0xa2, 0xee, 0xa2, 0x38, 0xeb, 0x5e, 0xca, 0xb2, 0x14, 0xa6, 0x6e, 0x88, 0xdf, 0xdd,
-	0x80, 0xd1, 0x0b, 0x4b, 0x92, 0xf6, 0x3e, 0x00, 0x48, 0x91, 0xc6, 0x12, 0x94, 0xce, 0xd0, 0x85,
-	0xda, 0x87, 0xfc, 0x93, 0x3b, 0x67, 0xec, 0x8c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x54, 0xfc, 0xa0,
-	0x60, 0xba, 0xd0, 0xde, 0x1e, 0x9d, 0x61, 0x92, 0x99, 0xbe, 0x02, 0x0b, 0xbe, 0xf3, 0x35, 0xa1,
-	0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84, 0xc2,
-	0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x47, 0x19, 0x20, 0x95, 0x62,
-	0x58, 0xd0, 0xc3, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x7c, 0xc1, 0x50, 0x64, 0x53, 0xe4,
-	0xc6, 0x34, 0xc2, 0x63, 0xbe, 0x7e, 0xdc, 0xec, 0x6b, 0xd2, 0xec, 0x09, 0xdd, 0xac, 0xeb, 0x98,
-	0x0c, 0xe4, 0xbc, 0x6d, 0x3e, 0xcd, 0xd2, 0xb3, 0x8c, 0x03, 0xb8, 0x96, 0xf2, 0xf4, 0x32, 0xec,
-	0x8a, 0x97, 0xb1, 0x5b, 0x4e, 0xd8, 0x79, 0x29, 0xab, 0x5d, 0x58, 0xc6, 0xc4, 0xfe, 0x26, 0x46,
-	0x71, 0x8e, 0x51, 0xe9, 0x32, 0x46, 0x1d, 0x4c, 0xfe, 0x5f, 0x4c, 0x48, 0xd9, 0xf4, 0xe1, 0x46,
-	0xc6, 0x4a, 0xbe, 0xdd, 0x33, 0xcc, 0xca, 0x97, 0x31, 0x5b, 0x4d, 0xb4, 0xe2, 0xf9, 0x20, 0xe5,
-	0xf8, 0x19, 0xac, 0x62, 0x62, 0x3f, 0x71, 0x30, 0x9b, 0x64, 0xb7, 0xf0, 0x3d, 0x46, 0xf2, 0x13,
-	0x2d, 0xcf, 0x4b, 0x1a, 0xe9, 0x23, 0x3a, 0xcc, 0x19, 0xb9, 0xf8, 0x3d, 0x46, 0x1e, 0x8a, 0x09,
-	0x29, 0x9b, 0x2d, 0xe8, 0x60, 0x32, 0xa9, 0x4d, 0xe5, 0x32, 0x26, 0x6d, 0x4c, 0xf2, 0x9a, 0x6c,
-	0x43, 0x27, 0x42, 0x2e, 0x23, 0x34, 0x1b, 0x04, 0xd5, 0xcb, 0x58, 0x2c, 0x29, 0xfa, 0x84, 0x87,
-	0xf9, 0x53, 0x68, 0xec, 0xc7, 0x43, 0xc4, 0x46, 0xc7, 0x49, 0x32, 0x78, 0x6e, 0xf9, 0xc7, 0xfc,
-	0x57, 0x11, 0xea, 0x3b, 0x43, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xcc, 0xc9, 0x82,
-	0x44, 0xe4, 0x64, 0x49, 0xfc, 0x1e, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d, 0xa9,
-	0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x00, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d, 0xb5,
-	0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x0e, 0xd4, 0x8f, 0xb9, 0x93, 0xd4,
-	0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x71, 0xba, 0x09, 0xf7, 0xa1, 0x79, 0x2a, 0x5d, 0xa6,
-	0x26, 0xc9, 0x18, 0xba, 0xa5, 0x2c, 0x49, 0xed, 0xdd, 0xc8, 0x7a, 0x56, 0x2e, 0x40, 0xe3, 0x34,
-	0x83, 0xea, 0x0d, 0xa0, 0x33, 0x45, 0x32, 0x23, 0x07, 0xad, 0x67, 0x73, 0x50, 0xfd, 0x9e, 0x21,
-	0x05, 0x65, 0x67, 0x66, 0xf3, 0xd2, 0x6f, 0x8b, 0xd0, 0xf8, 0x12, 0xb1, 0x27, 0x84, 0x9e, 0x49,
-	0x7d, 0x0d, 0x28, 0x07, 0x8e, 0x8f, 0x14, 0x47, 0xf1, 0x6d, 0xdc, 0x80, 0x2a, 0x3d, 0x97, 0x09,
-	0x44, 0xad, 0x67, 0x85, 0x9e, 0x8b, 0xc4, 0x60, 0xbc, 0x0c, 0x40, 0xcf, 0xed, 0xd0, 0x71, 0xcf,
-	0x90, 0xf2, 0x60, 0xd9, 0xaa, 0xd1, 0xf3, 0xbe, 0x44, 0xf0, 0x50, 0xa0, 0xe7, 0x36, 0xa2, 0x94,
-	0xd0, 0x48, 0xe5, 0xaa, 0x2a, 0x3d, 0xdf, 0x15, 0xb0, 0x9a, 0xeb, 0x51, 0x12, 0x86, 0xc8, 0x13,
-	0x39, 0x5a, 0xcc, 0x7d, 0x28, 0x11, 0x5c, 0x2a, 0xd3, 0x52, 0x17, 0xa5, 0x54, 0x96, 0x4a, 0x65,
-	0xa9, 0xd4, 0x8a, 0x9c, 0xc9, 0xb2, 0x52, 0x59, 0x22, 0xb5, 0x2a, 0xa5, 0xb2, 0x8c, 0x54, 0x96,
-	0x4a, 0xad, 0xe9, 0xb9, 0x4a, 0xaa, 0xf9, 0x9b, 0x02, 0xac, 0x4e, 0x16, 0x7e, 0xaa, 0x36, 0x7d,
-	0x0f, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0xa6, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30, 0xbe,
-	0x0f, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, 0x03,
-	0x99, 0x1e, 0x18, 0x5f, 0x51, 0xcc, 0xd0, 0x80, 0x51, 0xe4, 0xf8, 0xcf, 0xa3, 0xba, 0x37, 0xa0,
-	0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0xc0, 0x72, 0x4e, 0x8a, 0xb2, 0x75,
-	0x09, 0x4a, 0x23, 0x14, 0x08, 0xee, 0x4d, 0x8b, 0x7f, 0x9a, 0x0e, 0x74, 0x2c, 0xe4, 0x78, 0xcf,
-	0x4f, 0x1b, 0x25, 0xa2, 0x94, 0x8a, 0x58, 0x07, 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x64,
-	0xb4, 0x7e, 0x04, 0x9d, 0x9d, 0x11, 0x89, 0xd0, 0x80, 0x79, 0x38, 0x78, 0x1e, 0xed, 0xc8, 0x2f,
-	0x60, 0xf9, 0x31, 0xbb, 0xf8, 0x8a, 0x33, 0x8b, 0xf0, 0xb7, 0xe8, 0x39, 0xd9, 0x47, 0xc9, 0x13,
-	0x6d, 0x1f, 0x25, 0x4f, 0x78, 0x73, 0xe3, 0x92, 0x51, 0xec, 0x07, 0x62, 0x2b, 0x34, 0x2d, 0x05,
-	0x99, 0xdb, 0xd0, 0x90, 0x35, 0xf4, 0x21, 0xf1, 0xe2, 0x11, 0x9a, 0xb9, 0x07, 0xd7, 0x00, 0x42,
-	0x87, 0x3a, 0x3e, 0x62, 0x88, 0xca, 0x18, 0xaa, 0x59, 0x19, 0x8c, 0xf9, 0xbb, 0x22, 0xac, 0xc8,
-	0xfb, 0x86, 0x81, 0x6c, 0xb3, 0xb5, 0x09, 0x3d, 0xa8, 0x9e, 0x92, 0x88, 0x65, 0x18, 0x26, 0x30,
-	0x57, 0x91, 0xf7, 0xe7, 0x92, 0x1b, 0xff, 0xcc, 0x5d, 0x02, 0x94, 0x2e, 0xbf, 0x04, 0x98, 0x6a,
-	0xf3, 0xcb, 0xd3, 0x6d, 0x3e, 0xdf, 0x6d, 0x9a, 0x08, 0xcb, 0x3d, 0x5e, 0xb3, 0x6a, 0x0a, 0x73,
-	0xe0, 0x19, 0xb7, 0xa1, 0x3d, 0xe4, 0x5a, 0xda, 0xa7, 0x84, 0x9c, 0xd9, 0xa1, 0xc3, 0x4e, 0xc5,
-	0x56, 0xaf, 0x59, 0x4d, 0x81, 0xde, 0x27, 0xe4, 0xac, 0xef, 0xb0, 0x53, 0xe3, 0x43, 0x68, 0xa9,
-	0x32, 0xd0, 0x17, 0x2e, 0x8a, 0xd4, 0xe1, 0xa7, 0x76, 0x51, 0xd6, 0x7b, 0x56, 0xf3, 0x2c, 0x03,
-	0x45, 0xe6, 0x75, 0xb8, 0xf6, 0x10, 0x45, 0x8c, 0x92, 0x8b, 0xbc, 0x63, 0xcc, 0xff, 0x05, 0x38,
-	0x08, 0x18, 0xa2, 0x27, 0x8e, 0x8b, 0x22, 0xe3, 0xed, 0x2c, 0xa4, 0x8a, 0xa3, 0xa5, 0x0d, 0x79,
-	0xdd, 0x93, 0x0c, 0x58, 0x19, 0x1a, 0x73, 0x03, 0x16, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0x9a, 0xfe,
-	0x52, 0xf3, 0x1a, 0x6a, 0x9e, 0x40, 0x5a, 0x6a, 0xcc, 0xdc, 0xd7, 0x2d, 0x6c, 0xca, 0x4e, 0x2d,
-	0xd1, 0x06, 0xd4, 0xb0, 0xc6, 0xa9, 0xac, 0x32, 0x2d, 0x3a, 0x25, 0x31, 0x3f, 0x86, 0x65, 0xc9,
-	0x49, 0x72, 0xd6, 0x6c, 0x5e, 0x83, 0x45, 0xaa, 0xd5, 0x28, 0xa4, 0xf7, 0x3c, 0x8a, 0x48, 0x8d,
-	0x71, 0x7f, 0x7c, 0x81, 0x23, 0x96, 0x1a, 0xa2, 0xfd, 0xb1, 0x0c, 0x1d, 0x3e, 0x90, 0xe3, 0x69,
-	0x7e, 0x0a, 0x8d, 0x2d, 0xab, 0xff, 0x25, 0xc2, 0xc3, 0xd3, 0x63, 0x9e, 0x3d, 0xdf, 0xcf, 0xc3,
-	0xca, 0x60, 0x43, 0x69, 0x9b, 0x19, 0xb2, 0x72, 0x74, 0xe6, 0x67, 0xb0, 0xba, 0xe5, 0x79, 0x59,
-	0x94, 0xd6, 0xfa, 0x6d, 0xa8, 0x05, 0x19, 0x76, 0x99, 0x33, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x77,
-	0xc1, 0xd8, 0x43, 0xec, 0xa0, 0xff, 0xd8, 0x39, 0x1e, 0xa5, 0xd6, 0x5f, 0x87, 0x0a, 0x8e, 0x6c,
-	0x1c, 0x8e, 0xdf, 0x17, 0x5c, 0xaa, 0xd6, 0x22, 0x8e, 0x0e, 0xc2, 0xf1, 0xfb, 0xe6, 0x1d, 0x58,
-	0xce, 0x91, 0x5f, 0x92, 0x56, 0xb6, 0xc0, 0x18, 0xfc, 0x70, 0xce, 0x09, 0x8b, 0x62, 0x86, 0xc5,
-	0x1d, 0x58, 0x1e, 0xfc, 0x40, 0x69, 0x3f, 0x83, 0xe5, 0x47, 0xc1, 0x08, 0x07, 0x68, 0xa7, 0x7f,
-	0x74, 0x88, 0x92, 0x9c, 0x6a, 0x40, 0x99, 0xd7, 0x9e, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0xc7,
-	0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xb4, 0x5a, 0x0c, 0x8e, 0x77, 0xc2, 0x38, 0xe2, 0x87, 0x24, 0x2f,
-	0x92, 0x48, 0x30, 0xba, 0x10, 0x99, 0xa6, 0x6a, 0x55, 0xdc, 0x30, 0x7e, 0x14, 0x8c, 0x2e, 0xcc,
-	0xff, 0x11, 0x37, 0x09, 0x08, 0x79, 0x96, 0x13, 0x78, 0xc4, 0x7f, 0x88, 0xc6, 0x19, 0x09, 0x53,
-	0x7a, 0x3f, 0x2d, 0x40, 0x63, 0x6b, 0x88, 0x02, 0xf6, 0x10, 0x31, 0x07, 0x8f, 0x44, 0x67, 0x3a,
-	0x46, 0x34, 0xc2, 0x24, 0x50, 0x69, 0x43, 0x83, 0xc6, 0x2b, 0x50, 0xc7, 0x01, 0x66, 0xb6, 0xe7,
-	0x20, 0x9f, 0x04, 0x82, 0x4b, 0xd5, 0x02, 0x8e, 0x7a, 0x28, 0x30, 0xc6, 0x1b, 0xd0, 0x96, 0x97,
-	0x8a, 0xf6, 0xa9, 0x13, 0x78, 0x23, 0x9e, 0xb0, 0x4a, 0x22, 0xc5, 0xb4, 0x24, 0x7a, 0x5f, 0x61,
-	0x8d, 0x3b, 0xb0, 0xa4, 0xd2, 0x49, 0x4a, 0x59, 0x16, 0x94, 0x6d, 0x85, 0xcf, 0x91, 0xc6, 0x61,
-	0x48, 0x28, 0x8b, 0xec, 0x08, 0xb9, 0x2e, 0xf1, 0x43, 0xd5, 0xd6, 0xb5, 0x35, 0x7e, 0x20, 0xd1,
-	0xe6, 0x10, 0x96, 0xf7, 0xb8, 0x9d, 0xca, 0x92, 0x74, 0x7b, 0xb4, 0x7c, 0xe4, 0xdb, 0xc7, 0x23,
-	0xe2, 0x9e, 0xd9, 0x3c, 0xc9, 0x2b, 0x0f, 0xf3, 0xc2, 0x71, 0x9b, 0x23, 0x07, 0xf8, 0x5b, 0x71,
-	0x83, 0xc1, 0xa9, 0x4e, 0x09, 0x0b, 0x47, 0xf1, 0xd0, 0x0e, 0x29, 0x39, 0x46, 0xca, 0xc4, 0xb6,
-	0x8f, 0xfc, 0x7d, 0x89, 0xef, 0x73, 0xb4, 0xf9, 0xa7, 0x02, 0xac, 0xe4, 0x25, 0xa9, 0xd5, 0xde,
-	0x84, 0x95, 0xbc, 0x28, 0x55, 0xc6, 0xc8, 0x32, 0xb9, 0x93, 0x15, 0x28, 0x0b, 0x9a, 0xfb, 0xd0,
-	0x14, 0x57, 0xd0, 0xb6, 0x27, 0x39, 0xe5, 0x8b, 0xb7, 0xec, 0xba, 0x58, 0x0d, 0x27, 0xbb, 0x4a,
-	0x1f, 0xc2, 0x0d, 0x65, 0xbe, 0x3d, 0xad, 0xb6, 0x0c, 0x88, 0x55, 0x45, 0x70, 0x38, 0xa1, 0xfd,
-	0x17, 0xd0, 0x4d, 0x51, 0xdb, 0x17, 0x02, 0x99, 0x6e, 0xca, 0xe5, 0x09, 0x63, 0xb7, 0x3c, 0x8f,
-	0x8a, 0xdd, 0x5e, 0xb6, 0x66, 0x0d, 0x99, 0x0f, 0xe0, 0xfa, 0x00, 0x31, 0xe9, 0x0d, 0x87, 0xa9,
-	0x8e, 0x4a, 0x32, 0x5b, 0x82, 0xd2, 0x00, 0xb9, 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e,
-	0x45, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d, 0x86, 0x50, 0xf9, 0x74, 0xb0, 0xc7, 0xeb, 0x26,
-	0x1e, 0xd4, 0xb2, 0xce, 0x52, 0x67, 0x6a, 0xd3, 0xaa, 0x08, 0xf8, 0xc0, 0x33, 0x3e, 0x83, 0x65,
-	0x39, 0xe4, 0x9e, 0x3a, 0xc1, 0x10, 0xd9, 0x21, 0x19, 0x61, 0x57, 0x86, 0x7e, 0xeb, 0x5e, 0x4f,
-	0xa5, 0x21, 0xc5, 0x67, 0x47, 0x90, 0xf4, 0x05, 0x85, 0xd5, 0x19, 0x4e, 0xa2, 0xcc, 0xbf, 0x17,
-	0xa0, 0xa2, 0x8e, 0x35, 0x7e, 0x34, 0x7b, 0x14, 0x8f, 0x11, 0x55, 0xc1, 0xae, 0x20, 0xe3, 0x75,
-	0x68, 0xc9, 0x2f, 0x9b, 0x84, 0x0c, 0x93, 0xe4, 0xb0, 0x6c, 0x4a, 0xec, 0x23, 0x89, 0x14, 0xd7,
-	0x96, 0xe2, 0xe2, 0x50, 0xf5, 0xe8, 0x0a, 0xe2, 0xf8, 0x93, 0x88, 0x2b, 0x25, 0x0e, 0xc7, 0x9a,
-	0xa5, 0x20, 0xbe, 0xb9, 0x34, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, 0x73, 0xf9, 0x24, 0x0e, 0x98,
-	0x1d, 0x12, 0x1c, 0x30, 0x75, 0x1a, 0x82, 0x40, 0xf5, 0x39, 0xc6, 0x58, 0x87, 0xea, 0x49, 0x64,
-	0x0b, 0x6b, 0x44, 0xe5, 0x9b, 0x9c, 0xd0, 0xca, 0x6a, 0xab, 0x72, 0x12, 0x89, 0x0f, 0xf3, 0xd7,
-	0x05, 0x58, 0x94, 0x97, 0xfc, 0x46, 0x0b, 0x8a, 0x49, 0xf5, 0x52, 0xc4, 0xa2, 0x12, 0x14, 0x5a,
-	0xc9, 0x8a, 0x45, 0x7c, 0xf3, 0x1c, 0x33, 0xf6, 0xe5, 0x19, 0xac, 0x8c, 0x18, 0xfb, 0xe2, 0xf0,
-	0x7d, 0x1d, 0x5a, 0x69, 0x11, 0x24, 0xc6, 0xa5, 0x31, 0xcd, 0x04, 0x2b, 0xc8, 0xe6, 0xda, 0x64,
-	0xfe, 0x18, 0x20, 0xbd, 0xec, 0xe6, 0xe1, 0x10, 0x27, 0xca, 0xf0, 0x4f, 0x8e, 0x19, 0x26, 0xe5,
-	0x13, 0xff, 0x34, 0x6e, 0x43, 0xcb, 0xf1, 0x3c, 0xcc, 0xa7, 0x3b, 0xa3, 0x3d, 0xec, 0x25, 0x09,
-	0x24, 0x8f, 0x35, 0xff, 0x52, 0x80, 0xf6, 0x0e, 0x09, 0x2f, 0x3e, 0xc5, 0x23, 0x94, 0xc9, 0x6e,
-	0x42, 0x49, 0x55, 0x3d, 0xf1, 0x6f, 0xde, 0x11, 0x9c, 0xe0, 0x11, 0x92, 0xdb, 0x5e, 0x46, 0x5d,
-	0x95, 0x23, 0xc4, 0x96, 0xd7, 0x83, 0xc9, 0xd5, 0x66, 0x53, 0x0e, 0x1e, 0x12, 0x4f, 0xf4, 0x3e,
-	0x1e, 0xa6, 0x76, 0x72, 0x91, 0xd9, 0xb4, 0x2a, 0x1e, 0xa6, 0x62, 0x48, 0x19, 0xb2, 0x20, 0x2e,
-	0xaa, 0xb3, 0x86, 0x2c, 0x4a, 0x0c, 0x37, 0x64, 0x15, 0x16, 0xc9, 0xc9, 0x49, 0x84, 0x98, 0x58,
-	0xab, 0x92, 0xa5, 0xa0, 0x24, 0x05, 0x57, 0x33, 0x29, 0x78, 0x45, 0x9c, 0x6b, 0x8f, 0x1e, 0x1d,
-	0xee, 0x8e, 0x51, 0xc0, 0xf4, 0x09, 0x7c, 0x17, 0xaa, 0x1a, 0xf5, 0x43, 0xae, 0x80, 0xdf, 0x84,
-	0xd6, 0x96, 0xe7, 0x0d, 0x9e, 0x38, 0xa1, 0xf6, 0x47, 0x17, 0x2a, 0xfd, 0x9d, 0x83, 0xbe, 0x74,
-	0x49, 0x89, 0x1b, 0xa0, 0x40, 0x7e, 0xe2, 0xef, 0x21, 0x76, 0x88, 0x18, 0xc5, 0x6e, 0x72, 0xe2,
-	0xdf, 0x82, 0x8a, 0xc2, 0xf0, 0x99, 0xbe, 0xfc, 0xd4, 0x47, 0x80, 0x02, 0xcd, 0xff, 0x03, 0xe3,
-	0x47, 0xbc, 0x76, 0x45, 0xb2, 0x71, 0x51, 0x92, 0xde, 0x84, 0xce, 0x58, 0x60, 0x6d, 0x59, 0xd4,
-	0x65, 0x96, 0xa1, 0x2d, 0x07, 0x44, 0x7e, 0x10, 0xb2, 0x8f, 0x60, 0x59, 0x96, 0xda, 0x92, 0xcf,
-	0x15, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6, 0xc4, 0xf7, 0xbd, 0xa7, 0x86, 0x3a, 0xc6, 0xd4,
-	0xcd, 0x8e, 0xb1, 0x07, 0xed, 0x89, 0x67, 0x38, 0x43, 0x5d, 0xf5, 0xcd, 0x7e, 0x9d, 0xeb, 0xad,
-	0x6e, 0xc8, 0x67, 0xbd, 0x0d, 0xfd, 0xac, 0xb7, 0xb1, 0xeb, 0x87, 0xec, 0xc2, 0xd8, 0x85, 0x56,
-	0xfe, 0xc1, 0xca, 0x78, 0x51, 0x57, 0xc6, 0x33, 0x9e, 0xb1, 0xe6, 0xb2, 0xd9, 0x83, 0xf6, 0xc4,
-	0xdb, 0x95, 0xd6, 0x67, 0xf6, 0x93, 0xd6, 0x5c, 0x46, 0x0f, 0xa0, 0x9e, 0x79, 0xac, 0x32, 0xba,
-	0x92, 0xc9, 0xf4, 0xfb, 0xd5, 0x5c, 0x06, 0x3b, 0xd0, 0xcc, 0xbd, 0x1f, 0x19, 0x3d, 0x65, 0xcf,
-	0x8c, 0x47, 0xa5, 0xb9, 0x4c, 0xb6, 0xa1, 0x9e, 0x79, 0xc6, 0xd1, 0x5a, 0x4c, 0xbf, 0x15, 0xf5,
-	0x6e, 0xcc, 0x18, 0x51, 0xa7, 0xe5, 0x1e, 0xb4, 0x27, 0xde, 0x76, 0xb4, 0x4b, 0x66, 0x3f, 0xf9,
-	0xcc, 0x55, 0xe6, 0x73, 0xb1, 0x44, 0x99, 0xd6, 0x3d, 0xb3, 0x44, 0xd3, 0x2f, 0x39, 0xbd, 0x97,
-	0x66, 0x0f, 0x2a, 0xad, 0x76, 0xa1, 0x95, 0x7f, 0xc4, 0xd1, 0xcc, 0x66, 0x3e, 0xed, 0x5c, 0xbe,
-	0xde, 0xb9, 0xf7, 0x9c, 0x74, 0xbd, 0x67, 0x3d, 0xf3, 0xcc, 0x65, 0xb4, 0x05, 0xa0, 0x1a, 0x75,
-	0x0f, 0x07, 0x89, 0xa3, 0xa7, 0x2e, 0x08, 0x12, 0x47, 0xcf, 0x68, 0xea, 0x1f, 0x00, 0xc8, 0xfe,
-	0xda, 0x23, 0x31, 0x33, 0xae, 0x6b, 0x35, 0x26, 0x9a, 0xfa, 0x5e, 0x77, 0x7a, 0x60, 0x8a, 0x01,
-	0xa2, 0xf4, 0x2a, 0x0c, 0x3e, 0x01, 0x48, 0xfb, 0x76, 0xcd, 0x60, 0xaa, 0x93, 0xbf, 0xc4, 0x07,
-	0x8d, 0x6c, 0x97, 0x6e, 0x28, 0x5b, 0x67, 0x74, 0xee, 0x97, 0xb0, 0x68, 0x4f, 0x74, 0x61, 0xf9,
-	0x60, 0x9b, 0x6c, 0xce, 0x7a, 0x53, 0x9d, 0x98, 0x71, 0x1f, 0x1a, 0xd9, 0xf6, 0x4b, 0x6b, 0x31,
-	0xa3, 0x25, 0xeb, 0xe5, 0x5a, 0x30, 0xe3, 0x01, 0xb4, 0xf2, 0xad, 0x97, 0x0e, 0xa9, 0x99, 0x0d,
-	0x59, 0x4f, 0x5d, 0x2c, 0x66, 0xc8, 0xdf, 0x05, 0x48, 0x5b, 0x34, 0xed, 0xbe, 0xa9, 0xa6, 0x6d,
-	0x42, 0xea, 0x1e, 0xb4, 0x27, 0x5a, 0x2f, 0x6d, 0xf1, 0xec, 0x8e, 0xec, 0xb2, 0xbd, 0x9e, 0x69,
-	0xa4, 0x74, 0x08, 0x4e, 0xb7, 0x62, 0x3a, 0x04, 0x67, 0x75, 0x5d, 0xdb, 0x50, 0x1f, 0x4c, 0xf3,
-	0x18, 0xcc, 0xe5, 0x31, 0xab, 0x97, 0x7a, 0x0f, 0x20, 0x3d, 0xb6, 0xb4, 0x17, 0xa6, 0x0e, 0xb2,
-	0x5e, 0x53, 0x5f, 0xfe, 0x4a, 0xba, 0x1d, 0x68, 0xe6, 0xee, 0x47, 0x74, 0xba, 0x9b, 0x75, 0x69,
-	0x72, 0xd9, 0x21, 0x90, 0xbf, 0x4c, 0xd0, 0x2b, 0x38, 0xf3, 0x8a, 0xe1, 0xb2, 0x38, 0xce, 0x76,
-	0x7e, 0x3a, 0x82, 0x66, 0x74, 0x83, 0xdf, 0x93, 0x57, 0xb2, 0xdd, 0x5d, 0x26, 0xaf, 0xcc, 0x68,
-	0xfa, 0xe6, 0x32, 0xda, 0x87, 0xf6, 0x9e, 0x2e, 0xdc, 0x55, 0x53, 0xa1, 0xd7, 0x6f, 0xba, 0x89,
-	0xea, 0xf5, 0x66, 0x0d, 0xa9, 0x75, 0xf9, 0x1c, 0x3a, 0x53, 0x0d, 0x85, 0xb1, 0x96, 0x5c, 0xc1,
-	0xcf, 0xec, 0x34, 0xe6, 0xaa, 0x75, 0x00, 0x4b, 0x93, 0xfd, 0x84, 0xf1, 0x72, 0x12, 0x13, 0xb3,
-	0xfa, 0x8c, 0xb9, 0xac, 0x3e, 0x84, 0xaa, 0xae, 0x11, 0x0d, 0xf5, 0xd4, 0x31, 0x51, 0x33, 0xce,
-	0x9d, 0x7a, 0x5f, 0x84, 0x7c, 0x52, 0x7f, 0xa5, 0x21, 0x3f, 0x51, 0xa5, 0xf5, 0xd4, 0xcb, 0x44,
-	0x42, 0x79, 0x1f, 0x2a, 0xaa, 0x0c, 0x33, 0x56, 0x92, 0xcd, 0x96, 0xa9, 0xca, 0x2e, 0x8b, 0xb0,
-	0x3d, 0xc4, 0x32, 0xc5, 0x95, 0x16, 0x3a, 0x5d, 0x6f, 0xe9, 0x3d, 0x92, 0x1b, 0x51, 0x6b, 0xb1,
-	0x05, 0x8d, 0x6c, 0x79, 0xa5, 0x97, 0x74, 0x46, 0xc9, 0x35, 0x4f, 0x93, 0xed, 0xf3, 0xa7, 0xdf,
-	0xad, 0xbd, 0xf0, 0xb7, 0xef, 0xd6, 0x5e, 0xf8, 0xd5, 0xb3, 0xb5, 0xc2, 0xd3, 0x67, 0x6b, 0x85,
-	0xbf, 0x3e, 0x5b, 0x2b, 0xfc, 0xf3, 0xd9, 0x5a, 0xe1, 0x27, 0x3f, 0xff, 0x0f, 0xff, 0x73, 0x45,
-	0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x8e, 0x31, 0x65, 0x99, 0xa1, 0xf0, 0x6c, 0x28, 0xff, 0x78, 0x95,
-	0xf9, 0x3f, 0x16, 0xd7, 0xf2, 0x78, 0x51, 0xc0, 0xef, 0xfe, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x77,
-	0xb9, 0x27, 0x67, 0xdc, 0x25, 0x00, 0x00,
+	// 3249 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0xcb, 0x72, 0x1c, 0x47,
+	0x72, 0x9a, 0x07, 0x30, 0x33, 0x39, 0x2f, 0x4c, 0x0f, 0x08, 0x0e, 0x47, 0x14, 0x44, 0x35, 0x25,
+	0x0a, 0x94, 0x4c, 0x40, 0xa2, 0x14, 0xa2, 0x1e, 0x21, 0xd3, 0x00, 0x08, 0x01, 0x90, 0x04, 0x71,
+	0xdc, 0x43, 0x58, 0x0e, 0x3b, 0xec, 0x8e, 0x46, 0x77, 0x61, 0xa6, 0x84, 0xe9, 0xae, 0x56, 0x75,
+	0x35, 0x08, 0xc8, 0x11, 0x0e, 0x9f, 0xec, 0x9b, 0x8f, 0xbe, 0xe9, 0x07, 0x1c, 0xfe, 0x03, 0x5f,
+	0x7d, 0x50, 0xf8, 0xb4, 0xc7, 0xbd, 0xec, 0xc6, 0x8a, 0x9f, 0xb0, 0x5f, 0xb0, 0x51, 0xaf, 0x7e,
+	0xcc, 0x03, 0xd2, 0x22, 0x18, 0xb1, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x56, 0x56, 0x66,
+	0xd5, 0xc0, 0x60, 0x84, 0xd9, 0x38, 0x3e, 0xd9, 0x74, 0x89, 0xbf, 0x75, 0xe6, 0x30, 0xe7, 0x81,
+	0x4b, 0x02, 0xe6, 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x35, 0xc1, 0x27, 0xd1, 0x56,
+	0x48, 0x09, 0x23, 0x2e, 0x99, 0xa8, 0xaf, 0x68, 0xcb, 0x19, 0xa1, 0x80, 0x6d, 0x0a, 0xc0, 0x28,
+	0x8f, 0x68, 0xe8, 0xf6, 0x6b, 0xc4, 0xc5, 0x12, 0xd1, 0xaf, 0xb9, 0x91, 0xfe, 0xac, 0xb3, 0xcb,
+	0x10, 0x45, 0x0a, 0x78, 0x75, 0x44, 0xc8, 0x68, 0x82, 0x24, 0x8f, 0x93, 0xf8, 0x74, 0x0b, 0xf9,
+	0x21, 0xbb, 0x94, 0x83, 0xe6, 0x8f, 0x45, 0x58, 0xdb, 0xa5, 0xc8, 0x61, 0x68, 0x57, 0x2b, 0x60,
+	0xa1, 0xef, 0x63, 0x14, 0x31, 0xe3, 0x0d, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x15, 0xee, 0x14,
+	0x36, 0x6a, 0x56, 0x3d, 0xc1, 0x1d, 0x7a, 0xc6, 0x4d, 0xa8, 0xa0, 0x0b, 0xe4, 0xf2, 0xd1, 0xa2,
+	0x18, 0x5d, 0xe6, 0xe0, 0xa1, 0x67, 0xbc, 0x0f, 0xf5, 0x88, 0x51, 0x1c, 0x8c, 0xec, 0x38, 0x42,
+	0xb4, 0x57, 0xba, 0x53, 0xd8, 0xa8, 0x3f, 0x5c, 0xd9, 0xe4, 0x2a, 0x6f, 0x0e, 0xc5, 0xc0, 0x71,
+	0x84, 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x83, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x5e, 0xf9,
+	0x4e, 0x69, 0xa3, 0xfe, 0xb0, 0x21, 0xc9, 0x9f, 0x08, 0xa4, 0xa5, 0x07, 0x8d, 0xfb, 0x50, 0x8d,
+	0x18, 0xa1, 0xce, 0x08, 0x45, 0xbd, 0x25, 0x41, 0xd8, 0xd4, 0x7c, 0x05, 0xd6, 0x4a, 0x86, 0x8d,
+	0xdb, 0x50, 0x7a, 0xba, 0x7b, 0xd8, 0x5b, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda,
+	0xb8, 0x0b, 0xcd, 0xc8, 0x09, 0xbc, 0x13, 0x72, 0x61, 0x87, 0xd8, 0x0b, 0xa2, 0x5e, 0xe5, 0x4e,
+	0x61, 0xa3, 0x6a, 0x35, 0x14, 0x72, 0xc0, 0x71, 0xe6, 0xa7, 0x70, 0x63, 0xc8, 0x1c, 0xca, 0xae,
+	0xe1, 0x1d, 0xf3, 0x18, 0xd6, 0x2c, 0xe4, 0x93, 0xf3, 0x6b, 0xb9, 0xb6, 0x07, 0x15, 0x86, 0x7d,
+	0x44, 0x62, 0x26, 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0x4f, 0x01, 0x8c, 0xbd, 0x0b, 0xe4, 0x0e,
+	0x28, 0x71, 0x51, 0x14, 0xfd, 0x85, 0x96, 0xeb, 0x6d, 0xa8, 0x84, 0x52, 0x81, 0x5e, 0x59, 0x90,
+	0xab, 0x55, 0xd0, 0x5a, 0xe9, 0x51, 0xf3, 0x3b, 0x58, 0x1d, 0xe2, 0x51, 0xe0, 0x4c, 0x5e, 0xa2,
+	0xbe, 0x6b, 0xb0, 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0x73, 0x00, 0xc6, 0xb7, 0x0e,
+	0x66, 0x2f, 0x4f, 0x92, 0xf9, 0x00, 0xba, 0x39, 0x8e, 0x51, 0x48, 0x82, 0x08, 0x09, 0x05, 0x98,
+	0xc3, 0xe2, 0x48, 0x30, 0x5b, 0xb2, 0x14, 0x64, 0x12, 0x58, 0x3b, 0x0e, 0xbd, 0x6b, 0xee, 0xa6,
+	0x87, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x7b, 0xa0, 0x28, 0x9c, 0xba, 0x2a, 0x9d, 0xfa, 0x35,
+	0x0e, 0xe2, 0x0b, 0x4b, 0x8f, 0x59, 0x29, 0x99, 0x8a, 0x4f, 0x16, 0x5d, 0x27, 0x3e, 0x3f, 0x85,
+	0x1b, 0x03, 0x27, 0x8e, 0xae, 0xa3, 0xab, 0xf9, 0x19, 0x8f, 0xed, 0x28, 0xf6, 0xaf, 0x35, 0xf9,
+	0xbf, 0x0b, 0x50, 0xdd, 0x0d, 0xe3, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0x75, 0xa8, 0x33, 0xc2, 0x9c,
+	0x89, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, 0x49, 0xf0, 0x06, 0x34, 0x42, 0x44, 0xdd,
+	0x30, 0x56, 0x14, 0xc5, 0x3b, 0xa5, 0x8d, 0xb2, 0x55, 0x97, 0x38, 0x49, 0xb2, 0x09, 0x5d, 0x31,
+	0x66, 0xe3, 0xc0, 0x3e, 0x43, 0x34, 0x40, 0x13, 0x9f, 0x78, 0x48, 0x04, 0x47, 0xd9, 0xea, 0x88,
+	0xa1, 0xc3, 0xe0, 0xab, 0x64, 0xc0, 0x78, 0x07, 0x3a, 0x09, 0x3d, 0x8f, 0x78, 0x41, 0x5d, 0x16,
+	0xd4, 0x6d, 0x45, 0x7d, 0xac, 0xd0, 0xe6, 0xbf, 0x42, 0xeb, 0xd9, 0x98, 0x12, 0xc6, 0x26, 0x38,
+	0x18, 0x3d, 0x71, 0x98, 0xc3, 0xb7, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1,
+	0x2e, 0x74, 0x98, 0xa4, 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x25, 0x19, 0x18, 0x28, 0xe2,
+	0xb7, 0xa0, 0x95, 0x12, 0xf3, 0xcd, 0xad, 0xf4, 0x6d, 0x26, 0xd8, 0x67, 0xd8, 0x47, 0xe6, 0xb9,
+	0xf0, 0x95, 0x58, 0x64, 0xe3, 0x5d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x08, 0x69, 0xc9, 0x08, 0xd1,
+	0xee, 0xb4, 0xaa, 0x89, 0x53, 0x3e, 0x87, 0x36, 0x4b, 0x14, 0xb7, 0x3d, 0x87, 0x39, 0xf9, 0xa0,
+	0xca, 0x5b, 0x65, 0xb5, 0x58, 0x0e, 0x36, 0x3f, 0x83, 0xda, 0x00, 0x7b, 0x91, 0x14, 0xdc, 0x83,
+	0x8a, 0x1b, 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x0a, 0x4b, 0x13, 0xec, 0x63, 0xa6,
+	0xcc, 0x94, 0x80, 0x49, 0x00, 0x8e, 0x90, 0x4f, 0xe8, 0xa5, 0x70, 0xd8, 0x2a, 0x2c, 0x65, 0x17,
+	0x57, 0x02, 0xc6, 0xab, 0x50, 0xf3, 0x9d, 0x8b, 0x64, 0x51, 0xf9, 0x48, 0xd5, 0x77, 0x2e, 0xa4,
+	0xf2, 0x3d, 0xa8, 0x9c, 0x3a, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x72, 0x56,
+	0xe0, 0xff, 0x15, 0xa1, 0x2e, 0x25, 0x4a, 0x85, 0x57, 0x61, 0xc9, 0x75, 0xdc, 0x71, 0x22, 0x52,
+	0x00, 0xc6, 0x3d, 0xad, 0x48, 0x31, 0x9b, 0xe1, 0x52, 0x4d, 0xb5, 0x6a, 0x5b, 0x00, 0xd1, 0x73,
+	0x27, 0x54, 0xba, 0x95, 0x16, 0x10, 0xd7, 0x38, 0x8d, 0x54, 0xf7, 0x03, 0x68, 0xc8, 0xb8, 0x53,
+	0x53, 0xca, 0x0b, 0xa6, 0xd4, 0x25, 0x95, 0x9c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x31, 0x46,
+	0xd4, 0xa1, 0xee, 0xf8, 0xb2, 0xb7, 0x24, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0x3c, 0x84,
+	0x25, 0x9e, 0x5b, 0xa2, 0xde, 0xb2, 0x38, 0xeb, 0x6e, 0x67, 0x59, 0x0a, 0x53, 0x37, 0xc5, 0xef,
+	0x5e, 0xc0, 0xe8, 0xa5, 0x25, 0x49, 0xfb, 0x1f, 0x03, 0xa4, 0x48, 0x63, 0x05, 0x4a, 0x67, 0xe8,
+	0x52, 0xed, 0x43, 0xfe, 0xc9, 0x9d, 0x73, 0xee, 0x4c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x5a, 0xfc,
+	0xb8, 0x60, 0xba, 0xd0, 0xde, 0x99, 0x9c, 0x61, 0x92, 0x99, 0xbe, 0x0a, 0x4b, 0xbe, 0xf3, 0x1d,
+	0xa1, 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84,
+	0xc2, 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x7d, 0x19, 0x20, 0x95,
+	0x62, 0x58, 0xd0, 0xc7, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x72, 0xc9, 0x50, 0x64, 0x53,
+	0xe4, 0xc6, 0x34, 0xc2, 0xe7, 0x7c, 0xfd, 0xb8, 0xd9, 0x37, 0xa4, 0xd9, 0x53, 0xba, 0x59, 0x37,
+	0x31, 0x19, 0xca, 0x79, 0x3b, 0x7c, 0x9a, 0xa5, 0x67, 0x19, 0x87, 0x70, 0x23, 0xe5, 0xe9, 0x65,
+	0xd8, 0x15, 0xaf, 0x62, 0xd7, 0x4d, 0xd8, 0x79, 0x29, 0xab, 0x3d, 0xe8, 0x62, 0x62, 0x7f, 0x1f,
+	0xa3, 0x38, 0xc7, 0xa8, 0x74, 0x15, 0xa3, 0x0e, 0x26, 0x7f, 0x2b, 0x26, 0xa4, 0x6c, 0x06, 0x70,
+	0x2b, 0x63, 0x25, 0xdf, 0xee, 0x19, 0x66, 0xe5, 0xab, 0x98, 0xad, 0x25, 0x5a, 0xf1, 0x7c, 0x90,
+	0x72, 0xfc, 0x12, 0xd6, 0x30, 0xb1, 0x9f, 0x3b, 0x98, 0x4d, 0xb3, 0x5b, 0xfa, 0x05, 0x23, 0xf9,
+	0x89, 0x96, 0xe7, 0x25, 0x8d, 0xf4, 0x11, 0x1d, 0xe5, 0x8c, 0x5c, 0xfe, 0x05, 0x23, 0x8f, 0xc4,
+	0x84, 0x94, 0xcd, 0x36, 0x74, 0x30, 0x99, 0xd6, 0xa6, 0x72, 0x15, 0x93, 0x36, 0x26, 0x79, 0x4d,
+	0x76, 0xa0, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x0d, 0x82, 0xea, 0x55, 0x2c, 0x56, 0x14, 0x7d, 0xc2,
+	0xc3, 0xfc, 0x47, 0x68, 0x1c, 0xc4, 0x23, 0xc4, 0x26, 0x27, 0x49, 0x32, 0x78, 0x69, 0xf9, 0xc7,
+	0xfc, 0x63, 0x11, 0xea, 0xbb, 0x23, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xce, 0xc9,
+	0x82, 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x21, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d,
+	0x99, 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x04, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d,
+	0xb5, 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x3e, 0xd4, 0x4f, 0xb8, 0x93,
+	0xd4, 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x49, 0xba, 0x09, 0x0f, 0xa0, 0x39, 0x96, 0x2e,
+	0x53, 0x93, 0x64, 0x0c, 0xdd, 0x55, 0x96, 0xa4, 0xf6, 0x6e, 0x66, 0x3d, 0x2b, 0x17, 0xa0, 0x31,
+	0xce, 0xa0, 0xfa, 0x43, 0xe8, 0xcc, 0x90, 0xcc, 0xc9, 0x41, 0x1b, 0xd9, 0x1c, 0x54, 0x7f, 0x68,
+	0x48, 0x41, 0xd9, 0x99, 0xd9, 0xbc, 0xf4, 0x9f, 0x45, 0x68, 0x7c, 0x83, 0xd8, 0x73, 0x42, 0xcf,
+	0xa4, 0xbe, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa3, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e, 0xc8,
+	0x04, 0xa2, 0xd6, 0xb3, 0x42, 0x2f, 0x44, 0x62, 0x30, 0x5e, 0x03, 0xa0, 0x17, 0x76, 0xe8, 0xb8,
+	0x67, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xc5, 0x40, 0x22, 0x78, 0x28, 0xd0, 0x0b, 0x1b, 0x51,
+	0x4a, 0x68, 0xa4, 0x72, 0x55, 0x95, 0x5e, 0xec, 0x09, 0x58, 0xcd, 0xf5, 0x28, 0x09, 0x43, 0xe4,
+	0x89, 0x1c, 0x2d, 0xe6, 0x3e, 0x91, 0x08, 0x2e, 0x95, 0x69, 0xa9, 0xcb, 0x52, 0x2a, 0x4b, 0xa5,
+	0xb2, 0x54, 0x6a, 0x45, 0xce, 0x64, 0x59, 0xa9, 0x2c, 0x91, 0x5a, 0x95, 0x52, 0x59, 0x46, 0x2a,
+	0x4b, 0xa5, 0xd6, 0xf4, 0x5c, 0x25, 0xd5, 0xfc, 0x8f, 0x02, 0xac, 0x4d, 0x17, 0x7e, 0xaa, 0x36,
+	0xfd, 0x10, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0x66, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30,
+	0x7e, 0x04, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20,
+	0x03, 0x99, 0x1e, 0x18, 0xdf, 0x52, 0xcc, 0xd0, 0x90, 0x51, 0xe4, 0xf8, 0x2f, 0xa3, 0xba, 0x37,
+	0xa0, 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0x43, 0x37, 0x27, 0x45, 0xd9,
+	0xba, 0x02, 0xa5, 0x09, 0x0a, 0x04, 0xf7, 0xa6, 0xc5, 0x3f, 0x4d, 0x07, 0x3a, 0x16, 0x72, 0xbc,
+	0x97, 0xa7, 0x8d, 0x12, 0x51, 0x4a, 0x45, 0x6c, 0x80, 0x91, 0x15, 0xa1, 0x54, 0xd1, 0x5a, 0x17,
+	0x32, 0x5a, 0x3f, 0x85, 0xce, 0xee, 0x84, 0x44, 0x68, 0xc8, 0x3c, 0x1c, 0xbc, 0x8c, 0x76, 0xe4,
+	0x5f, 0xa0, 0xfb, 0x8c, 0x5d, 0x7e, 0xcb, 0x99, 0x45, 0xf8, 0x07, 0xf4, 0x92, 0xec, 0xa3, 0xe4,
+	0xb9, 0xb6, 0x8f, 0x92, 0xe7, 0xbc, 0xb9, 0x71, 0xc9, 0x24, 0xf6, 0x03, 0xb1, 0x15, 0x9a, 0x96,
+	0x82, 0xcc, 0x1d, 0x68, 0xc8, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0xcd, 0xdd, 0x83, 0xeb, 0x00,
+	0xa1, 0x43, 0x1d, 0x1f, 0x31, 0x44, 0x65, 0x0c, 0xd5, 0xac, 0x0c, 0xc6, 0xfc, 0xaf, 0x22, 0xac,
+	0xca, 0xfb, 0x86, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x7d, 0xa8, 0x8e, 0x49, 0xc4, 0x32, 0x0c, 0x13,
+	0x98, 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0xe6, 0x2e, 0x01, 0x4a, 0x57, 0x5f, 0x02, 0xcc,
+	0xb4, 0xf9, 0xe5, 0xd9, 0x36, 0x9f, 0xef, 0x36, 0x4d, 0x84, 0xe5, 0x1e, 0xaf, 0x59, 0x35, 0x85,
+	0x39, 0xf4, 0x8c, 0x7b, 0xd0, 0x1e, 0x71, 0x2d, 0xed, 0x31, 0x21, 0x67, 0x76, 0xe8, 0xb0, 0xb1,
+	0xd8, 0xea, 0x35, 0xab, 0x29, 0xd0, 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0x3e, 0x81, 0x96,
+	0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, 0x1d, 0x7e, 0x6a, 0x17, 0x65, 0xbd, 0x67, 0x35, 0xcf, 0x32,
+	0x50, 0x64, 0xde, 0x84, 0x1b, 0x4f, 0x50, 0xc4, 0x28, 0xb9, 0xcc, 0x3b, 0xc6, 0xfc, 0x6b, 0x80,
+	0xc3, 0x80, 0x21, 0x7a, 0xea, 0xb8, 0x28, 0x32, 0xde, 0xcb, 0x42, 0xaa, 0x38, 0x5a, 0xd9, 0x94,
+	0xd7, 0x3d, 0xc9, 0x80, 0x95, 0xa1, 0x31, 0x37, 0x61, 0xd9, 0x22, 0x31, 0x4f, 0x47, 0x6f, 0xea,
+	0x2f, 0x35, 0xaf, 0xa1, 0xe6, 0x09, 0xa4, 0xa5, 0xc6, 0xcc, 0x03, 0xdd, 0xc2, 0xa6, 0xec, 0xd4,
+	0x12, 0x6d, 0x42, 0x0d, 0x6b, 0x9c, 0xca, 0x2a, 0xb3, 0xa2, 0x53, 0x12, 0xf3, 0x33, 0xe8, 0x4a,
+	0x4e, 0x92, 0xb3, 0x66, 0xf3, 0x26, 0x2c, 0x53, 0xad, 0x46, 0x21, 0xbd, 0xe7, 0x51, 0x44, 0x6a,
+	0xcc, 0x3c, 0x84, 0xdb, 0x72, 0xf2, 0x5e, 0x38, 0x46, 0x3e, 0xa2, 0xce, 0xe4, 0x88, 0xc4, 0x01,
+	0x4b, 0xb8, 0x64, 0x23, 0xa0, 0x70, 0x65, 0x04, 0x70, 0xd7, 0x7e, 0x8d, 0x23, 0x96, 0xfa, 0x44,
+	0xbb, 0xb6, 0x0b, 0x1d, 0x3e, 0x90, 0x53, 0xcf, 0xfc, 0x02, 0x1a, 0xdb, 0xd6, 0xe0, 0x1b, 0x84,
+	0x47, 0xe3, 0x13, 0x9e, 0x88, 0x3f, 0xca, 0xc3, 0x4a, 0x98, 0xa1, 0x0c, 0xcf, 0x0c, 0x59, 0x39,
+	0x3a, 0xf3, 0x4b, 0x58, 0xdb, 0xf6, 0xbc, 0x2c, 0x4a, 0xab, 0xfe, 0x1e, 0xd4, 0x82, 0x0c, 0xbb,
+	0xcc, 0xf1, 0x97, 0xa3, 0x4e, 0x89, 0xcc, 0x07, 0x60, 0xec, 0x23, 0x76, 0x38, 0x78, 0xe6, 0x9c,
+	0x4c, 0x52, 0x47, 0xde, 0x84, 0x0a, 0x8e, 0x6c, 0x1c, 0x9e, 0x7f, 0x24, 0xb8, 0x54, 0xad, 0x65,
+	0x1c, 0x1d, 0x86, 0xe7, 0x1f, 0x99, 0xf7, 0xa1, 0x9b, 0x23, 0xbf, 0x22, 0x43, 0x6d, 0x83, 0x31,
+	0xfc, 0xf5, 0x9c, 0x13, 0x16, 0xc5, 0x0c, 0x8b, 0xfb, 0xd0, 0x1d, 0xfe, 0x4a, 0x69, 0xff, 0x04,
+	0xdd, 0xa7, 0xc1, 0x04, 0x07, 0x68, 0x77, 0x70, 0x7c, 0x84, 0x92, 0xf4, 0x6c, 0x40, 0x99, 0x97,
+	0xb1, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0x27, 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xff, 0x5a, 0x0e,
+	0x4e, 0x76, 0xc3, 0x38, 0xe2, 0xe7, 0x2d, 0xaf, 0xb7, 0x48, 0x30, 0xb9, 0x14, 0x49, 0xab, 0x6a,
+	0x55, 0xdc, 0x30, 0x7e, 0x1a, 0x4c, 0x2e, 0xcd, 0xbf, 0x12, 0x97, 0x12, 0x08, 0x79, 0x96, 0x13,
+	0x78, 0xc4, 0x7f, 0x82, 0xce, 0x33, 0x12, 0x66, 0xf4, 0xfe, 0xa9, 0x00, 0x8d, 0xed, 0x11, 0x0a,
+	0xd8, 0x13, 0xc4, 0x1c, 0x3c, 0x11, 0x4d, 0xee, 0x39, 0xa2, 0x11, 0x26, 0x81, 0xca, 0x40, 0x1a,
+	0x34, 0x5e, 0x87, 0x3a, 0x0e, 0x30, 0xb3, 0x3d, 0x07, 0xf9, 0x24, 0x10, 0x5c, 0xaa, 0x16, 0x70,
+	0xd4, 0x13, 0x81, 0x31, 0xde, 0x86, 0xb6, 0xbc, 0x9f, 0xb4, 0xc7, 0x4e, 0xe0, 0x4d, 0x78, 0xee,
+	0x2b, 0x89, 0x6c, 0xd5, 0x92, 0xe8, 0x03, 0x85, 0x35, 0xee, 0xc3, 0x8a, 0x8a, 0xcb, 0x94, 0xb2,
+	0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2,
+	0x87, 0xaa, 0x43, 0x6c, 0x6b, 0xfc, 0x50, 0xa2, 0xcd, 0x11, 0x74, 0xf7, 0xb9, 0x9d, 0xca, 0x92,
+	0x74, 0xa7, 0xb5, 0x7c, 0xe4, 0xdb, 0x27, 0x13, 0xe2, 0x9e, 0xd9, 0xfc, 0xbc, 0x50, 0x1e, 0xe6,
+	0x35, 0xe8, 0x0e, 0x47, 0x0e, 0xf1, 0x0f, 0xe2, 0x32, 0x84, 0x53, 0x8d, 0x09, 0x0b, 0x27, 0xf1,
+	0xc8, 0x0e, 0x29, 0x39, 0x41, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9,
+	0xbf, 0x05, 0x58, 0xcd, 0x4b, 0x52, 0xab, 0xbd, 0x05, 0xab, 0x79, 0x51, 0xaa, 0x22, 0x92, 0x15,
+	0x77, 0x27, 0x2b, 0x50, 0xd6, 0x46, 0x8f, 0xa0, 0x29, 0x6e, 0xb3, 0x6d, 0x4f, 0x72, 0xca, 0xd7,
+	0x81, 0xd9, 0x75, 0xb1, 0x1a, 0x4e, 0x76, 0x95, 0x3e, 0x81, 0x5b, 0xca, 0x7c, 0x7b, 0x56, 0x6d,
+	0x19, 0x10, 0x6b, 0x8a, 0xe0, 0x68, 0x4a, 0xfb, 0xaf, 0xa1, 0x97, 0xa2, 0x76, 0x2e, 0x05, 0x32,
+	0xdd, 0x94, 0xdd, 0x29, 0x63, 0xb7, 0x3d, 0x8f, 0x8a, 0xdd, 0x5e, 0xb6, 0xe6, 0x0d, 0x99, 0x8f,
+	0xe1, 0xe6, 0x10, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xe6, 0x4c, 0x32, 0x5b, 0x81, 0xd2, 0x10, 0xb9,
+	0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, 0x47, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d,
+	0x86, 0x50, 0xf9, 0x62, 0xb8, 0xcf, 0x4b, 0x30, 0x1e, 0xd4, 0xb2, 0x64, 0x53, 0xc7, 0x73, 0xd3,
+	0xaa, 0x08, 0xf8, 0xd0, 0x33, 0xbe, 0x84, 0xae, 0x1c, 0x72, 0xc7, 0x4e, 0x30, 0x42, 0x76, 0x48,
+	0x26, 0xd8, 0x95, 0xa1, 0xdf, 0x7a, 0xd8, 0x57, 0x69, 0x48, 0xf1, 0xd9, 0x15, 0x24, 0x03, 0x41,
+	0x61, 0x75, 0x46, 0xd3, 0x28, 0xf3, 0x77, 0x05, 0xa8, 0xa8, 0xfc, 0xc8, 0x4f, 0x79, 0x8f, 0xe2,
+	0x73, 0x44, 0x55, 0xb0, 0x2b, 0xc8, 0x78, 0x0b, 0x5a, 0xf2, 0xcb, 0x26, 0x21, 0xc3, 0x24, 0x39,
+	0x77, 0x9b, 0x12, 0xfb, 0x54, 0x22, 0xc5, 0x0d, 0xa8, 0xb8, 0x83, 0x54, 0xed, 0xbe, 0x82, 0x38,
+	0xfe, 0x34, 0xe2, 0x4a, 0x89, 0x73, 0xb6, 0x66, 0x29, 0x88, 0x6f, 0x2e, 0xcd, 0x6f, 0x49, 0xf0,
+	0xd3, 0x20, 0xdf, 0x5c, 0x3e, 0x4f, 0xed, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0xc1, 0x0a, 0x02, 0x35,
+	0xe0, 0x18, 0x63, 0x03, 0xaa, 0xa7, 0x91, 0x2d, 0xac, 0x11, 0x45, 0x74, 0x92, 0xea, 0x95, 0xd5,
+	0x56, 0xe5, 0x34, 0x12, 0x1f, 0xe6, 0xbf, 0x17, 0x60, 0x59, 0xbe, 0x17, 0x18, 0x2d, 0x28, 0x26,
+	0x85, 0x50, 0x11, 0x8b, 0xa2, 0x52, 0x68, 0x25, 0x8b, 0x1f, 0xf1, 0xcd, 0x73, 0xcc, 0xb9, 0x2f,
+	0x8f, 0x73, 0x65, 0xc4, 0xb9, 0x2f, 0xce, 0xf1, 0xb7, 0xa0, 0x95, 0xd6, 0x53, 0x62, 0x5c, 0x1a,
+	0xd3, 0x4c, 0xb0, 0x82, 0x6c, 0xa1, 0x4d, 0xe6, 0xdf, 0x03, 0xa4, 0xf7, 0xe6, 0x3c, 0x1c, 0xe2,
+	0x44, 0x19, 0xfe, 0xc9, 0x31, 0xa3, 0xa4, 0x12, 0xe3, 0x9f, 0xc6, 0x3d, 0x68, 0x39, 0x9e, 0x87,
+	0xf9, 0x74, 0x67, 0xb2, 0x8f, 0xbd, 0x24, 0x81, 0xe4, 0xb1, 0xe6, 0xff, 0x17, 0xa0, 0xbd, 0x4b,
+	0xc2, 0xcb, 0x2f, 0xf0, 0x04, 0x65, 0xb2, 0x9b, 0x50, 0x52, 0x15, 0x62, 0xfc, 0x9b, 0x37, 0x17,
+	0xa7, 0x78, 0x82, 0xe4, 0xb6, 0x97, 0x51, 0x57, 0xe5, 0x08, 0xb1, 0xe5, 0xf5, 0x60, 0x72, 0x4b,
+	0xda, 0x94, 0x83, 0x47, 0xc4, 0x13, 0x6d, 0x94, 0x87, 0xa9, 0x9d, 0xdc, 0x89, 0x36, 0xad, 0x8a,
+	0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x3b, 0xef, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59,
+	0x83, 0x65, 0x72, 0x7a, 0x1a, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c,
+	0x0a, 0x5e, 0x15, 0xe7, 0xda, 0xd3, 0xa7, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0xfa, 0x04, 0x7e, 0x00,
+	0x55, 0x8d, 0xfa, 0x35, 0xb7, 0xc9, 0xef, 0x40, 0x6b, 0xdb, 0xf3, 0x86, 0xcf, 0x9d, 0x50, 0xfb,
+	0xa3, 0x07, 0x95, 0xc1, 0xee, 0xe1, 0x40, 0xba, 0xa4, 0xc4, 0x0d, 0x50, 0x20, 0x3f, 0xf1, 0xf7,
+	0x11, 0x3b, 0x42, 0x8c, 0x62, 0x37, 0x39, 0xf1, 0xef, 0x42, 0x45, 0x61, 0xf8, 0x4c, 0x5f, 0x7e,
+	0xea, 0x23, 0x40, 0x81, 0xe6, 0xdf, 0x80, 0xf1, 0x77, 0xbc, 0x0c, 0x46, 0xb2, 0x07, 0x52, 0x92,
+	0xde, 0x81, 0xce, 0xb9, 0xc0, 0xda, 0xb2, 0x3e, 0xcc, 0x2c, 0x43, 0x5b, 0x0e, 0x88, 0xfc, 0x20,
+	0x64, 0x1f, 0x43, 0x57, 0x56, 0xed, 0x92, 0xcf, 0x35, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6,
+	0xc4, 0xf7, 0xc3, 0x1f, 0xbb, 0xea, 0x18, 0x53, 0x97, 0x44, 0xc6, 0x3e, 0xb4, 0xa7, 0x5e, 0xf4,
+	0x0c, 0x75, 0x6b, 0x38, 0xff, 0xa1, 0xaf, 0xbf, 0xb6, 0x29, 0x5f, 0x08, 0x37, 0xf5, 0x0b, 0xe1,
+	0xe6, 0x9e, 0x1f, 0xb2, 0x4b, 0x63, 0x0f, 0x5a, 0xf9, 0xb7, 0x2f, 0xe3, 0x55, 0x5d, 0x62, 0xcd,
+	0x79, 0x11, 0x5b, 0xc8, 0x66, 0x1f, 0xda, 0x53, 0xcf, 0x60, 0x5a, 0x9f, 0xf9, 0xaf, 0x63, 0x0b,
+	0x19, 0x3d, 0x86, 0x7a, 0xe6, 0xdd, 0xcb, 0xe8, 0x49, 0x26, 0xb3, 0x4f, 0x61, 0x0b, 0x19, 0xec,
+	0x42, 0x33, 0xf7, 0x14, 0x65, 0xf4, 0x95, 0x3d, 0x73, 0xde, 0xa7, 0x16, 0x32, 0xd9, 0x81, 0x7a,
+	0xe6, 0x45, 0x48, 0x6b, 0x31, 0xfb, 0xec, 0xd4, 0xbf, 0x35, 0x67, 0x44, 0x9d, 0x96, 0xfb, 0xd0,
+	0x9e, 0x7a, 0x26, 0xd2, 0x2e, 0x99, 0xff, 0x7a, 0xb4, 0x50, 0x99, 0x21, 0xdc, 0x98, 0x5b, 0x25,
+	0x1b, 0x66, 0x96, 0xdd, 0xfc, 0x12, 0x7a, 0x21, 0xd3, 0xaf, 0xc4, 0xba, 0x67, 0xae, 0x16, 0x32,
+	0xeb, 0x3e, 0xfb, 0xd2, 0xd4, 0xbf, 0x3d, 0x7f, 0x50, 0x99, 0xba, 0x07, 0xad, 0xfc, 0x23, 0x93,
+	0x66, 0x36, 0xf7, 0xe9, 0xe9, 0xea, 0x20, 0xca, 0xbd, 0x37, 0xa5, 0x41, 0x34, 0xef, 0x19, 0x6a,
+	0x21, 0xa3, 0x6d, 0x00, 0x75, 0x91, 0xe0, 0xe1, 0x20, 0x59, 0xbd, 0x99, 0x0b, 0x8c, 0x64, 0xf5,
+	0xe6, 0x5c, 0x3a, 0x3c, 0x06, 0x90, 0xfd, 0xbf, 0x47, 0x62, 0x66, 0xdc, 0xd4, 0x6a, 0x4c, 0x5d,
+	0x3a, 0xf4, 0x7b, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x39, 0x40, 0x7a, 0xaf,
+	0xa0, 0x19, 0xcc, 0xdc, 0x34, 0x5c, 0xe1, 0x83, 0x46, 0xf6, 0x16, 0xc1, 0x50, 0xb6, 0xce, 0xb9,
+	0x59, 0xb8, 0x82, 0x45, 0x7b, 0xaa, 0x4b, 0xcc, 0x47, 0xf0, 0x74, 0xf3, 0xd8, 0x9f, 0xe9, 0x14,
+	0x8d, 0x47, 0xd0, 0xc8, 0xb6, 0x87, 0x5a, 0x8b, 0x39, 0x2d, 0x63, 0x3f, 0xd7, 0x22, 0x1a, 0x8f,
+	0xa1, 0x95, 0xef, 0xe7, 0x74, 0x48, 0xcd, 0xed, 0xf2, 0xfa, 0xea, 0xe2, 0x33, 0x43, 0xfe, 0x01,
+	0x40, 0xda, 0xf7, 0x69, 0xf7, 0xcd, 0x74, 0x82, 0x53, 0x52, 0xf7, 0xa1, 0x3d, 0xd5, 0xcf, 0x69,
+	0x8b, 0xe7, 0xb7, 0x79, 0x57, 0x25, 0x90, 0x4c, 0x77, 0xa6, 0x43, 0x70, 0xb6, 0xbf, 0xd3, 0x21,
+	0x38, 0xaf, 0x95, 0xdb, 0x81, 0xfa, 0x70, 0x96, 0xc7, 0x70, 0x21, 0x8f, 0x79, 0x0d, 0xda, 0x87,
+	0x00, 0xe9, 0x59, 0xa8, 0xbd, 0x30, 0x73, 0x3a, 0xf6, 0x9b, 0xfa, 0x72, 0x5a, 0xd2, 0xed, 0x42,
+	0x33, 0x77, 0x7f, 0xa3, 0x73, 0xe8, 0xbc, 0x4b, 0x9d, 0xab, 0x4e, 0x96, 0xfc, 0x65, 0x87, 0x5e,
+	0xc1, 0xb9, 0x57, 0x20, 0x57, 0xc5, 0x71, 0xb6, 0x9d, 0xd4, 0x11, 0x34, 0xa7, 0xc5, 0xfc, 0x85,
+	0xbc, 0x92, 0x6d, 0x19, 0x33, 0x79, 0x65, 0x4e, 0x27, 0xb9, 0x90, 0xd1, 0x01, 0xb4, 0xf7, 0x75,
+	0x37, 0xa0, 0x3a, 0x15, 0xbd, 0x7e, 0xb3, 0x9d, 0x59, 0xbf, 0x3f, 0x6f, 0x48, 0xad, 0xcb, 0x57,
+	0xd0, 0x99, 0xe9, 0x52, 0x8c, 0xf5, 0xe4, 0x89, 0x60, 0x6e, 0xfb, 0xb2, 0x50, 0xad, 0x43, 0x58,
+	0x99, 0x6e, 0x52, 0x8c, 0xd7, 0x92, 0x98, 0x98, 0xd7, 0xbc, 0x2c, 0x64, 0xf5, 0x09, 0x54, 0x75,
+	0xe1, 0x69, 0xa8, 0xa7, 0x98, 0xa9, 0x42, 0x74, 0xe1, 0xd4, 0x47, 0x22, 0xe4, 0x93, 0xa2, 0x2e,
+	0x0d, 0xf9, 0xa9, 0xd2, 0xaf, 0xaf, 0x5e, 0x4e, 0x12, 0xca, 0x47, 0x50, 0x51, 0xb5, 0x9d, 0xb1,
+	0x9a, 0x6c, 0xb6, 0x4c, 0xa9, 0x77, 0x55, 0x84, 0xed, 0x23, 0x96, 0xa9, 0xd8, 0xb4, 0xd0, 0xd9,
+	0x22, 0x4e, 0xef, 0x91, 0xdc, 0x88, 0x5a, 0x8b, 0x6d, 0x68, 0x64, 0x6b, 0x36, 0xbd, 0xa4, 0x73,
+	0xea, 0xb8, 0x45, 0x9a, 0xec, 0x5c, 0xfc, 0xf4, 0xf3, 0xfa, 0x2b, 0xbf, 0xfd, 0x79, 0xfd, 0x95,
+	0x7f, 0x7b, 0xb1, 0x5e, 0xf8, 0xe9, 0xc5, 0x7a, 0xe1, 0x37, 0x2f, 0xd6, 0x0b, 0x7f, 0x78, 0xb1,
+	0x5e, 0xf8, 0x87, 0x7f, 0xfe, 0x33, 0xff, 0x13, 0x46, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x9d, 0x63,
+	0xca, 0x32, 0x43, 0xe1, 0xd9, 0x48, 0xfe, 0x31, 0x2c, 0xf3, 0x7f, 0x31, 0xae, 0xe5, 0xc9, 0xb2,
+	0x80, 0x3f, 0xf8, 0x53, 0x00, 0x00, 0x00, 0xff, 0xff, 0xeb, 0xb0, 0x39, 0x1d, 0x7c, 0x26, 0x00,
+	0x00,
 }
 
 func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) {
@@ -4858,6 +4901,47 @@ func (m *UpdateRoutesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *UpdateEphemeralMountsRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *UpdateEphemeralMountsRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *UpdateEphemeralMountsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Storages) > 0 {
+		for iNdEx := len(m.Storages) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Storages[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintAgent(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
 func (m *ListInterfacesRequest) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -6901,6 +6985,24 @@ func (m *UpdateRoutesRequest) Size() (n int) {
 	return n
 }
 
+func (m *UpdateEphemeralMountsRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Storages) > 0 {
+		for _, e := range m.Storages {
+			l = e.Size()
+			n += 1 + l + sovAgent(uint64(l))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
 func (m *ListInterfacesRequest) Size() (n int) {
 	if m == nil {
 		return 0
@@ -7664,7 +7766,7 @@ func (this *MemoryStats) String() string {
 		return "nil"
 	}
 	keysForStats := make([]string, 0, len(this.Stats))
-	for k := range this.Stats {
+	for k, _ := range this.Stats {
 		keysForStats = append(keysForStats, k)
 	}
 	github_com_gogo_protobuf_sortkeys.Strings(keysForStats)
@@ -7775,7 +7877,7 @@ func (this *CgroupStats) String() string {
 		return "nil"
 	}
 	keysForHugetlbStats := make([]string, 0, len(this.HugetlbStats))
-	for k := range this.HugetlbStats {
+	for k, _ := range this.HugetlbStats {
 		keysForHugetlbStats = append(keysForHugetlbStats, k)
 	}
 	github_com_gogo_protobuf_sortkeys.Strings(keysForHugetlbStats)
@@ -8008,6 +8110,22 @@ func (this *UpdateRoutesRequest) String() string {
 	}, "")
 	return s
 }
+func (this *UpdateEphemeralMountsRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	repeatedStringForStorages := "[]*Storage{"
+	for _, f := range this.Storages {
+		repeatedStringForStorages += strings.Replace(f.String(), "Storage", "Storage", 1) + ","
+	}
+	repeatedStringForStorages += "}"
+	s := strings.Join([]string{`&UpdateEphemeralMountsRequest{`,
+		`Storages:` + repeatedStringForStorages + `,`,
+		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func (this *ListInterfacesRequest) String() string {
 	if this == nil {
 		return "nil"
@@ -8355,6 +8473,7 @@ type AgentServiceService interface {
 	SignalProcess(ctx context.Context, req *SignalProcessRequest) (*types.Empty, error)
 	WaitProcess(ctx context.Context, req *WaitProcessRequest) (*WaitProcessResponse, error)
 	UpdateContainer(ctx context.Context, req *UpdateContainerRequest) (*types.Empty, error)
+	UpdateEphemeralMounts(ctx context.Context, req *UpdateEphemeralMountsRequest) (*types.Empty, error)
 	StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error)
 	PauseContainer(ctx context.Context, req *PauseContainerRequest) (*types.Empty, error)
 	ResumeContainer(ctx context.Context, req *ResumeContainerRequest) (*types.Empty, error)
@@ -8436,6 +8555,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag
 			}
 			return svc.UpdateContainer(ctx, &req)
 		},
+		"UpdateEphemeralMounts": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req UpdateEphemeralMountsRequest
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.UpdateEphemeralMounts(ctx, &req)
+		},
 		"StatsContainer": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
 			var req StatsContainerRequest
 			if err := unmarshal(&req); err != nil {
@@ -8701,6 +8827,14 @@ func (c *agentServiceClient) UpdateContainer(ctx context.Context, req *UpdateCon
 	return &resp, nil
 }
 
+func (c *agentServiceClient) UpdateEphemeralMounts(ctx context.Context, req *UpdateEphemeralMountsRequest) (*types.Empty, error) {
+	var resp types.Empty
+	if err := c.client.Call(ctx, "grpc.AgentService", "UpdateEphemeralMounts", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
 func (c *agentServiceClient) StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error) {
 	var resp StatsContainerResponse
 	if err := c.client.Call(ctx, "grpc.AgentService", "StatsContainer", req, &resp); err != nil {
@@ -13934,6 +14068,91 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *UpdateEphemeralMountsRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowAgent
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: UpdateEphemeralMountsRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: UpdateEphemeralMountsRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Storages", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowAgent
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthAgent
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Storages = append(m.Storages, &Storage{})
+			if err := m.Storages[len(m.Storages)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipAgent(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *ListInterfacesRequest) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0

src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go

@@ -423,7 +423,13 @@ type Hooks struct {
 	// Poststart is a list of hooks to be run after the container process is started.
 	Poststart []Hook `protobuf:"bytes,2,rep,name=Poststart,proto3" json:"Poststart"`
 	// Poststop is a list of hooks to be run after the container process exits.
-	Poststop             []Hook   `protobuf:"bytes,3,rep,name=Poststop,proto3" json:"Poststop"`
+	Poststop []Hook `protobuf:"bytes,3,rep,name=Poststop,proto3" json:"Poststop"`
+	// Createruntime is a list of hooks to be run during the creation of runtime(sandbox).
+	CreateRuntime []Hook `protobuf:"bytes,4,rep,name=CreateRuntime,proto3" json:"CreateRuntime"`
+	// CreateContainer is a list of hooks to be run after VM is started, and before container is created.
+	CreateContainer []Hook `protobuf:"bytes,5,rep,name=CreateContainer,proto3" json:"CreateContainer"`
+	// StartContainer is a list of hooks to be run after container is created, but before it is started.
+	StartContainer       []Hook   `protobuf:"bytes,6,rep,name=StartContainer,proto3" json:"StartContainer"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -1390,6 +1396,7 @@ type LinuxSyscall struct {
 	Names  []string `protobuf:"bytes,1,rep,name=Names,proto3" json:"Names,omitempty"`
 	Action string   `protobuf:"bytes,2,opt,name=Action,proto3" json:"Action,omitempty"`
 	// Types that are valid to be assigned to ErrnoRet:
+	//
 	//	*LinuxSyscall_Errnoret
 	ErrnoRet             isLinuxSyscall_ErrnoRet `protobuf_oneof:"ErrnoRet"`
 	Args                 []LinuxSeccompArg       `protobuf:"bytes,4,rep,name=Args,proto3" json:"Args"`
@@ -1546,140 +1553,143 @@ func init() {
 }
 
 var fileDescriptor_82a9ef0098ca0b24 = []byte{
-	// 2121 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0xcb, 0x73, 0x5b, 0x49,
-	0xd5, 0xcf, 0x95, 0x64, 0x59, 0x6a, 0xc5, 0x79, 0xf4, 0x64, 0x32, 0xf7, 0xcb, 0x97, 0xd2, 0x78,
-	0x2e, 0x29, 0x30, 0x10, 0xac, 0x22, 0xe1, 0x31, 0x0c, 0x8f, 0x2a, 0xd9, 0x4e, 0x62, 0xd5, 0xc4,
-	0xb1, 0x68, 0xd9, 0x13, 0x98, 0xc5, 0x54, 0xb5, 0xaf, 0xda, 0x52, 0x8f, 0xaf, 0x6e, 0xdf, 0xea,
-	0xdb, 0xb2, 0x63, 0x56, 0xb0, 0x63, 0xcf, 0x82, 0x35, 0x1b, 0x1e, 0xff, 0x01, 0xc5, 0x8a, 0x1d,
-	0x29, 0x56, 0x2c, 0xa9, 0xa2, 0x0a, 0x88, 0xf7, 0xec, 0x59, 0x52, 0xa7, 0xfb, 0xdc, 0xab, 0x96,
-	0xe4, 0xc0, 0x04, 0x56, 0xea, 0xf3, 0x3b, 0x8f, 0x7e, 0x9c, 0xe7, 0x15, 0x79, 0x36, 0x92, 0x66,
-	0x3c, 0x3d, 0xda, 0x8c, 0xd5, 0xa4, 0x73, 0xc2, 0x0d, 0xff, 0x4a, 0xac, 0x52, 0xc3, 0x65, 0x2a,
-	0x74, 0xbe, 0x44, 0xe7, 0x3a, 0xee, 0x24, 0xf2, 0x28, 0xef, 0x64, 0x5a, 0x19, 0x15, 0xab, 0x04,
-	0x57, 0x79, 0x47, 0xc5, 0x72, 0xd3, 0x2e, 0x69, 0x6d, 0xa4, 0xb3, 0xf8, 0x4e, 0x34, 0x52, 0x23,
-	0xe5, 0x98, 0x47, 0xd3, 0xe3, 0x0e, 0x50, 0x96, 0xb0, 0x2b, 0x27, 0x19, 0xfd, 0xa1, 0x4a, 0x6a,
-	0x83, 0x4c, 0xc4, 0x34, 0x24, 0xab, 0x1f, 0x09, 0x9d, 0x4b, 0x95, 0x86, 0xc1, 0x7a, 0xb0, 0xd1,
-	0x64, 0x05, 0x49, 0xbf, 0x40, 0x56, 0xfb, 0x5a, 0xc5, 0x22, 0xcf, 0xc3, 0xca, 0x7a, 0xb0, 0xd1,
-	0x7a, 0xb0, 0xb6, 0x09, 0xe6, 0x37, 0x11, 0x64, 0x05, 0x97, 0xb6, 0x49, 0x8d, 0x29, 0x65, 0xc2,
-	0xaa, 0x95, 0x22, 0x4e, 0x0a, 0x10, 0x66, 0x71, 0x7a, 0x87, 0x34, 0x76, 0x55, 0x6e, 0x52, 0x3e,
-	0x11, 0x61, 0xcd, 0xee, 0x51, 0xd2, 0xf4, 0x8b, 0xa4, 0xbe, 0xa7, 0xa6, 0xa9, 0xc9, 0xc3, 0x95,
-	0xf5, 0xea, 0x46, 0xeb, 0x41, 0xcb, 0x69, 0x5b, 0x6c, 0xab, 0xf6, 0xf2, 0xaf, 0xef, 0x5e, 0x61,
-	0x28, 0x40, 0xdf, 0x23, 0x2b, 0xbb, 0x4a, 0x9d, 0xe4, 0x61, 0xdd, 0xee, 0x83, 0x92, 0x16, 0x62,
-	0x8e, 0x43, 0xbf, 0x4b, 0x5a, 0xdd, 0x34, 0x55, 0x86, 0x1b, 0xa9, 0xd2, 0x3c, 0x5c, 0xb5, 0x26,
-	0xff, 0xdf, 0x09, 0xc2, 0x6d, 0x37, 0x3d, 0xee, 0xa3, 0xd4, 0xe8, 0x73, 0xe6, 0xcb, 0xc3, 0x0e,
-	0x4f, 0x65, 0x3a, 0x7d, 0x11, 0x36, 0xfc, 0x1d, 0x2c, 0xc4, 0x1c, 0x07, 0x1e, 0x65, 0xa0, 0x12,
-	0xae, 0x65, 0x1e, 0x36, 0xfd, 0x47, 0x41, 0x90, 0x15, 0x5c, 0x10, 0x7c, 0x2e, 0xd3, 0xa1, 0x3a,
-	0xcb, 0x43, 0xe2, 0x0b, 0x22, 0xc8, 0x0a, 0xee, 0x9d, 0xef, 0x91, 0x1b, 0x8b, 0xa7, 0xa2, 0x37,
-	0x48, 0xf5, 0x44, 0x9c, 0xa3, 0x43, 0x60, 0x49, 0x6f, 0x91, 0x95, 0x53, 0x9e, 0x4c, 0x85, 0x75,
-	0x45, 0x93, 0x39, 0xe2, 0x83, 0xca, 0xfb, 0x41, 0xf4, 0xbb, 0x6a, 0xe9, 0x27, 0x78, 0xe9, 0x03,
-	0xa1, 0x27, 0x32, 0xe5, 0x89, 0x55, 0x6e, 0xb0, 0x92, 0xa6, 0x5f, 0x26, 0xad, 0x6d, 0x95, 0xe6,
-	0x2a, 0x11, 0x03, 0xf9, 0x23, 0x81, 0x2e, 0x6d, 0xba, 0x43, 0x6d, 0xa9, 0x17, 0xcc, 0xe7, 0xd2,
-	0x7b, 0xa4, 0x76, 0x98, 0x0b, 0x3d, 0xef, 0x52, 0x40, 0xd0, 0x27, 0x96, 0x4b, 0x29, 0xa9, 0x75,
-	0xf5, 0x28, 0x0f, 0x6b, 0xeb, 0xd5, 0x8d, 0x26, 0xb3, 0x6b, 0x38, 0xfa, 0xa3, 0xf4, 0xd4, 0x7a,
-	0xb3, 0xc9, 0x60, 0x09, 0xc8, 0xf6, 0xd9, 0xd0, 0x7a, 0xad, 0xc9, 0x60, 0x49, 0xbf, 0x4d, 0xae,
-	0x6e, 0xf3, 0x8c, 0x1f, 0xc9, 0x44, 0x1a, 0x29, 0xc0, 0x4f, 0xb0, 0xcb, 0x3b, 0xde, 0x73, 0xfb,
-	0x6c, 0x36, 0x27, 0x4c, 0xbf, 0x4a, 0x56, 0x59, 0x22, 0x27, 0xd2, 0xe4, 0x61, 0xc3, 0xfa, 0xf7,
-	0x26, 0x86, 0xe5, 0xfe, 0xa0, 0xf7, 0x03, 0xc7, 0xc1, 0x43, 0x16, 0x72, 0x74, 0x83, 0x5c, 0x7f,
-	0xa6, 0x9e, 0x89, 0xb3, 0xbe, 0x96, 0xa7, 0x32, 0x11, 0x23, 0xe1, 0x9c, 0xd7, 0x60, 0x8b, 0x30,
-	0x48, 0x76, 0xb3, 0x8c, 0xeb, 0x89, 0xd2, 0x7d, 0xad, 0x8e, 0x65, 0x22, 0xac, 0xf7, 0x9a, 0x6c,
-	0x11, 0xa6, 0xeb, 0xa4, 0xb5, 0xbf, 0xbf, 0x37, 0x88, 0x95, 0x16, 0xdd, 0xe1, 0xa7, 0x61, 0x6b,
-	0x3d, 0xd8, 0xa8, 0x32, 0x1f, 0xa2, 0x11, 0xb9, 0x3a, 0x10, 0x09, 0xdc, 0xe6, 0x29, 0x3f, 0x12,
-	0x49, 0x78, 0xd5, 0x1a, 0x9a, 0xc3, 0xa2, 0x87, 0xa4, 0xba, 0xa5, 0x5e, 0xd0, 0xdb, 0xa4, 0xbe,
-	0x2b, 0xe4, 0x68, 0x6c, 0xac, 0xd7, 0xd6, 0x18, 0x52, 0xe0, 0xf5, 0xe7, 0x72, 0x68, 0xc6, 0xd6,
-	0x5b, 0x6b, 0xcc, 0x11, 0x51, 0xea, 0x9c, 0x03, 0x0f, 0x7b, 0xd8, 0xdb, 0x41, 0x15, 0x58, 0x02,
-	0xf2, 0xa4, 0xb7, 0x83, 0xd2, 0xb0, 0xa4, 0x9f, 0x27, 0xd7, 0xba, 0xc3, 0xa1, 0x84, 0xd8, 0xe2,
-	0xc9, 0x13, 0x39, 0xcc, 0xc3, 0xea, 0x7a, 0x75, 0x63, 0x8d, 0x2d, 0xa0, 0x10, 0x39, 0x60, 0xd3,
-	0xcf, 0xd1, 0x82, 0x8e, 0x7e, 0x15, 0x90, 0x9b, 0x4b, 0x5e, 0x01, 0x8d, 0x2d, 0x35, 0x4d, 0x87,
-	0x32, 0x1d, 0x85, 0x81, 0xf5, 0x76, 0x49, 0xd3, 0xbb, 0xa4, 0xf9, 0xe8, 0xf8, 0x58, 0xc4, 0x46,
-	0x9e, 0x42, 0xa4, 0x01, 0x73, 0x06, 0xc0, 0xd3, 0xf5, 0xd2, 0xb1, 0xd0, 0xd2, 0xf0, 0xa3, 0x44,
-	0xd8, 0x03, 0x35, 0x99, 0x0f, 0x81, 0x7e, 0x1f, 0xe2, 0xd6, 0x18, 0x31, 0xc4, 0xe8, 0x9a, 0x01,
-	0x50, 0xb2, 0xba, 0x93, 0x23, 0x29, 0x52, 0x83, 0x61, 0x56, 0x90, 0x51, 0x8f, 0xb4, 0xbc, 0x30,
-	0x80, 0xf8, 0x3c, 0x38, 0xcf, 0x04, 0xe6, 0x91, 0x5d, 0x03, 0xb6, 0xcb, 0xf5, 0xd0, 0xbe, 0x51,
-	0x8d, 0xd9, 0x35, 0x60, 0x03, 0x75, 0xec, 0x0a, 0x58, 0x8d, 0xd9, 0x75, 0xa4, 0xc8, 0x8a, 0xad,
-	0x3b, 0x70, 0xda, 0xa1, 0xc8, 0x8d, 0x4c, 0x6d, 0x82, 0xa2, 0x2d, 0x1f, 0x02, 0xef, 0xe5, 0x6a,
-	0xaa, 0xe3, 0x22, 0x39, 0x91, 0x02, 0xb3, 0x06, 0xb6, 0xaf, 0xba, 0xed, 0x61, 0x0d, 0x67, 0x57,
-	0x99, 0xab, 0x4e, 0xee, 0x5e, 0x05, 0x19, 0x7d, 0xc3, 0x55, 0x51, 0xd0, 0xea, 0x73, 0x33, 0x2e,
-	0x0e, 0x0d, 0x6b, 0x78, 0x6b, 0x26, 0xf8, 0x50, 0xa5, 0xc9, 0xb9, 0xdd, 0xa3, 0xc1, 0x4a, 0x3a,
-	0xfa, 0x59, 0x80, 0x75, 0x91, 0xde, 0x27, 0x8d, 0xbe, 0x16, 0xb9, 0xe1, 0xda, 0x58, 0x8f, 0x94,
-	0x89, 0x0b, 0x6c, 0xcc, 0x89, 0x52, 0x82, 0x6e, 0x92, 0x66, 0x5f, 0xe5, 0xc6, 0x89, 0x57, 0x5e,
-	0x23, 0x3e, 0x13, 0xb1, 0xd6, 0x2d, 0xa1, 0x32, 0xeb, 0xb2, 0xcb, 0xad, 0xa3, 0x44, 0xf4, 0x31,
-	0xa9, 0x01, 0x7e, 0xe9, 0x6d, 0x8a, 0xb2, 0x51, 0x59, 0x2e, 0x1b, 0xd5, 0x59, 0xd9, 0x08, 0xc9,
-	0xea, 0x81, 0x9c, 0x08, 0x35, 0x35, 0x36, 0x20, 0xab, 0xac, 0x20, 0xa3, 0xdf, 0xac, 0x60, 0x9d,
-	0xa6, 0xdf, 0x21, 0xad, 0xc3, 0xde, 0xce, 0x1e, 0xcf, 0x32, 0x99, 0x8e, 0x72, 0xbc, 0xf4, 0x2d,
-	0xaf, 0x8e, 0x94, 0x4c, 0x3c, 0xa0, 0x2f, 0x0e, 0xda, 0x4f, 0x3c, 0xed, 0xca, 0x7f, 0xd6, 0xf6,
-	0xc4, 0x69, 0x87, 0xd4, 0x07, 0xe7, 0x79, 0x6c, 0x12, 0x7c, 0x0d, 0xbf, 0x7c, 0x6d, 0x3a, 0x8e,
-	0x6b, 0x31, 0x28, 0x46, 0x1f, 0x90, 0x26, 0x13, 0x2e, 0x34, 0x72, 0x7b, 0xa5, 0xf9, 0xcd, 0x4a,
-	0x1e, 0x9b, 0x89, 0x41, 0xf0, 0x6d, 0x8f, 0xb4, 0x9a, 0x66, 0xb9, 0x7d, 0xc5, 0x15, 0x17, 0x7c,
-	0x1e, 0x44, 0x3f, 0x20, 0xe4, 0x19, 0x9f, 0x88, 0x3c, 0xe3, 0x60, 0xb6, 0xbe, 0x74, 0x87, 0x92,
-	0x89, 0x77, 0xf0, 0xa4, 0xa1, 0x94, 0xee, 0x88, 0x53, 0x19, 0x8b, 0xa2, 0x55, 0xde, 0xf4, 0x14,
-	0x1d, 0xa7, 0x28, 0xa5, 0x28, 0x47, 0xef, 0x93, 0xd5, 0x81, 0x88, 0x63, 0x35, 0xc9, 0xb0, 0x49,
-	0x52, 0x4f, 0x05, 0x39, 0xac, 0x10, 0xa1, 0xf7, 0xc9, 0x4d, 0x88, 0xe9, 0xe3, 0xbc, 0xaf, 0x55,
-	0xc6, 0x47, 0x2e, 0x83, 0x9a, 0xf6, 0x12, 0xcb, 0x0c, 0xb8, 0xec, 0x1e, 0xcf, 0x4f, 0xc4, 0x10,
-	0x2e, 0x06, 0x6d, 0xd3, 0xd6, 0x05, 0x0f, 0xa2, 0xf7, 0xc8, 0x5a, 0x11, 0xf7, 0x4e, 0xa6, 0x65,
-	0x65, 0xe6, 0x41, 0xda, 0x26, 0xc4, 0xa6, 0xae, 0x5f, 0x76, 0x3d, 0x84, 0x76, 0x48, 0xa3, 0x97,
-	0x1a, 0x91, 0xb0, 0xa1, 0x09, 0xd7, 0xec, 0x25, 0xde, 0xf2, 0x9d, 0x8e, 0x2c, 0x56, 0x0a, 0xdd,
-	0xf9, 0x16, 0x69, 0x79, 0x0e, 0x7d, 0xa3, 0xee, 0xfc, 0x6e, 0x39, 0x06, 0x80, 0xd0, 0x70, 0x3a,
-	0x99, 0x14, 0x8a, 0x8e, 0x00, 0x81, 0x62, 0x64, 0xb8, 0x5c, 0xe0, 0x13, 0x72, 0x6d, 0x3e, 0x18,
-	0x6d, 0xb7, 0x50, 0xb9, 0x29, 0x4b, 0x3f, 0x52, 0x36, 0x58, 0x8a, 0x79, 0xb1, 0xec, 0x02, 0x3e,
-	0x64, 0x0b, 0x1d, 0x34, 0xff, 0xaa, 0x65, 0xd9, 0x75, 0xf4, 0x3e, 0xda, 0x2f, 0xe3, 0xe2, 0x75,
-	0x65, 0xd3, 0x46, 0x60, 0x65, 0x96, 0xc7, 0xd1, 0x2f, 0x02, 0xd2, 0xf2, 0x42, 0xe5, 0x75, 0xb9,
-	0x6e, 0x6d, 0x55, 0x3c, 0x5b, 0xb7, 0xc8, 0xca, 0x1e, 0xff, 0x54, 0xb9, 0xe9, 0xa2, 0xca, 0x1c,
-	0x61, 0x51, 0x99, 0x2a, 0x8d, 0xd9, 0xee, 0x08, 0xa8, 0x7c, 0x8f, 0x65, 0x22, 0xf6, 0xd4, 0x50,
-	0xd8, 0xe8, 0x5f, 0x63, 0x25, 0x5d, 0xf4, 0xbf, 0xfa, 0x52, 0xff, 0x5b, 0x2d, 0xfb, 0x5f, 0xf4,
-	0xb7, 0x0a, 0x5e, 0x6f, 0x96, 0x53, 0xdf, 0x9c, 0x45, 0x7d, 0xb0, 0x94, 0xb9, 0x8e, 0xe3, 0x12,
-	0x6c, 0x31, 0xf6, 0x61, 0x56, 0x15, 0x13, 0xa5, 0xcf, 0x71, 0x78, 0xf2, 0xb3, 0xc5, 0x31, 0x18,
-	0x0a, 0xd0, 0x75, 0x52, 0xdd, 0xee, 0x1f, 0xe2, 0xf8, 0x74, 0xcd, 0x1f, 0x6c, 0xfa, 0x87, 0x0c,
-	0x58, 0xf4, 0x73, 0xa4, 0xd6, 0x87, 0x76, 0xec, 0x0a, 0xc1, 0x75, 0x4f, 0x04, 0x60, 0x66, 0x99,
-	0x90, 0x6d, 0x5b, 0x89, 0x8a, 0x4f, 0x7a, 0xfb, 0xf6, 0xf2, 0xf3, 0xd9, 0x86, 0x1c, 0x56, 0x88,
-	0xd0, 0xc7, 0xe4, 0xda, 0xee, 0x74, 0x24, 0x32, 0x3e, 0x12, 0x4f, 0xdd, 0x80, 0xe4, 0xca, 0x41,
-	0xe8, 0x29, 0xcd, 0x09, 0xe0, 0x05, 0x17, 0xb4, 0x60, 0xd7, 0x67, 0xc2, 0x9c, 0x29, 0x7d, 0x82,
-	0x93, 0x99, 0xbf, 0x2b, 0x72, 0x58, 0x21, 0x12, 0xfd, 0xa5, 0x88, 0x02, 0xbc, 0xfa, 0x2d, 0x28,
-	0xce, 0x13, 0xe9, 0x46, 0x99, 0x2a, 0x73, 0x04, 0xc4, 0x26, 0x13, 0xb9, 0xd0, 0xa7, 0xae, 0x06,
-	0x54, 0xdc, 0xb8, 0xe4, 0x41, 0x36, 0x36, 0xcf, 0x78, 0x86, 0x41, 0x61, 0xd7, 0x10, 0xe9, 0x1f,
-	0x0a, 0x9d, 0x8a, 0x04, 0x83, 0x02, 0x29, 0x98, 0x0f, 0xdc, 0xea, 0x60, 0xbb, 0x6f, 0x5f, 0xa6,
-	0xca, 0x66, 0x00, 0xe4, 0x3f, 0x68, 0x67, 0x32, 0x85, 0x6f, 0x97, 0xba, 0x6d, 0xea, 0x1e, 0x42,
-	0xbf, 0x44, 0x6e, 0xec, 0xc8, 0x1c, 0x06, 0x8d, 0xfd, 0xfd, 0xbd, 0x0f, 0x65, 0x92, 0x08, 0x6d,
-	0x2f, 0xda, 0x60, 0x4b, 0x78, 0xf4, 0xc7, 0x80, 0x34, 0x0a, 0xc7, 0xc1, 0x71, 0x06, 0x63, 0xae,
-	0x6d, 0xe0, 0x80, 0x51, 0xa4, 0xe0, 0xca, 0xdf, 0x9f, 0x2a, 0xc3, 0xf1, 0x5a, 0x8e, 0x00, 0xe9,
-	0xbe, 0xd0, 0x52, 0x0d, 0x71, 0xae, 0x40, 0x0a, 0x66, 0x4c, 0x26, 0x78, 0x62, 0xe4, 0x44, 0xb0,
-	0x69, 0x0a, 0x3f, 0x78, 0xbb, 0x45, 0x18, 0x86, 0xb7, 0x02, 0x42, 0x4b, 0x2b, 0xd6, 0xd2, 0x02,
-	0x0a, 0x4f, 0xb7, 0x9d, 0x4d, 0x73, 0x1c, 0xb1, 0xed, 0x1a, 0xb0, 0x3d, 0x31, 0x71, 0xb3, 0x75,
-	0x93, 0xd9, 0x75, 0x74, 0x86, 0x73, 0xdc, 0x73, 0x3b, 0x5d, 0x62, 0xd6, 0x96, 0xd9, 0x18, 0x5c,
-	0x9a, 0x8d, 0x15, 0x3f, 0x1b, 0x6f, 0x93, 0xba, 0xd3, 0xc5, 0x0a, 0x82, 0x14, 0xbc, 0xf8, 0x53,
-	0xc1, 0x8f, 0x91, 0x57, 0xb3, 0x3c, 0x0f, 0x89, 0x0e, 0xc9, 0x5b, 0x76, 0xe3, 0x83, 0xb1, 0x56,
-	0xc6, 0x24, 0xe2, 0xbf, 0xd8, 0x9a, 0x92, 0x1a, 0xe3, 0x46, 0x14, 0x33, 0x1a, 0xac, 0xa3, 0x7f,
-	0x54, 0xc9, 0x55, 0x3f, 0x15, 0xbc, 0xf3, 0x05, 0xff, 0xe6, 0x7c, 0x95, 0xc5, 0xf3, 0xd1, 0x2e,
-	0xb9, 0xea, 0xbf, 0xc9, 0x25, 0x1d, 0xdd, 0x67, 0x63, 0xda, 0xcc, 0xa9, 0xd0, 0x43, 0xf2, 0x76,
-	0x71, 0x3b, 0xe8, 0x46, 0x5b, 0x59, 0x8e, 0xb6, 0x6a, 0xd6, 0xd6, 0xff, 0x79, 0xb6, 0xe6, 0x5f,
-	0x01, 0xad, 0x5d, 0xae, 0x4d, 0x9f, 0x93, 0xdb, 0x05, 0xe3, 0xb9, 0x96, 0x46, 0xcc, 0xec, 0xae,
-	0x7c, 0x36, 0xbb, 0xaf, 0x51, 0xf7, 0x0d, 0xc3, 0x8e, 0xbd, 0xfd, 0xfe, 0x00, 0x0d, 0xd7, 0xdf,
-	0xd0, 0xf0, 0xbc, 0x3a, 0xfd, 0x21, 0x79, 0x67, 0x6e, 0x4b, 0xcf, 0xf2, 0xea, 0x67, 0xb3, 0xfc,
-	0x3a, 0xfd, 0xe8, 0x3d, 0xd2, 0x2c, 0x2b, 0xe4, 0xe5, 0x75, 0x26, 0xfa, 0x49, 0xf1, 0xad, 0xe2,
-	0x17, 0x72, 0x90, 0xed, 0x26, 0x89, 0x3a, 0xc3, 0x8f, 0x62, 0x47, 0xfc, 0xcf, 0xbd, 0xe9, 0x36,
-	0xa9, 0x77, 0x63, 0xfb, 0xff, 0x88, 0x9b, 0xcb, 0x90, 0x8a, 0x12, 0x8c, 0x4a, 0xac, 0x90, 0x30,
-	0xc9, 0x6e, 0x27, 0x3c, 0xcf, 0xcb, 0x86, 0x5d, 0x90, 0x74, 0x8b, 0x90, 0xbe, 0x96, 0x4a, 0xbb,
-	0xcf, 0x60, 0x37, 0x80, 0xde, 0x5d, 0x98, 0x45, 0xf4, 0x31, 0x8f, 0x05, 0x4a, 0x9d, 0x17, 0x43,
-	0xdc, 0x4c, 0x2b, 0x7a, 0x4c, 0xe8, 0x72, 0x65, 0x87, 0xbe, 0xd9, 0xe7, 0x23, 0x91, 0x43, 0xb7,
-	0x77, 0xfd, 0xb8, 0xa4, 0x67, 0x2f, 0xe7, 0xbe, 0x81, 0xf0, 0xe5, 0x76, 0xc9, 0xed, 0xcb, 0xf7,
-	0x84, 0x77, 0x82, 0xe1, 0xa0, 0xe8, 0xeb, 0xb0, 0xb6, 0xf6, 0x91, 0x8f, 0xf9, 0x54, 0xd2, 0xd1,
-	0x2f, 0x03, 0x7c, 0x80, 0x62, 0x0c, 0xbc, 0x47, 0xd6, 0x76, 0xc4, 0x31, 0x9f, 0x26, 0xa6, 0x1b,
-	0x7b, 0x1f, 0x51, 0xf3, 0x20, 0x48, 0x75, 0x75, 0x3c, 0x96, 0x46, 0xc4, 0x66, 0xaa, 0x45, 0xf1,
-	0x7d, 0x30, 0x0f, 0xc2, 0xe1, 0x1f, 0x27, 0x7c, 0x94, 0xe3, 0xa7, 0x82, 0x23, 0xe8, 0xd7, 0x48,
-	0x03, 0x26, 0x34, 0x9e, 0x24, 0x39, 0x26, 0xdc, 0xdc, 0x5c, 0xea, 0x58, 0xc5, 0x47, 0x4a, 0x21,
-	0x19, 0x49, 0x72, 0xdd, 0x3f, 0x67, 0x57, 0x8f, 0xc0, 0x7c, 0x2f, 0x1d, 0x8a, 0x17, 0x58, 0xe1,
-	0x1d, 0x01, 0xe8, 0x47, 0xe5, 0x7c, 0x57, 0x63, 0x8e, 0x80, 0x37, 0xb0, 0x8b, 0x83, 0x33, 0x85,
-	0x65, 0xa9, 0xa4, 0xe9, 0x35, 0x52, 0xd9, 0xcf, 0xf0, 0x4b, 0xba, 0xb2, 0x9f, 0x45, 0x3f, 0x2f,
-	0xdf, 0xc4, 0x6d, 0x0e, 0x26, 0xed, 0xc4, 0x85, 0xdf, 0xce, 0x8e, 0x70, 0x21, 0x55, 0x76, 0x48,
-	0x1b, 0x52, 0xf6, 0x6d, 0xee, 0x92, 0x86, 0xd0, 0x3a, 0x55, 0x5a, 0x60, 0xe9, 0xdd, 0xbd, 0xc2,
-	0x4a, 0x84, 0x76, 0xbc, 0xff, 0x61, 0x5a, 0x0f, 0xde, 0x5e, 0x9e, 0xc8, 0xbb, 0xba, 0xf8, 0x84,
-	0xb1, 0x82, 0x5b, 0x84, 0x34, 0x1e, 0x81, 0x32, 0x13, 0x26, 0xfa, 0x3a, 0x59, 0x9b, 0x9b, 0x7b,
-	0xc1, 0x0f, 0x4f, 0x1f, 0x6e, 0xf3, 0x78, 0x2c, 0x06, 0xf1, 0x58, 0x4c, 0x78, 0xe1, 0xad, 0x39,
-	0x70, 0xeb, 0xa7, 0xc1, 0xcb, 0x57, 0xed, 0x2b, 0x7f, 0x7e, 0xd5, 0xbe, 0xf2, 0xcf, 0x57, 0xed,
-	0xe0, 0xc7, 0x17, 0xed, 0xe0, 0xd7, 0x17, 0xed, 0xe0, 0xb7, 0x17, 0xed, 0xe0, 0xf7, 0x17, 0xed,
-	0xe0, 0xe5, 0x45, 0x3b, 0xf8, 0xd3, 0x45, 0x3b, 0xf8, 0xfb, 0x45, 0x3b, 0xf8, 0xf8, 0x93, 0x37,
-	0xfc, 0x97, 0x53, 0xbb, 0xf6, 0xd7, 0x39, 0x95, 0xda, 0x78, 0xac, 0xec, 0x64, 0xd4, 0xe1, 0x23,
-	0x91, 0x1a, 0xef, 0x1f, 0x50, 0xb8, 0xe9, 0x51, 0xdd, 0xd2, 0x0f, 0xff, 0x15, 0x00, 0x00, 0xff,
-	0xff, 0x10, 0x2c, 0x26, 0x0c, 0x4e, 0x15, 0x00, 0x00,
+	// 2163 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0xcd, 0x73, 0x1b, 0x49,
+	0x15, 0xcf, 0x48, 0xb2, 0x2c, 0xb5, 0x22, 0x67, 0xd3, 0x9b, 0xcd, 0x0e, 0x21, 0xa5, 0xf5, 0x0e,
+	0x29, 0x30, 0x10, 0xac, 0x22, 0x81, 0x25, 0x84, 0x8f, 0x2a, 0x59, 0x4e, 0x62, 0xd5, 0xc6, 0xb1,
+	0x68, 0xd9, 0x6b, 0xd8, 0xc3, 0x56, 0xb5, 0x47, 0x6d, 0xa9, 0xd7, 0xa3, 0xe9, 0xa9, 0x9e, 0x96,
+	0x1d, 0x73, 0x82, 0x1b, 0xff, 0x01, 0x67, 0x2e, 0x7c, 0xfc, 0x07, 0x14, 0x27, 0x38, 0x91, 0xe2,
+	0xc4, 0x91, 0x2a, 0xaa, 0x80, 0xf8, 0xce, 0x9d, 0x23, 0xf5, 0xfa, 0x63, 0xd4, 0x92, 0x6c, 0xd8,
+	0xc0, 0xad, 0xdf, 0x7b, 0xbf, 0xf7, 0xfa, 0xe3, 0x7d, 0xce, 0xa0, 0x17, 0x23, 0xae, 0xc6, 0xd3,
+	0xa3, 0xcd, 0x58, 0x4c, 0xda, 0x27, 0x54, 0xd1, 0xaf, 0xc5, 0x22, 0x55, 0x94, 0xa7, 0x4c, 0xe6,
+	0x4b, 0x74, 0x2e, 0xe3, 0x76, 0xc2, 0x8f, 0xf2, 0x76, 0x26, 0x85, 0x12, 0xb1, 0x48, 0xec, 0x2a,
+	0x6f, 0x8b, 0x98, 0x6f, 0xea, 0x25, 0xae, 0x8c, 0x64, 0x16, 0xdf, 0x89, 0x46, 0x62, 0x24, 0x8c,
+	0xf0, 0x68, 0x7a, 0xdc, 0x06, 0x4a, 0x13, 0x7a, 0x65, 0x90, 0xd1, 0x1f, 0xcb, 0xa8, 0x32, 0xc8,
+	0x58, 0x8c, 0x43, 0xb4, 0xfa, 0x11, 0x93, 0x39, 0x17, 0x69, 0x18, 0xac, 0x07, 0x1b, 0x75, 0xe2,
+	0x48, 0xfc, 0x25, 0xb4, 0xda, 0x97, 0x22, 0x66, 0x79, 0x1e, 0x96, 0xd6, 0x83, 0x8d, 0xc6, 0x83,
+	0xe6, 0x26, 0x98, 0xdf, 0xb4, 0x4c, 0xe2, 0xa4, 0xb8, 0x85, 0x2a, 0x44, 0x08, 0x15, 0x96, 0x35,
+	0x0a, 0x19, 0x14, 0x70, 0x88, 0xe6, 0xe3, 0x3b, 0xa8, 0xb6, 0x23, 0x72, 0x95, 0xd2, 0x09, 0x0b,
+	0x2b, 0x7a, 0x8f, 0x82, 0xc6, 0x5f, 0x46, 0xd5, 0x5d, 0x31, 0x4d, 0x55, 0x1e, 0xae, 0xac, 0x97,
+	0x37, 0x1a, 0x0f, 0x1a, 0x46, 0x5b, 0xf3, 0xb6, 0x2a, 0xaf, 0xfe, 0xf6, 0xde, 0x35, 0x62, 0x01,
+	0xf8, 0x7d, 0xb4, 0xb2, 0x23, 0xc4, 0x49, 0x1e, 0x56, 0xf5, 0x3e, 0x16, 0xa9, 0x59, 0xc4, 0x48,
+	0xf0, 0xf7, 0x50, 0xa3, 0x93, 0xa6, 0x42, 0x51, 0xc5, 0x45, 0x9a, 0x87, 0xab, 0xda, 0xe4, 0xe7,
+	0x0d, 0x10, 0x6e, 0xbb, 0xe9, 0x49, 0x9f, 0xa4, 0x4a, 0x9e, 0x13, 0x1f, 0x0f, 0x3b, 0x3c, 0xe7,
+	0xe9, 0xf4, 0x65, 0x58, 0xf3, 0x77, 0xd0, 0x2c, 0x62, 0x24, 0xf0, 0x28, 0x03, 0x91, 0x50, 0xc9,
+	0xf3, 0xb0, 0xee, 0x3f, 0x8a, 0x65, 0x12, 0x27, 0x05, 0xe0, 0x21, 0x4f, 0x87, 0xe2, 0x2c, 0x0f,
+	0x91, 0x0f, 0xb4, 0x4c, 0xe2, 0xa4, 0x77, 0xbe, 0x8f, 0xde, 0x5a, 0x3c, 0x15, 0x7e, 0x0b, 0x95,
+	0x4f, 0xd8, 0xb9, 0x75, 0x08, 0x2c, 0xf1, 0x2d, 0xb4, 0x72, 0x4a, 0x93, 0x29, 0xd3, 0xae, 0xa8,
+	0x13, 0x43, 0x3c, 0x2e, 0x3d, 0x0a, 0xa2, 0xdf, 0x95, 0x0b, 0x3f, 0xc1, 0x4b, 0xef, 0x33, 0x39,
+	0xe1, 0x29, 0x4d, 0xb4, 0x72, 0x8d, 0x14, 0x34, 0xfe, 0x2a, 0x6a, 0x74, 0x45, 0x9a, 0x8b, 0x84,
+	0x0d, 0xf8, 0x8f, 0x99, 0x75, 0x69, 0xdd, 0x1c, 0x6a, 0x4b, 0xbc, 0x24, 0xbe, 0x14, 0xdf, 0x43,
+	0x95, 0x83, 0x9c, 0xc9, 0x79, 0x97, 0x02, 0xc7, 0xfa, 0x44, 0x4b, 0x31, 0x46, 0x95, 0x8e, 0x1c,
+	0xe5, 0x61, 0x65, 0xbd, 0xbc, 0x51, 0x27, 0x7a, 0x0d, 0x47, 0x7f, 0x92, 0x9e, 0x6a, 0x6f, 0xd6,
+	0x09, 0x2c, 0x81, 0xd3, 0x3d, 0x1b, 0x6a, 0xaf, 0xd5, 0x09, 0x2c, 0xf1, 0x77, 0xd0, 0xf5, 0x2e,
+	0xcd, 0xe8, 0x11, 0x4f, 0xb8, 0xe2, 0x0c, 0xfc, 0x04, 0xbb, 0xbc, 0xeb, 0x3d, 0xb7, 0x2f, 0x26,
+	0x73, 0x60, 0xfc, 0x75, 0xb4, 0x4a, 0x12, 0x3e, 0xe1, 0x2a, 0x0f, 0x6b, 0xda, 0xbf, 0x37, 0x6d,
+	0x58, 0xee, 0x0d, 0x7a, 0x3f, 0x34, 0x12, 0x7b, 0x48, 0x87, 0xc3, 0x1b, 0xe8, 0xc6, 0x0b, 0xf1,
+	0x82, 0x9d, 0xf5, 0x25, 0x3f, 0xe5, 0x09, 0x1b, 0x31, 0xe3, 0xbc, 0x1a, 0x59, 0x64, 0x03, 0xb2,
+	0x93, 0x65, 0x54, 0x4e, 0x84, 0xec, 0x4b, 0x71, 0xcc, 0x13, 0xa6, 0xbd, 0x57, 0x27, 0x8b, 0x6c,
+	0xbc, 0x8e, 0x1a, 0x7b, 0x7b, 0xbb, 0x83, 0x58, 0x48, 0xd6, 0x19, 0x7e, 0x1a, 0x36, 0xd6, 0x83,
+	0x8d, 0x32, 0xf1, 0x59, 0x38, 0x42, 0xd7, 0x07, 0x2c, 0x81, 0xdb, 0x3c, 0xa7, 0x47, 0x2c, 0x09,
+	0xaf, 0x6b, 0x43, 0x73, 0xbc, 0xe8, 0x21, 0x2a, 0x6f, 0x89, 0x97, 0xf8, 0x36, 0xaa, 0xee, 0x30,
+	0x3e, 0x1a, 0x2b, 0xed, 0xb5, 0x26, 0xb1, 0x14, 0x78, 0xfd, 0x90, 0x0f, 0xd5, 0x58, 0x7b, 0xab,
+	0x49, 0x0c, 0x11, 0xa5, 0xc6, 0x39, 0xf0, 0xb0, 0x07, 0xbd, 0x6d, 0xab, 0x02, 0x4b, 0xe0, 0x3c,
+	0xeb, 0x6d, 0x5b, 0x34, 0x2c, 0xf1, 0x17, 0xd1, 0x5a, 0x67, 0x38, 0xe4, 0x10, 0x5b, 0x34, 0x79,
+	0xc6, 0x87, 0x79, 0x58, 0x5e, 0x2f, 0x6f, 0x34, 0xc9, 0x02, 0x17, 0x22, 0x07, 0x6c, 0xfa, 0x39,
+	0xea, 0xe8, 0xe8, 0x57, 0x01, 0xba, 0xb9, 0xe4, 0x15, 0xd0, 0xd8, 0x12, 0xd3, 0x74, 0xc8, 0xd3,
+	0x51, 0x18, 0x68, 0x6f, 0x17, 0x34, 0xbe, 0x8b, 0xea, 0x4f, 0x8e, 0x8f, 0x59, 0xac, 0xf8, 0x29,
+	0x44, 0x1a, 0x08, 0x67, 0x0c, 0x78, 0xba, 0x5e, 0x3a, 0x66, 0x92, 0x2b, 0x7a, 0x94, 0x30, 0x7d,
+	0xa0, 0x3a, 0xf1, 0x59, 0xa0, 0xdf, 0x87, 0xb8, 0x55, 0x8a, 0x0d, 0x6d, 0x74, 0xcd, 0x18, 0x50,
+	0xb2, 0x3a, 0x93, 0x23, 0xce, 0x52, 0x65, 0xc3, 0xcc, 0x91, 0x51, 0x0f, 0x35, 0xbc, 0x30, 0x80,
+	0xf8, 0xdc, 0x3f, 0xcf, 0x98, 0xcd, 0x23, 0xbd, 0x06, 0xde, 0x0e, 0x95, 0x43, 0xfd, 0x46, 0x15,
+	0xa2, 0xd7, 0xc0, 0x1b, 0x88, 0x63, 0x53, 0xc0, 0x2a, 0x44, 0xaf, 0x23, 0x81, 0x56, 0x74, 0xdd,
+	0x81, 0xd3, 0x0e, 0x59, 0xae, 0x78, 0xaa, 0x13, 0xd4, 0xda, 0xf2, 0x59, 0xe0, 0xbd, 0x5c, 0x4c,
+	0x65, 0xec, 0x92, 0xd3, 0x52, 0x60, 0x56, 0xc1, 0xf6, 0x65, 0xb3, 0x3d, 0xac, 0xe1, 0xec, 0x22,
+	0x33, 0xd5, 0xc9, 0xdc, 0xcb, 0x91, 0xd1, 0x07, 0xa6, 0x8a, 0x82, 0x56, 0x9f, 0xaa, 0xb1, 0x3b,
+	0x34, 0xac, 0xe1, 0xad, 0x09, 0xa3, 0x43, 0x91, 0x26, 0xe7, 0x7a, 0x8f, 0x1a, 0x29, 0xe8, 0xe8,
+	0x0f, 0x25, 0x5b, 0x17, 0xf1, 0x7d, 0x54, 0xeb, 0x4b, 0x96, 0x2b, 0x2a, 0x95, 0xf6, 0x48, 0x91,
+	0xb8, 0x20, 0xb6, 0x39, 0x51, 0x20, 0xf0, 0x26, 0xaa, 0xf7, 0x45, 0xae, 0x0c, 0xbc, 0x74, 0x05,
+	0x7c, 0x06, 0xd1, 0xd6, 0x35, 0x21, 0x32, 0xed, 0xb2, 0xcb, 0xad, 0x5b, 0x04, 0xfe, 0x00, 0x35,
+	0xbb, 0x92, 0x51, 0xc5, 0xc8, 0x34, 0x55, 0x5c, 0x07, 0xd5, 0xe5, 0x2a, 0xf3, 0x30, 0xfc, 0x18,
+	0xdd, 0x30, 0x8c, 0xae, 0xeb, 0x7c, 0xb6, 0x31, 0x2c, 0x6b, 0x2e, 0x02, 0xf1, 0x23, 0xb4, 0x36,
+	0x80, 0xa3, 0xce, 0x54, 0xab, 0x57, 0xa8, 0x2e, 0xe0, 0xa2, 0x8f, 0x51, 0x05, 0xa4, 0x97, 0xbe,
+	0xbd, 0x2b, 0x72, 0xa5, 0xe5, 0x22, 0x57, 0x9e, 0x15, 0xb9, 0x10, 0xad, 0xee, 0xf3, 0x09, 0x13,
+	0x53, 0xa5, 0xd3, 0xa7, 0x4c, 0x1c, 0x19, 0xfd, 0x66, 0xc5, 0x76, 0x15, 0xfc, 0x5d, 0xd4, 0x38,
+	0xe8, 0x6d, 0xef, 0xd2, 0x2c, 0xe3, 0xe9, 0x28, 0xb7, 0x2e, 0xba, 0xe5, 0x55, 0xbd, 0x42, 0x68,
+	0x8f, 0xe9, 0xc3, 0x41, 0xfb, 0x99, 0xa7, 0x5d, 0xfa, 0xef, 0xda, 0x1e, 0x1c, 0xb7, 0x51, 0x75,
+	0x70, 0x9e, 0xc7, 0x2a, 0xb1, 0xbe, 0xf3, 0x8b, 0xed, 0xa6, 0x91, 0x98, 0x86, 0x68, 0x61, 0xf8,
+	0x01, 0xaa, 0x13, 0x66, 0x02, 0x39, 0xd7, 0x57, 0x9a, 0xdf, 0xac, 0x90, 0x91, 0x19, 0x0c, 0x52,
+	0xa5, 0x3b, 0x92, 0x62, 0x9a, 0xe5, 0xfa, 0x15, 0x57, 0x4c, 0xaa, 0x78, 0x2c, 0xfc, 0x18, 0xa1,
+	0x17, 0x74, 0xc2, 0xf2, 0x8c, 0x82, 0xd9, 0xea, 0xd2, 0x1d, 0x0a, 0xa1, 0xbd, 0x83, 0x87, 0x86,
+	0xc2, 0xbf, 0xcd, 0x4e, 0x79, 0xcc, 0x5c, 0x63, 0xbf, 0xe9, 0x29, 0x1a, 0x89, 0x2b, 0xfc, 0x16,
+	0x87, 0xef, 0xa3, 0xd5, 0x01, 0x8b, 0x63, 0x31, 0xc9, 0x6c, 0x4b, 0xc7, 0x9e, 0x8a, 0x95, 0x10,
+	0x07, 0xc1, 0xf7, 0xd1, 0x4d, 0xc8, 0xc0, 0xe3, 0xbc, 0x2f, 0x45, 0x46, 0x47, 0x26, 0xdf, 0xeb,
+	0xfa, 0x12, 0xcb, 0x02, 0xb8, 0xec, 0x2e, 0xcd, 0x4f, 0xd8, 0x10, 0x2e, 0x06, 0x4d, 0x5e, 0x57,
+	0x31, 0x8f, 0x85, 0xef, 0xa1, 0xa6, 0xcb, 0x52, 0x83, 0x69, 0x68, 0xcc, 0x3c, 0x13, 0xb7, 0x10,
+	0xd2, 0x85, 0xc6, 0x6f, 0x12, 0x1e, 0x07, 0xb7, 0x51, 0xad, 0x97, 0x2a, 0x96, 0x90, 0xa1, 0x0a,
+	0x9b, 0xfa, 0x12, 0x6f, 0xfb, 0x4e, 0xb7, 0x22, 0x52, 0x80, 0xee, 0x7c, 0x1b, 0x35, 0x3c, 0x87,
+	0xbe, 0xd1, 0x2c, 0xf1, 0x5e, 0x31, 0xb4, 0x00, 0x68, 0x38, 0x9d, 0x4c, 0x9c, 0xa2, 0x21, 0x00,
+	0xe0, 0x06, 0x9c, 0xcb, 0x01, 0x9f, 0xa0, 0xb5, 0xf9, 0x60, 0xd4, 0xbd, 0x4d, 0xe4, 0xaa, 0x68,
+	0x54, 0x96, 0xd2, 0xc1, 0xe2, 0x12, 0xb0, 0xe8, 0x59, 0x3e, 0x4b, 0x97, 0x65, 0x18, 0x55, 0xca,
+	0x5a, 0xa4, 0xd7, 0xd1, 0x23, 0x6b, 0xbf, 0x88, 0x8b, 0xab, 0x8a, 0xbc, 0x8e, 0xc0, 0xd2, 0x2c,
+	0x8f, 0xa3, 0x5f, 0x04, 0xa8, 0xe1, 0x85, 0xca, 0x55, 0xb9, 0xae, 0x6d, 0x95, 0x3c, 0x5b, 0xb7,
+	0xd0, 0xca, 0x2e, 0xfd, 0x54, 0x98, 0x59, 0xa8, 0x4c, 0x0c, 0xa1, 0xb9, 0x3c, 0x15, 0xd2, 0x66,
+	0xbb, 0x21, 0xa0, 0x4e, 0x3f, 0xe5, 0x09, 0xdb, 0x15, 0x43, 0xa6, 0xa3, 0xbf, 0x49, 0x0a, 0xda,
+	0x75, 0xeb, 0xea, 0x52, 0xb7, 0x5e, 0x2d, 0xba, 0x75, 0xf4, 0xf7, 0x92, 0xbd, 0xde, 0x2c, 0xa7,
+	0xbe, 0x35, 0x8b, 0xfa, 0x60, 0x29, 0x73, 0x8d, 0xc4, 0x24, 0xd8, 0x62, 0xec, 0xc3, 0x64, 0xcd,
+	0x26, 0x42, 0x9e, 0xdb, 0x51, 0xcf, 0xcf, 0x16, 0x23, 0x20, 0x16, 0x80, 0xd7, 0x51, 0xb9, 0xdb,
+	0x3f, 0xb0, 0xc3, 0xde, 0x9a, 0x3f, 0x86, 0xf5, 0x0f, 0x08, 0x88, 0xf0, 0x17, 0x50, 0xa5, 0x0f,
+	0xc3, 0x83, 0x29, 0x04, 0x37, 0x3c, 0x08, 0xb0, 0x89, 0x16, 0x42, 0xb6, 0x6d, 0x25, 0x22, 0x3e,
+	0xe9, 0xed, 0xe9, 0xcb, 0xcf, 0x67, 0x9b, 0x95, 0x10, 0x07, 0xc1, 0x4f, 0xd1, 0xda, 0xce, 0x74,
+	0xc4, 0x32, 0x3a, 0x62, 0xcf, 0xcd, 0x38, 0x67, 0xca, 0x41, 0xe8, 0x29, 0xcd, 0x01, 0x5c, 0xed,
+	0x9e, 0xd7, 0x82, 0x5d, 0x5f, 0x30, 0x75, 0x26, 0xe4, 0x89, 0x9d, 0x23, 0xfd, 0x5d, 0xad, 0x84,
+	0x38, 0x48, 0xf4, 0x57, 0x17, 0x05, 0xf6, 0xea, 0xb7, 0xa0, 0x38, 0x4f, 0xb8, 0x19, 0xbc, 0xca,
+	0xc4, 0x10, 0x10, 0x9b, 0x84, 0xe5, 0x4c, 0x9e, 0x9a, 0x1a, 0x50, 0x32, 0xc3, 0x9d, 0xc7, 0xd2,
+	0xb1, 0x79, 0x46, 0x33, 0x1b, 0x14, 0x7a, 0x0d, 0x91, 0xfe, 0x21, 0x93, 0x29, 0x4b, 0x6c, 0x50,
+	0x58, 0x0a, 0xa6, 0x19, 0xb3, 0xda, 0xef, 0xf6, 0xf5, 0xcb, 0x94, 0xc9, 0x8c, 0x01, 0xf9, 0x0f,
+	0xda, 0x19, 0x4f, 0xe1, 0x4b, 0xab, 0xaa, 0x47, 0x10, 0x8f, 0x83, 0xbf, 0x82, 0xde, 0xda, 0xe6,
+	0x39, 0x8c, 0x45, 0x7b, 0x7b, 0xbb, 0x1f, 0xf2, 0x24, 0x61, 0x52, 0x5f, 0xb4, 0x46, 0x96, 0xf8,
+	0xd1, 0x9f, 0x02, 0x54, 0x73, 0x8e, 0x83, 0xe3, 0x0c, 0xc6, 0x54, 0xea, 0xc0, 0x01, 0xa3, 0x96,
+	0x82, 0x2b, 0xff, 0x60, 0x2a, 0x14, 0xb5, 0xd7, 0x32, 0x04, 0xa0, 0xfb, 0x4c, 0x72, 0x31, 0xb4,
+	0x53, 0x90, 0xa5, 0x60, 0x22, 0x26, 0x8c, 0x26, 0xd0, 0x9c, 0x67, 0xad, 0x1c, 0xf4, 0x16, 0xd9,
+	0x30, 0x6a, 0x3a, 0x96, 0xb5, 0xb4, 0xa2, 0x2d, 0x2d, 0x70, 0xe1, 0xe9, 0xba, 0xd9, 0x34, 0xb7,
+	0x1f, 0x04, 0x7a, 0x0d, 0xbc, 0x5d, 0x36, 0x31, 0x5f, 0x02, 0x75, 0xa2, 0xd7, 0xd1, 0x99, 0x9d,
+	0x3a, 0x0f, 0xf5, 0x2c, 0x6c, 0xb3, 0xb6, 0xc8, 0xc6, 0xe0, 0xd2, 0x6c, 0x2c, 0xf9, 0xd9, 0x78,
+	0x1b, 0x55, 0x8d, 0xae, 0xad, 0x20, 0x96, 0x82, 0x17, 0x7f, 0xce, 0xe8, 0xb1, 0x95, 0x55, 0xb4,
+	0xcc, 0xe3, 0x44, 0x07, 0xe8, 0x6d, 0xbd, 0xf1, 0xfe, 0x58, 0x0a, 0xa5, 0x12, 0xf6, 0x3f, 0x6c,
+	0x8d, 0x51, 0x85, 0x50, 0xc5, 0xdc, 0x44, 0x09, 0xeb, 0xe8, 0x9f, 0x65, 0x74, 0xdd, 0x4f, 0x05,
+	0xef, 0x7c, 0xc1, 0x7f, 0x38, 0x5f, 0x69, 0xf1, 0x7c, 0xb8, 0x83, 0xae, 0xfb, 0x6f, 0x72, 0x49,
+	0x47, 0xf7, 0xc5, 0x36, 0x6d, 0xe6, 0x54, 0xf0, 0x01, 0x7a, 0xc7, 0xdd, 0x0e, 0xba, 0xd1, 0x56,
+	0x96, 0x5b, 0x5b, 0x66, 0x4c, 0xfb, 0x9c, 0x67, 0x6b, 0xfe, 0x15, 0xac, 0xb5, 0xcb, 0xb5, 0xf1,
+	0x21, 0xba, 0xed, 0x04, 0x87, 0x92, 0x2b, 0x36, 0xb3, 0xbb, 0xf2, 0xd9, 0xec, 0x5e, 0xa1, 0xee,
+	0x1b, 0x86, 0x1d, 0x7b, 0x7b, 0xfd, 0x81, 0x35, 0x5c, 0x7d, 0x43, 0xc3, 0xf3, 0xea, 0xf8, 0x47,
+	0xe8, 0xdd, 0xb9, 0x2d, 0x3d, 0xcb, 0xab, 0x9f, 0xcd, 0xf2, 0x55, 0xfa, 0xd1, 0xfb, 0xa8, 0x5e,
+	0x54, 0xc8, 0xcb, 0xeb, 0x4c, 0xf4, 0x53, 0xf7, 0x65, 0xe5, 0x17, 0x72, 0xc0, 0x76, 0x92, 0x44,
+	0x9c, 0xd9, 0x4f, 0x78, 0x43, 0xfc, 0xdf, 0xbd, 0xe9, 0x36, 0xaa, 0x76, 0x62, 0xfd, 0x37, 0xc7,
+	0xcc, 0x65, 0x96, 0x8a, 0x12, 0x1b, 0x95, 0xb6, 0x42, 0xc2, 0x24, 0xdb, 0x4d, 0x68, 0x9e, 0x17,
+	0x0d, 0xdb, 0x91, 0x78, 0x0b, 0xa1, 0xbe, 0xe4, 0x42, 0x9a, 0x8f, 0x76, 0x33, 0x80, 0xde, 0x5d,
+	0x98, 0x45, 0xe4, 0x31, 0x8d, 0x99, 0x45, 0x9d, 0xbb, 0x21, 0x6e, 0xa6, 0x15, 0x3d, 0x45, 0x78,
+	0xb9, 0xb2, 0x43, 0xdf, 0xec, 0xd3, 0x11, 0xcb, 0xa1, 0xdb, 0x9b, 0x7e, 0x5c, 0xd0, 0xb3, 0x97,
+	0x33, 0x5f, 0x6c, 0xf6, 0xe5, 0x76, 0xd0, 0xed, 0xcb, 0xf7, 0x84, 0x77, 0x82, 0xe1, 0xc0, 0xf5,
+	0x75, 0x58, 0x6b, 0xfb, 0x56, 0x6e, 0xf3, 0xa9, 0xa0, 0xa3, 0x5f, 0x06, 0xf6, 0x01, 0xdc, 0x18,
+	0x78, 0x0f, 0x35, 0xb7, 0xd9, 0x31, 0x9d, 0x26, 0xaa, 0x13, 0x7b, 0x9f, 0x7c, 0xf3, 0x4c, 0x40,
+	0x75, 0x64, 0x3c, 0xe6, 0x8a, 0xc5, 0x6a, 0x2a, 0x99, 0xfb, 0x3e, 0x98, 0x67, 0xc2, 0xe1, 0x9f,
+	0x26, 0x74, 0x94, 0xdb, 0x4f, 0x05, 0x43, 0xe0, 0x6f, 0xa0, 0x1a, 0x4c, 0x68, 0x34, 0x49, 0x72,
+	0x9b, 0x70, 0x73, 0x73, 0xa9, 0x11, 0xb9, 0x4f, 0x2a, 0x87, 0x8c, 0x38, 0xba, 0xe1, 0x9f, 0xb3,
+	0x23, 0x47, 0x60, 0xbe, 0x97, 0x0e, 0xd9, 0x4b, 0x5b, 0xe1, 0x0d, 0x01, 0xdc, 0x8f, 0x8a, 0xf9,
+	0xae, 0x42, 0x0c, 0x01, 0x6f, 0xa0, 0x17, 0xfb, 0x67, 0xc2, 0x96, 0xa5, 0x82, 0xc6, 0x6b, 0xa8,
+	0xb4, 0x97, 0xd9, 0xef, 0xfe, 0xd2, 0x5e, 0x16, 0xfd, 0xbc, 0x78, 0x13, 0xb3, 0x39, 0x98, 0xd4,
+	0x13, 0x97, 0xfd, 0xd2, 0x37, 0x84, 0x09, 0xa9, 0xa2, 0x43, 0xea, 0x90, 0xd2, 0x6f, 0x73, 0x17,
+	0xd5, 0x98, 0x94, 0xa9, 0x90, 0xcc, 0x96, 0xde, 0x9d, 0x6b, 0xa4, 0xe0, 0xe0, 0xb6, 0xf7, 0xd7,
+	0xa8, 0xf1, 0xe0, 0x9d, 0xe5, 0x89, 0xbc, 0x23, 0xdd, 0x27, 0x8c, 0x06, 0x6e, 0x21, 0x54, 0x7b,
+	0x02, 0xca, 0x84, 0xa9, 0xe8, 0x9b, 0xa8, 0x39, 0x37, 0xf7, 0x82, 0x1f, 0x9e, 0x3f, 0xec, 0xd2,
+	0x78, 0xcc, 0x06, 0xf1, 0x98, 0x4d, 0xa8, 0xf3, 0xd6, 0x1c, 0x73, 0xeb, 0x67, 0xc1, 0xab, 0xd7,
+	0xad, 0x6b, 0x7f, 0x79, 0xdd, 0xba, 0xf6, 0xaf, 0xd7, 0xad, 0xe0, 0x27, 0x17, 0xad, 0xe0, 0xd7,
+	0x17, 0xad, 0xe0, 0xb7, 0x17, 0xad, 0xe0, 0xf7, 0x17, 0xad, 0xe0, 0xd5, 0x45, 0x2b, 0xf8, 0xf3,
+	0x45, 0x2b, 0xf8, 0xc7, 0x45, 0x2b, 0xf8, 0xf8, 0x93, 0x37, 0xfc, 0x27, 0x2b, 0x4d, 0xfb, 0x6b,
+	0x9f, 0x72, 0xa9, 0x3c, 0x51, 0x76, 0x32, 0x6a, 0xd3, 0x11, 0x4b, 0x95, 0xf7, 0xbf, 0x16, 0x6e,
+	0x7a, 0x54, 0xd5, 0xf4, 0xc3, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xce, 0xda, 0xe2, 0x21, 0xfc,
+	0x15, 0x00, 0x00,
 }
 
 func (this *Spec) Equal(that interface{}) bool {
@@ -2103,6 +2113,30 @@ func (this *Hooks) Equal(that interface{}) bool {
 			return false
 		}
 	}
+	if len(this.CreateRuntime) != len(that1.CreateRuntime) {
+		return false
+	}
+	for i := range this.CreateRuntime {
+		if !this.CreateRuntime[i].Equal(&that1.CreateRuntime[i]) {
+			return false
+		}
+	}
+	if len(this.CreateContainer) != len(that1.CreateContainer) {
+		return false
+	}
+	for i := range this.CreateContainer {
+		if !this.CreateContainer[i].Equal(&that1.CreateContainer[i]) {
+			return false
+		}
+	}
+	if len(this.StartContainer) != len(that1.StartContainer) {
+		return false
+	}
+	for i := range this.StartContainer {
+		if !this.StartContainer[i].Equal(&that1.StartContainer[i]) {
+			return false
+		}
+	}
 	if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) {
 		return false
 	}
@@ -3674,6 +3708,48 @@ func (m *Hooks) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 		i -= len(m.XXX_unrecognized)
 		copy(dAtA[i:], m.XXX_unrecognized)
 	}
+	if len(m.StartContainer) > 0 {
+		for iNdEx := len(m.StartContainer) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.StartContainer[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintOci(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x32
+		}
+	}
+	if len(m.CreateContainer) > 0 {
+		for iNdEx := len(m.CreateContainer) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.CreateContainer[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintOci(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x2a
+		}
+	}
+	if len(m.CreateRuntime) > 0 {
+		for iNdEx := len(m.CreateRuntime) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.CreateRuntime[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintOci(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x22
+		}
+	}
 	if len(m.Poststop) > 0 {
 		for iNdEx := len(m.Poststop) - 1; iNdEx >= 0; iNdEx-- {
 			{
@@ -5277,8 +5353,32 @@ func NewPopulatedHooks(r randyOci, easy bool) *Hooks {
 			this.Poststop[i] = *v21
 		}
 	}
+	if r.Intn(5) != 0 {
+		v22 := r.Intn(5)
+		this.CreateRuntime = make([]Hook, v22)
+		for i := 0; i < v22; i++ {
+			v23 := NewPopulatedHook(r, easy)
+			this.CreateRuntime[i] = *v23
+		}
+	}
+	if r.Intn(5) != 0 {
+		v24 := r.Intn(5)
+		this.CreateContainer = make([]Hook, v24)
+		for i := 0; i < v24; i++ {
+			v25 := NewPopulatedHook(r, easy)
+			this.CreateContainer[i] = *v25
+		}
+	}
+	if r.Intn(5) != 0 {
+		v26 := r.Intn(5)
+		this.StartContainer = make([]Hook, v26)
+		for i := 0; i < v26; i++ {
+			v27 := NewPopulatedHook(r, easy)
+			this.StartContainer[i] = *v27
+		}
+	}
 	if !easy && r.Intn(10) != 0 {
-		this.XXX_unrecognized = randUnrecognizedOci(r, 4)
+		this.XXX_unrecognized = randUnrecognizedOci(r, 7)
 	}
 	return this
 }
@@ -5286,14 +5386,14 @@ func NewPopulatedHooks(r randyOci, easy bool) *Hooks {
 func NewPopulatedHook(r randyOci, easy bool) *Hook {
 	this := &Hook{}
 	this.Path = string(randStringOci(r))
-	v22 := r.Intn(10)
-	this.Args = make([]string, v22)
-	for i := 0; i < v22; i++ {
+	v28 := r.Intn(10)
+	this.Args = make([]string, v28)
+	for i := 0; i < v28; i++ {
 		this.Args[i] = string(randStringOci(r))
 	}
-	v23 := r.Intn(10)
-	this.Env = make([]string, v23)
-	for i := 0; i < v23; i++ {
+	v29 := r.Intn(10)
+	this.Env = make([]string, v29)
+	for i := 0; i < v29; i++ {
 		this.Env[i] = string(randStringOci(r))
 	}
 	this.Timeout = int64(r.Int63())
@@ -5309,25 +5409,25 @@ func NewPopulatedHook(r randyOci, easy bool) *Hook {
 func NewPopulatedLinux(r randyOci, easy bool) *Linux {
 	this := &Linux{}
 	if r.Intn(5) != 0 {
-		v24 := r.Intn(5)
-		this.UIDMappings = make([]LinuxIDMapping, v24)
-		for i := 0; i < v24; i++ {
-			v25 := NewPopulatedLinuxIDMapping(r, easy)
-			this.UIDMappings[i] = *v25
+		v30 := r.Intn(5)
+		this.UIDMappings = make([]LinuxIDMapping, v30)
+		for i := 0; i < v30; i++ {
+			v31 := NewPopulatedLinuxIDMapping(r, easy)
+			this.UIDMappings[i] = *v31
 		}
 	}
 	if r.Intn(5) != 0 {
-		v26 := r.Intn(5)
-		this.GIDMappings = make([]LinuxIDMapping, v26)
-		for i := 0; i < v26; i++ {
-			v27 := NewPopulatedLinuxIDMapping(r, easy)
-			this.GIDMappings[i] = *v27
+		v32 := r.Intn(5)
+		this.GIDMappings = make([]LinuxIDMapping, v32)
+		for i := 0; i < v32; i++ {
+			v33 := NewPopulatedLinuxIDMapping(r, easy)
+			this.GIDMappings[i] = *v33
 		}
 	}
 	if r.Intn(5) != 0 {
-		v28 := r.Intn(10)
+		v34 := r.Intn(10)
 		this.Sysctl = make(map[string]string)
-		for i := 0; i < v28; i++ {
+		for i := 0; i < v34; i++ {
 			this.Sysctl[randStringOci(r)] = randStringOci(r)
 		}
 	}
@@ -5336,33 +5436,33 @@ func NewPopulatedLinux(r randyOci, easy bool) *Linux {
 	}
 	this.CgroupsPath = string(randStringOci(r))
 	if r.Intn(5) != 0 {
-		v29 := r.Intn(5)
-		this.Namespaces = make([]LinuxNamespace, v29)
-		for i := 0; i < v29; i++ {
-			v30 := NewPopulatedLinuxNamespace(r, easy)
-			this.Namespaces[i] = *v30
+		v35 := r.Intn(5)
+		this.Namespaces = make([]LinuxNamespace, v35)
+		for i := 0; i < v35; i++ {
+			v36 := NewPopulatedLinuxNamespace(r, easy)
+			this.Namespaces[i] = *v36
 		}
 	}
 	if r.Intn(5) != 0 {
-		v31 := r.Intn(5)
-		this.Devices = make([]LinuxDevice, v31)
-		for i := 0; i < v31; i++ {
-			v32 := NewPopulatedLinuxDevice(r, easy)
-			this.Devices[i] = *v32
+		v37 := r.Intn(5)
+		this.Devices = make([]LinuxDevice, v37)
+		for i := 0; i < v37; i++ {
+			v38 := NewPopulatedLinuxDevice(r, easy)
+			this.Devices[i] = *v38
 		}
 	}
 	if r.Intn(5) != 0 {
 		this.Seccomp = NewPopulatedLinuxSeccomp(r, easy)
 	}
 	this.RootfsPropagation = string(randStringOci(r))
-	v33 := r.Intn(10)
-	this.MaskedPaths = make([]string, v33)
-	for i := 0; i < v33; i++ {
+	v39 := r.Intn(10)
+	this.MaskedPaths = make([]string, v39)
+	for i := 0; i < v39; i++ {
 		this.MaskedPaths[i] = string(randStringOci(r))
 	}
-	v34 := r.Intn(10)
-	this.ReadonlyPaths = make([]string, v34)
-	for i := 0; i < v34; i++ {
+	v40 := r.Intn(10)
+	this.ReadonlyPaths = make([]string, v40)
+	for i := 0; i < v40; i++ {
 		this.ReadonlyPaths[i] = string(randStringOci(r))
 	}
 	this.MountLabel = string(randStringOci(r))
@@ -5438,11 +5538,11 @@ func NewPopulatedLinuxDevice(r randyOci, easy bool) *LinuxDevice {
 func NewPopulatedLinuxResources(r randyOci, easy bool) *LinuxResources {
 	this := &LinuxResources{}
 	if r.Intn(5) != 0 {
-		v35 := r.Intn(5)
-		this.Devices = make([]LinuxDeviceCgroup, v35)
-		for i := 0; i < v35; i++ {
-			v36 := NewPopulatedLinuxDeviceCgroup(r, easy)
-			this.Devices[i] = *v36
+		v41 := r.Intn(5)
+		this.Devices = make([]LinuxDeviceCgroup, v41)
+		for i := 0; i < v41; i++ {
+			v42 := NewPopulatedLinuxDeviceCgroup(r, easy)
+			this.Devices[i] = *v42
 		}
 	}
 	if r.Intn(5) != 0 {
@@ -5458,11 +5558,11 @@ func NewPopulatedLinuxResources(r randyOci, easy bool) *LinuxResources {
 		this.BlockIO = NewPopulatedLinuxBlockIO(r, easy)
 	}
 	if r.Intn(5) != 0 {
-		v37 := r.Intn(5)
-		this.HugepageLimits = make([]LinuxHugepageLimit, v37)
-		for i := 0; i < v37; i++ {
-			v38 := NewPopulatedLinuxHugepageLimit(r, easy)
-			this.HugepageLimits[i] = *v38
+		v43 := r.Intn(5)
+		this.HugepageLimits = make([]LinuxHugepageLimit, v43)
+		for i := 0; i < v43; i++ {
+			v44 := NewPopulatedLinuxHugepageLimit(r, easy)
+			this.HugepageLimits[i] = *v44
 		}
 	}
 	if r.Intn(5) != 0 {
@@ -5564,44 +5664,44 @@ func NewPopulatedLinuxBlockIO(r randyOci, easy bool) *LinuxBlockIO {
 	this := &LinuxBlockIO{}
 	this.Weight = uint32(r.Uint32())
 	this.LeafWeight = uint32(r.Uint32())
-	if r.Intn(5) != 0 {
-		v39 := r.Intn(5)
-		this.WeightDevice = make([]LinuxWeightDevice, v39)
-		for i := 0; i < v39; i++ {
-			v40 := NewPopulatedLinuxWeightDevice(r, easy)
-			this.WeightDevice[i] = *v40
-		}
-	}
-	if r.Intn(5) != 0 {
-		v41 := r.Intn(5)
-		this.ThrottleReadBpsDevice = make([]LinuxThrottleDevice, v41)
-		for i := 0; i < v41; i++ {
-			v42 := NewPopulatedLinuxThrottleDevice(r, easy)
-			this.ThrottleReadBpsDevice[i] = *v42
-		}
-	}
-	if r.Intn(5) != 0 {
-		v43 := r.Intn(5)
-		this.ThrottleWriteBpsDevice = make([]LinuxThrottleDevice, v43)
-		for i := 0; i < v43; i++ {
-			v44 := NewPopulatedLinuxThrottleDevice(r, easy)
-			this.ThrottleWriteBpsDevice[i] = *v44
-		}
-	}
 	if r.Intn(5) != 0 {
 		v45 := r.Intn(5)
-		this.ThrottleReadIOPSDevice = make([]LinuxThrottleDevice, v45)
+		this.WeightDevice = make([]LinuxWeightDevice, v45)
 		for i := 0; i < v45; i++ {
-			v46 := NewPopulatedLinuxThrottleDevice(r, easy)
-			this.ThrottleReadIOPSDevice[i] = *v46
+			v46 := NewPopulatedLinuxWeightDevice(r, easy)
+			this.WeightDevice[i] = *v46
 		}
 	}
 	if r.Intn(5) != 0 {
 		v47 := r.Intn(5)
-		this.ThrottleWriteIOPSDevice = make([]LinuxThrottleDevice, v47)
+		this.ThrottleReadBpsDevice = make([]LinuxThrottleDevice, v47)
 		for i := 0; i < v47; i++ {
 			v48 := NewPopulatedLinuxThrottleDevice(r, easy)
-			this.ThrottleWriteIOPSDevice[i] = *v48
+			this.ThrottleReadBpsDevice[i] = *v48
+		}
+	}
+	if r.Intn(5) != 0 {
+		v49 := r.Intn(5)
+		this.ThrottleWriteBpsDevice = make([]LinuxThrottleDevice, v49)
+		for i := 0; i < v49; i++ {
+			v50 := NewPopulatedLinuxThrottleDevice(r, easy)
+			this.ThrottleWriteBpsDevice[i] = *v50
+		}
+	}
+	if r.Intn(5) != 0 {
+		v51 := r.Intn(5)
+		this.ThrottleReadIOPSDevice = make([]LinuxThrottleDevice, v51)
+		for i := 0; i < v51; i++ {
+			v52 := NewPopulatedLinuxThrottleDevice(r, easy)
+			this.ThrottleReadIOPSDevice[i] = *v52
+		}
+	}
+	if r.Intn(5) != 0 {
+		v53 := r.Intn(5)
+		this.ThrottleWriteIOPSDevice = make([]LinuxThrottleDevice, v53)
+		for i := 0; i < v53; i++ {
+			v54 := NewPopulatedLinuxThrottleDevice(r, easy)
+			this.ThrottleWriteIOPSDevice[i] = *v54
 		}
 	}
 	if !easy && r.Intn(10) != 0 {
@@ -5645,11 +5745,11 @@ func NewPopulatedLinuxNetwork(r randyOci, easy bool) *LinuxNetwork {
 	this := &LinuxNetwork{}
 	this.ClassID = uint32(r.Uint32())
 	if r.Intn(5) != 0 {
-		v49 := r.Intn(5)
-		this.Priorities = make([]LinuxInterfacePriority, v49)
-		for i := 0; i < v49; i++ {
-			v50 := NewPopulatedLinuxInterfacePriority(r, easy)
-			this.Priorities[i] = *v50
+		v55 := r.Intn(5)
+		this.Priorities = make([]LinuxInterfacePriority, v55)
+		for i := 0; i < v55; i++ {
+			v56 := NewPopulatedLinuxInterfacePriority(r, easy)
+			this.Priorities[i] = *v56
 		}
 	}
 	if !easy && r.Intn(10) != 0 {
@@ -5681,22 +5781,22 @@ func NewPopulatedLinuxInterfacePriority(r randyOci, easy bool) *LinuxInterfacePr
 func NewPopulatedLinuxSeccomp(r randyOci, easy bool) *LinuxSeccomp {
 	this := &LinuxSeccomp{}
 	this.DefaultAction = string(randStringOci(r))
-	v51 := r.Intn(10)
-	this.Architectures = make([]string, v51)
-	for i := 0; i < v51; i++ {
+	v57 := r.Intn(10)
+	this.Architectures = make([]string, v57)
+	for i := 0; i < v57; i++ {
 		this.Architectures[i] = string(randStringOci(r))
 	}
-	v52 := r.Intn(10)
-	this.Flags = make([]string, v52)
-	for i := 0; i < v52; i++ {
+	v58 := r.Intn(10)
+	this.Flags = make([]string, v58)
+	for i := 0; i < v58; i++ {
 		this.Flags[i] = string(randStringOci(r))
 	}
 	if r.Intn(5) != 0 {
-		v53 := r.Intn(5)
-		this.Syscalls = make([]LinuxSyscall, v53)
-		for i := 0; i < v53; i++ {
-			v54 := NewPopulatedLinuxSyscall(r, easy)
-			this.Syscalls[i] = *v54
+		v59 := r.Intn(5)
+		this.Syscalls = make([]LinuxSyscall, v59)
+		for i := 0; i < v59; i++ {
+			v60 := NewPopulatedLinuxSyscall(r, easy)
+			this.Syscalls[i] = *v60
 		}
 	}
 	if !easy && r.Intn(10) != 0 {
@@ -5719,9 +5819,9 @@ func NewPopulatedLinuxSeccompArg(r randyOci, easy bool) *LinuxSeccompArg {
 
 func NewPopulatedLinuxSyscall(r randyOci, easy bool) *LinuxSyscall {
 	this := &LinuxSyscall{}
-	v55 := r.Intn(10)
-	this.Names = make([]string, v55)
-	for i := 0; i < v55; i++ {
+	v61 := r.Intn(10)
+	this.Names = make([]string, v61)
+	for i := 0; i < v61; i++ {
 		this.Names[i] = string(randStringOci(r))
 	}
 	this.Action = string(randStringOci(r))
@@ -5731,11 +5831,11 @@ func NewPopulatedLinuxSyscall(r randyOci, easy bool) *LinuxSyscall {
 		this.ErrnoRet = NewPopulatedLinuxSyscall_Errnoret(r, easy)
 	}
 	if r.Intn(5) != 0 {
-		v56 := r.Intn(5)
-		this.Args = make([]LinuxSeccompArg, v56)
-		for i := 0; i < v56; i++ {
-			v57 := NewPopulatedLinuxSeccompArg(r, easy)
-			this.Args[i] = *v57
+		v62 := r.Intn(5)
+		this.Args = make([]LinuxSeccompArg, v62)
+		for i := 0; i < v62; i++ {
+			v63 := NewPopulatedLinuxSeccompArg(r, easy)
+			this.Args[i] = *v63
 		}
 	}
 	if !easy && r.Intn(10) != 0 {
@@ -5777,9 +5877,9 @@ func randUTF8RuneOci(r randyOci) rune {
 	return rune(ru + 61)
 }
 func randStringOci(r randyOci) string {
-	v58 := r.Intn(100)
-	tmps := make([]rune, v58)
-	for i := 0; i < v58; i++ {
+	v64 := r.Intn(100)
+	tmps := make([]rune, v64)
+	for i := 0; i < v64; i++ {
 		tmps[i] = randUTF8RuneOci(r)
 	}
 	return string(tmps)
@@ -5801,11 +5901,11 @@ func randFieldOci(dAtA []byte, r randyOci, fieldNumber int, wire int) []byte {
 	switch wire {
 	case 0:
 		dAtA = encodeVarintPopulateOci(dAtA, uint64(key))
-		v59 := r.Int63()
+		v65 := r.Int63()
 		if r.Intn(2) == 0 {
-			v59 *= -1
+			v65 *= -1
 		}
-		dAtA = encodeVarintPopulateOci(dAtA, uint64(v59))
+		dAtA = encodeVarintPopulateOci(dAtA, uint64(v65))
 	case 1:
 		dAtA = encodeVarintPopulateOci(dAtA, uint64(key))
 		dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)))
@@ -6133,6 +6233,24 @@ func (m *Hooks) Size() (n int) {
 			n += 1 + l + sovOci(uint64(l))
 		}
 	}
+	if len(m.CreateRuntime) > 0 {
+		for _, e := range m.CreateRuntime {
+			l = e.Size()
+			n += 1 + l + sovOci(uint64(l))
+		}
+	}
+	if len(m.CreateContainer) > 0 {
+		for _, e := range m.CreateContainer {
+			l = e.Size()
+			n += 1 + l + sovOci(uint64(l))
+		}
+	}
+	if len(m.StartContainer) > 0 {
+		for _, e := range m.StartContainer {
+			l = e.Size()
+			n += 1 + l + sovOci(uint64(l))
+		}
+	}
 	if m.XXX_unrecognized != nil {
 		n += len(m.XXX_unrecognized)
 	}
@@ -6797,7 +6915,7 @@ func (this *Spec) String() string {
 	}
 	repeatedStringForMounts += "}"
 	keysForAnnotations := make([]string, 0, len(this.Annotations))
-	for k := range this.Annotations {
+	for k, _ := range this.Annotations {
 		keysForAnnotations = append(keysForAnnotations, k)
 	}
 	github_com_gogo_protobuf_sortkeys.Strings(keysForAnnotations)
@@ -6948,10 +7066,28 @@ func (this *Hooks) String() string {
 		repeatedStringForPoststop += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + ","
 	}
 	repeatedStringForPoststop += "}"
+	repeatedStringForCreateRuntime := "[]Hook{"
+	for _, f := range this.CreateRuntime {
+		repeatedStringForCreateRuntime += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + ","
+	}
+	repeatedStringForCreateRuntime += "}"
+	repeatedStringForCreateContainer := "[]Hook{"
+	for _, f := range this.CreateContainer {
+		repeatedStringForCreateContainer += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + ","
+	}
+	repeatedStringForCreateContainer += "}"
+	repeatedStringForStartContainer := "[]Hook{"
+	for _, f := range this.StartContainer {
+		repeatedStringForStartContainer += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + ","
+	}
+	repeatedStringForStartContainer += "}"
 	s := strings.Join([]string{`&Hooks{`,
 		`Prestart:` + repeatedStringForPrestart + `,`,
 		`Poststart:` + repeatedStringForPoststart + `,`,
 		`Poststop:` + repeatedStringForPoststop + `,`,
+		`CreateRuntime:` + repeatedStringForCreateRuntime + `,`,
+		`CreateContainer:` + repeatedStringForCreateContainer + `,`,
+		`StartContainer:` + repeatedStringForStartContainer + `,`,
 		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
 		`}`,
 	}, "")
@@ -6996,7 +7132,7 @@ func (this *Linux) String() string {
 	}
 	repeatedStringForDevices += "}"
 	keysForSysctl := make([]string, 0, len(this.Sysctl))
-	for k := range this.Sysctl {
+	for k, _ := range this.Sysctl {
 		keysForSysctl = append(keysForSysctl, k)
 	}
 	github_com_gogo_protobuf_sortkeys.Strings(keysForSysctl)
@@ -9298,6 +9434,108 @@ func (m *Hooks) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field CreateRuntime", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOci
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthOci
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthOci
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.CreateRuntime = append(m.CreateRuntime, Hook{})
+			if err := m.CreateRuntime[len(m.CreateRuntime)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 5:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field CreateContainer", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOci
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthOci
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthOci
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.CreateContainer = append(m.CreateContainer, Hook{})
+			if err := m.CreateContainer[len(m.CreateContainer)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 6:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field StartContainer", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOci
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthOci
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthOci
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.StartContainer = append(m.StartContainer, Hook{})
+			if err := m.StartContainer[len(m.StartContainer)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipOci(dAtA[iNdEx:])

src/runtime/virtcontainers/pkg/mock/mock.go

@@ -125,6 +125,10 @@ func (p *HybridVSockTTRPCMockImp) UpdateContainer(ctx context.Context, req *pb.U
 	return emptyResp, nil
 }
 
+func (p *HybridVSockTTRPCMockImp) UpdateEphemeralMounts(ctx context.Context, req *pb.UpdateEphemeralMountsRequest) (*gpb.Empty, error) {
+	return emptyResp, nil
+}
+
 func (p *HybridVSockTTRPCMockImp) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*gpb.Empty, error) {
 	return emptyResp, nil
 }

src/runtime/virtcontainers/qemu.go

@@ -1797,7 +1797,7 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op
 		return err
 	}
 
-	devID := device.ID
+	devID := *(*device).GetID()
 	machineType := q.HypervisorConfig().HypervisorMachineType
 
 	if op == AddDevice {
@@ -1814,29 +1814,31 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op
 		// for pc machine type instead of bridge. This is useful for devices that require
 		// a large PCI BAR which is a currently a limitation with PCI bridges.
 		if q.state.HotplugVFIOOnRootBus {
-
-			// In case MachineType is q35, a PCIe device is hotplugged on a PCIe Root Port.
-			switch machineType {
-			case QemuQ35:
-				if device.IsPCIe && q.state.PCIeRootPort <= 0 {
-					q.Logger().WithField("dev-id", device.ID).Warn("VFIO device is a PCIe device. It's recommended to add the PCIe Root Port by setting the pcie_root_port parameter in the configuration for q35")
-					device.Bus = ""
+			switch (*device).GetType() {
+			case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+				// In case MachineType is q35, a PCIe device is hotplugged on a PCIe Root Port.
+				pciDevice, ok := (*device).(config.VFIOPCIDev)
+				if !ok {
+					return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device)
 				}
-			default:
-				device.Bus = ""
-			}
+				switch machineType {
+				case QemuQ35:
+					if pciDevice.IsPCIe && q.state.PCIeRootPort <= 0 {
+						q.Logger().WithField("dev-id", (*device).GetID()).Warn("VFIO device is a PCIe device. It's recommended to add the PCIe Root Port by setting the pcie_root_port parameter in the configuration for q35")
+						pciDevice.Bus = ""
+					}
+				default:
+					pciDevice.Bus = ""
+				}
+				*device = pciDevice
 
-			switch device.Type {
-			case config.VFIODeviceNormalType:
-				err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, device.Bus, romFile)
-			case config.VFIODeviceMediatedType:
-				if utils.IsAPVFIOMediatedDevice(device.SysfsDev) {
-					err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev)
+				if pciDevice.Type == config.VFIOPCIDeviceNormalType {
+					err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, pciDevice.Bus, romFile)
 				} else {
-					err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", device.Bus, romFile)
+					err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), "", pciDevice.Bus, romFile)
 				}
-			default:
-				return fmt.Errorf("Incorrect VFIO device type found")
+			case config.VFIOAPDeviceMediatedType:
+				err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev())
 			}
 		} else {
 			addr, bridge, err := q.arch.addDeviceToBridge(ctx, devID, types.PCI)
@@ -1850,15 +1852,17 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op
 				}
 			}()
 
-			switch device.Type {
-			case config.VFIODeviceNormalType:
-				err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
-			case config.VFIODeviceMediatedType:
-				if utils.IsAPVFIOMediatedDevice(device.SysfsDev) {
-					err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev)
-				} else {
-					err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
+			switch (*device).GetType() {
+			case config.VFIOPCIDeviceNormalType:
+				pciDevice, ok := (*device).(config.VFIOPCIDev)
+				if !ok {
+					return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device)
 				}
+				err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, addr, bridge.ID, romFile)
+			case config.VFIOPCIDeviceMediatedType:
+				err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), addr, bridge.ID, romFile)
+			case config.VFIOAPDeviceMediatedType:
+				err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev())
 			default:
 				return fmt.Errorf("Incorrect VFIO device type found")
 			}
@@ -1866,13 +1870,24 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op
 		if err != nil {
 			return err
 		}
-		// XXX: Depending on whether we're doing root port or
-		// bridge hotplug, and how the bridge is set up in
-		// other parts of the code, we may or may not already
-		// have information about the slot number of the
-		// bridge and or the device.  For simplicity, just
-		// query both of them back from qemu
-		device.GuestPciPath, err = q.qomGetPciPath(devID)
+
+		switch (*device).GetType() {
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			pciDevice, ok := (*device).(config.VFIOPCIDev)
+			if !ok {
+				return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device)
+			}
+			// XXX: Depending on whether we're doing root port or
+			// bridge hotplug, and how the bridge is set up in
+			// other parts of the code, we may or may not already
+			// have information about the slot number of the
+			// bridge and or the device.  For simplicity, just
+			// query both of them back from qemu
+			guestPciPath, err := q.qomGetPciPath(devID)
+			pciDevice.GuestPciPath = guestPciPath
+			*device = pciDevice
+			return err
+		}
 		return err
 	} else {
 		q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device")

src/runtime/virtcontainers/qemu_arch_base.go

@@ -708,16 +708,17 @@ func (q *qemuArchBase) appendVhostUserDevice(ctx context.Context, devices []govm
 }
 
 func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDev config.VFIODev) []govmmQemu.Device {
-	if vfioDev.BDF == "" {
+	pciDevice := vfioDev.(config.VFIOPCIDev)
+	if pciDevice.BDF == "" {
 		return devices
 	}
 
 	devices = append(devices,
 		govmmQemu.VFIODevice{
-			BDF:      vfioDev.BDF,
-			VendorID: vfioDev.VendorID,
-			DeviceID: vfioDev.DeviceID,
-			Bus:      vfioDev.Bus,
+			BDF:      pciDevice.BDF,
+			VendorID: pciDevice.VendorID,
+			DeviceID: pciDevice.DeviceID,
+			Bus:      pciDevice.Bus,
 		},
 	)
 

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -463,7 +463,7 @@ func TestQemuArchBaseAppendVFIODevice(t *testing.T) {
 		},
 	}
 
-	vfDevice := config.VFIODev{
+	vfDevice := config.VFIOPCIDev{
 		BDF: bdf,
 	}
 
@@ -483,7 +483,7 @@ func TestQemuArchBaseAppendVFIODeviceWithVendorDeviceID(t *testing.T) {
 		},
 	}
 
-	vfDevice := config.VFIODev{
+	vfDevice := config.VFIOPCIDev{
 		BDF:      bdf,
 		VendorID: vendorID,
 		DeviceID: deviceID,

src/runtime/virtcontainers/sandbox.go

@@ -17,7 +17,9 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+
 	//"strconv"
+	"strings"
 	"sync"
 	"syscall"
 
@@ -48,6 +50,9 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
+
+	"google.golang.org/grpc/codes"
+	grpcStatus "google.golang.org/grpc/status"
 )
 
 // sandboxTracingTags defines tags for the trace span
@@ -1864,11 +1869,15 @@ func (s *Sandbox) HotplugAddDevice(ctx context.Context, device api.Device, devTy
 		// adding a group of VFIO devices
 		for _, dev := range vfioDevices {
 			if _, err := s.hypervisor.HotplugAddDevice(ctx, dev, VfioDev); err != nil {
+				bdf := ""
+				if pciDevice, ok := (*dev).(config.VFIOPCIDev); ok {
+					bdf = pciDevice.BDF
+				}
 				s.Logger().
 					WithFields(logrus.Fields{
 						"sandbox":         s.id,
-						"vfio-device-ID":  dev.ID,
-						"vfio-device-BDF": dev.BDF,
+						"vfio-device-ID":  (*dev).GetID(),
+						"vfio-device-BDF": bdf,
 					}).WithError(err).Error("failed to hotplug VFIO device")
 				return err
 			}
@@ -1917,11 +1926,15 @@ func (s *Sandbox) HotplugRemoveDevice(ctx context.Context, device api.Device, de
 		// remove a group of VFIO devices
 		for _, dev := range vfioDevices {
 			if _, err := s.hypervisor.HotplugRemoveDevice(ctx, dev, VfioDev); err != nil {
+				bdf := ""
+				if pciDevice, ok := (*dev).(config.VFIOPCIDev); ok {
+					bdf = pciDevice.BDF
+				}
 				s.Logger().WithError(err).
 					WithFields(logrus.Fields{
 						"sandbox":         s.id,
-						"vfio-device-ID":  dev.ID,
-						"vfio-device-BDF": dev.BDF,
+						"vfio-device-ID":  (*dev).GetID(),
+						"vfio-device-BDF": bdf,
 					}).Error("failed to hot unplug VFIO device")
 				return err
 			}
@@ -2121,11 +2134,69 @@ func (s *Sandbox) updateResources(ctx context.Context) error {
 		if newMemoryMB == finalMemoryMB {
 			break
 		}
+	}
 
+	tmpfsMounts, err := s.prepareEphemeralMounts(finalMemoryMB)
+	if err != nil {
+		return err
+	}
+	if err := s.agent.updateEphemeralMounts(ctx, tmpfsMounts); err != nil {
+		// upgrade path: if runtime is newer version, but agent is old
+		// then ignore errUnimplemented
+		if grpcStatus.Convert(err).Code() == codes.Unimplemented {
+			s.Logger().Warnf("agent does not support updateMounts")
+			return nil
+		}
+		return err
 	}
 
 	return nil
+}
 
+func (s *Sandbox) prepareEphemeralMounts(memoryMB uint32) ([]*grpc.Storage, error) {
+	tmpfsMounts := []*grpc.Storage{}
+	for _, c := range s.containers {
+		for _, mount := range c.mounts {
+			// if a tmpfs ephemeral mount is present
+			// update its size to occupy the entire sandbox's memory
+			if mount.Type == KataEphemeralDevType {
+				sizeLimited := false
+				for _, opt := range mount.Options {
+					if strings.HasPrefix(opt, "size") {
+						sizeLimited = true
+					}
+				}
+				if sizeLimited { // do not resize sizeLimited emptyDirs
+					continue
+				}
+
+				mountOptions := []string{"remount", fmt.Sprintf("size=%dM", memoryMB)}
+
+				origin_src := mount.Source
+				stat := syscall.Stat_t{}
+				err := syscall.Stat(origin_src, &stat)
+				if err != nil {
+					return nil, err
+				}
+
+				// if volume's gid isn't root group(default group), this means there's
+				// an specific fsGroup is set on this local volume, then it should pass
+				// to guest.
+				if stat.Gid != 0 {
+					mountOptions = append(mountOptions, fmt.Sprintf("%s=%d", fsGid, stat.Gid))
+				}
+
+				tmpfsMounts = append(tmpfsMounts, &grpc.Storage{
+					Driver:     KataEphemeralDevType,
+					MountPoint: filepath.Join(ephemeralPath(), filepath.Base(mount.Source)),
+					Source:     "tmpfs",
+					Fstype:     "tmpfs",
+					Options:    mountOptions,
+				})
+			}
+		}
+	}
+	return tmpfsMounts, nil
 }
 
 func (s *Sandbox) updateMemory(ctx context.Context, newMemoryMB uint32) error {

src/runtime/virtcontainers/sandbox_test.go

@@ -179,6 +179,65 @@ func TestCalculateSandboxMem(t *testing.T) {
 	}
 }
 
+func TestPrepareEphemeralMounts(t *testing.T) {
+	sandbox := &Sandbox{}
+	sandbox.containers = map[string]*Container{
+		"container1": {
+			mounts: []Mount{
+				{
+					// happy path
+					Type: KataEphemeralDevType,
+					Options: []string{
+						"rw",
+						"relatime",
+					},
+					Source: "/tmp",
+				},
+				{
+					// should be ignored because it is not KataEphemeralDevType
+					Type: KataLocalDevType,
+					Options: []string{
+						"rw",
+						"relatime",
+					},
+					Source: "/tmp",
+				},
+				{
+					// should be ignored because it is sizeLimited
+					Type: KataLocalDevType,
+					Options: []string{
+						"rw",
+						"relatime",
+						"size=1M",
+					},
+					Source: "/tmp",
+				},
+			},
+		},
+	}
+
+	storages, err := sandbox.prepareEphemeralMounts(1024)
+	assert.NoError(t, err)
+	assert.Equal(t, len(storages), 1)
+	for _, s := range storages {
+		assert.Equal(t, s.Driver, KataEphemeralDevType)
+		assert.Equal(t, s.Source, "tmpfs")
+		assert.Equal(t, s.Fstype, "tmpfs")
+		assert.Equal(t, s.MountPoint, filepath.Join(ephemeralPath(), "tmp"))
+		assert.Equal(t, len(s.Options), 2) // remount, size=1024M
+
+		validSet := map[string]struct{}{
+			"remount":    {},
+			"size=1024M": {},
+		}
+		for _, opt := range s.Options {
+			if _, ok := validSet[opt]; !ok {
+				t.Error("invalid mount opt: " + opt)
+			}
+		}
+	}
+}
+
 func TestCalculateSandboxMemHandlesNegativeLimits(t *testing.T) {
 	sandbox := &Sandbox{}
 	sandbox.config = &SandboxConfig{}

src/runtime/virtcontainers/utils/utils_linux.go

@@ -89,8 +89,7 @@ func FindContextID() (*os.File, uint64, error) {
 const (
 	procMountsFile = "/proc/mounts"
 
-	fieldsPerLine  = 6
-	vfioAPSysfsDir = "vfio_ap"
+	fieldsPerLine = 6
 )
 
 const (
@@ -142,18 +141,6 @@ func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string
 	}
 }
 
-// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device
-// by checking for the existence of "vfio_ap" in the path
-func IsAPVFIOMediatedDevice(sysfsdev string) bool {
-	split := strings.Split(sysfsdev, string(os.PathSeparator))
-	for _, el := range split {
-		if el == vfioAPSysfsDir {
-			return true
-		}
-	}
-	return false
-}
-
 func waitProcessUsingPidfd(pid int, timeoutSecs uint, logger *logrus.Entry) (bool, error) {
 	pidfd, err := unix.PidfdOpen(pid, 0)
 

src/runtime/virtcontainers/utils/utils_linux_test.go

@@ -63,19 +63,3 @@ func TestGetDevicePathAndFsTypeOptionsSuccessful(t *testing.T) {
 	assert.Equal(fstype, fstypeOut)
 	assert.Equal(fsOptions, optsOut)
 }
-
-func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) {
-	assert := assert.New(t)
-
-	// Should be false for a PCI device
-	isAPMdev := IsAPVFIOMediatedDevice("/sys/bus/pci/devices/0000:00:02.0/a297db4a-f4c2-11e6-90f6-d3b88d6c9525")
-	assert.False(isAPMdev)
-}
-
-func TestIsAPVFIOMediatedDeviceTrue(t *testing.T) {
-	assert := assert.New(t)
-
-	// Typical AP sysfsdev
-	isAPMdev := IsAPVFIOMediatedDevice("/sys/devices/vfio_ap/matrix/a297db4a-f4c2-11e6-90f6-d3b88d6c9525")
-	assert.True(isAPMdev)
-}

src/tools/agent-ctl/Cargo.lock

@@ -250,9 +250,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/tools/runk/Cargo.lock

@@ -233,9 +233,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/tools/runk/libcontainer/Cargo.toml

@@ -20,7 +20,7 @@ chrono = { version = "0.4.19", features = ["serde"] }
 serde = { version = "1.0.133", features = ["derive"] }
 serde_json = "1.0.74"
 scopeguard = "1.1.0"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 procfs = "0.14.0"
 
 [dev-dependencies]