diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index adb35d78c..097b022b7 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -641,9 +641,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", @@ -2079,6 +2079,7 @@ dependencies = [ "async-recursion", "async-trait", "capctl", + "cfg-if 1.0.0", "cgroups-rs", "clap", "futures", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 8aae69f42..0ecd2fb38 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -49,10 +49,11 @@ slog-scope = "4.1.2" slog-stdlog = "4.0.0" log = "0.4.11" +cfg-if = "1.0.0" prometheus = { version = "0.13.0", features = ["process"] } procfs = "0.12.0" anyhow = "1.0.32" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } # Tracing tracing = "0.1.26" diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index d4ba4e6f3..8c9c2230d 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -25,7 +25,7 @@ scan_fmt = "0.2.6" regex = "1.5.6" path-absolutize = "1.2.0" anyhow = "1.0.32" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } rlimit = "0.5.3" cfg-if = "0.1.0" diff --git a/src/agent/src/ap.rs b/src/agent/src/ap.rs new file mode 100644 index 000000000..202d33064 --- /dev/null +++ b/src/agent/src/ap.rs @@ -0,0 +1,79 @@ +// Copyright (c) IBM Corp. 2023 +// +// SPDX-License-Identifier: Apache-2.0 +// +use std::fmt; +use std::str::FromStr; + +use anyhow::{anyhow, Context}; + +// IBM Adjunct Processor (AP) is used for cryptographic operations +// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x). +// In Linux, virtual cryptographic devices are called AP queues. +// The name of an AP queue respects a format . in hexadecimal notation [1, p.467]: +// - is an adapter ID +// - is an adapter domain ID +// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf + +#[derive(Debug)] +pub struct Address { + pub adapter_id: u8, + pub adapter_domain: u16, +} + +impl Address { + pub fn new(adapter_id: u8, adapter_domain: u16) -> Address { + Address { + adapter_id, + adapter_domain, + } + } +} + +impl FromStr for Address { + type Err = anyhow::Error; + + fn from_str(s: &str) -> anyhow::Result { + let split: Vec<&str> = s.split('.').collect(); + if split.len() != 2 { + return Err(anyhow!( + "Wrong AP bus format. It needs to be in the form . (e.g. 0a.003f), got {:?}", + s + )); + } + + let adapter_id = u8::from_str_radix(split[0], 16).context(format!( + "Wrong AP bus format. AP ID needs to be in the form (e.g. 0a), got {:?}", + split[0] + ))?; + let adapter_domain = u16::from_str_radix(split[1], 16).context(format!( + "Wrong AP bus format. AP domain needs to be in the form (e.g. 003f), got {:?}", + split[1] + ))?; + + Ok(Address::new(adapter_id, adapter_domain)) + } +} + +impl fmt::Display for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_str() { + let device = Address::from_str("a.1").unwrap(); + assert_eq!(format!("{}", device), "0a.0001"); + + assert!(Address::from_str("").is_err()); + assert!(Address::from_str(".").is_err()); + assert!(Address::from_str("0.0.0").is_err()); + assert!(Address::from_str("0g.0000").is_err()); + assert!(Address::from_str("0a.10000").is_err()); + } +} diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 2dcef740a..dd8185bb6 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -16,13 +16,12 @@ use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; -#[cfg(target_arch = "s390x")] -use crate::ccw; use crate::linux_abi::*; use crate::pci; use crate::sandbox::Sandbox; use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher}; use anyhow::{anyhow, Context, Result}; +use cfg_if::cfg_if; use oci::{LinuxDeviceCgroup, LinuxResources, Spec}; use protocols::agent::Device; use tracing::instrument; @@ -46,14 +45,22 @@ pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm"; pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral"; pub const DRIVER_LOCAL_TYPE: &str = "local"; pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind"; -// VFIO device to be bound to a guest kernel driver -pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk"; -// VFIO device to be bound to vfio-pci and made available inside the +// VFIO PCI device to be bound to a guest kernel driver +pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk"; +// VFIO PCI device to be bound to vfio-pci and made available inside the // container as a VFIO device node -pub const DRIVER_VFIO_TYPE: &str = "vfio"; +pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci"; +pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap"; pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs"; pub const FS_TYPE_HUGETLB: &str = "hugetlbfs"; +cfg_if! { + if #[cfg(target_arch = "s390x")] { + use crate::ap; + use crate::ccw; + } +} + #[instrument] pub fn online_device(path: &str) -> Result<()> { fs::write(path, "1")?; @@ -280,7 +287,7 @@ pub async fn get_virtio_blk_ccw_device_name( sandbox: &Arc>, device: &ccw::Device, ) -> Result { - let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device); + let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device); let uev = wait_for_uevent(sandbox, matcher).await?; let devname = uev.devname; return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() { @@ -401,6 +408,39 @@ async fn get_vfio_device_name(sandbox: &Arc>, grp: IommuGroup) -> Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname)) } +#[cfg(target_arch = "s390x")] +#[derive(Debug)] +struct ApMatcher { + syspath: String, +} + +#[cfg(target_arch = "s390x")] +impl ApMatcher { + fn new(address: ap::Address) -> ApMatcher { + ApMatcher { + syspath: format!( + "{}/card{:02x}/{}", + AP_ROOT_BUS_PATH, address.adapter_id, address + ), + } + } +} + +#[cfg(target_arch = "s390x")] +impl UeventMatcher for ApMatcher { + fn is_match(&self, uev: &Uevent) -> bool { + uev.action == "add" && uev.devpath == self.syspath + } +} + +#[cfg(target_arch = "s390x")] +#[instrument] +async fn wait_for_ap_device(sandbox: &Arc>, address: ap::Address) -> Result<()> { + let matcher = ApMatcher::new(address); + wait_for_uevent(sandbox, matcher).await?; + Ok(()) +} + /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN) #[instrument] fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { @@ -701,7 +741,7 @@ async fn virtio_nvdimm_device_handler( Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into()) } -fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { +fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> { let mut tokens = opt.split('='); let hostbdf = tokens.next()?; let path = tokens.next()?; @@ -716,14 +756,18 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { // Each option should have the form "DDDD:BB:DD.F=" // DDDD:BB:DD.F is the device's PCI address in the host // is a PCI path to the device in the guest (see pci.rs) -async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> Result { - let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE; +#[instrument] +async fn vfio_pci_device_handler( + device: &Device, + sandbox: &Arc>, +) -> Result { + let vfio_in_guest = device.field_type != DRIVER_VFIO_PCI_GK_TYPE; let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new(); let mut group = None; for opt in device.options.iter() { - let (host, pcipath) = - split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?; + let (host, pcipath) = split_vfio_pci_option(opt) + .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?; let host = pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?; let pcipath = pci::Path::from_str(pcipath)?; @@ -765,6 +809,28 @@ async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> }) } +// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options +// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that +// amount to the hotplug timeout. +#[cfg(target_arch = "s390x")] +#[instrument] +async fn vfio_ap_device_handler( + device: &Device, + sandbox: &Arc>, +) -> Result { + // Force AP bus rescan + fs::write(AP_SCANS_PATH, "1")?; + for apqn in device.options.iter() { + wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?; + } + Ok(Default::default()) +} + +#[cfg(not(target_arch = "s390x"))] +async fn vfio_ap_device_handler(_: &Device, _: &Arc>) -> Result { + Err(anyhow!("AP is only supported on s390x")) +} + #[instrument] pub async fn add_devices( devices: &[Device], @@ -830,7 +896,10 @@ async fn add_device(device: &Device, sandbox: &Arc>) -> Result virtiommio_blk_device_handler(device, sandbox).await, DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await, DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await, - DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await, + DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => { + vfio_pci_device_handler(device, sandbox).await + } + DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await, _ => Err(anyhow!("Unknown device type {}", device.field_type)), } } @@ -1380,7 +1449,7 @@ mod tests { #[cfg(target_arch = "s390x")] #[tokio::test] async fn test_virtio_blk_ccw_matcher() { - let root_bus = create_ccw_root_bus_path(); + let root_bus = CCW_ROOT_BUS_PATH; let subsystem = "block"; let devname = "vda"; let relpath = "0.0.0002"; @@ -1489,13 +1558,13 @@ mod tests { } #[test] - fn test_split_vfio_option() { + fn test_split_vfio_pci_option() { assert_eq!( - split_vfio_option("0000:01:00.0=02/01"), + split_vfio_pci_option("0000:01:00.0=02/01"), Some(("0000:01:00.0", "02/01")) ); - assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None); - assert_eq!(split_vfio_option("0000:01:00.0"), None); + assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None); + assert_eq!(split_vfio_pci_option("0000:01:00.0"), None); } #[test] @@ -1574,4 +1643,35 @@ mod tests { // Test dev2 assert!(pci_iommu_group(&syspci, dev2).is_err()); } + + #[cfg(target_arch = "s390x")] + #[tokio::test] + async fn test_vfio_ap_matcher() { + let subsystem = "ap"; + let card = "0a"; + let relpath = format!("{}.0001", card); + + let mut uev = Uevent::default(); + uev.action = U_EVENT_ACTION_ADD.to_string(); + uev.subsystem = subsystem.to_string(); + uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath); + + let ap_address = ap::Address::from_str(&relpath).unwrap(); + let matcher = ApMatcher::new(ap_address); + + assert!(matcher.is_match(&uev)); + + let mut uev_remove = uev.clone(); + uev_remove.action = U_EVENT_ACTION_REMOVE.to_string(); + assert!(!matcher.is_match(&uev_remove)); + + let mut uev_other_device = uev.clone(); + uev_other_device.devpath = format!( + "{}/card{}/{}", + AP_ROOT_BUS_PATH, + card, + format!("{}.0002", card) + ); + assert!(!matcher.is_match(&uev_other_device)); + } } diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index c4d304d28..042acd0ae 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -3,6 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 // +use cfg_if::cfg_if; + /// Linux ABI related constants. #[cfg(target_arch = "aarch64")] @@ -64,10 +66,14 @@ pub fn create_pci_root_bus_path() -> String { ret } -#[cfg(target_arch = "s390x")] -pub fn create_ccw_root_bus_path() -> String { - String::from("/devices/css0") +cfg_if! { + if #[cfg(target_arch = "s390x")] { + pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0"; + pub const AP_ROOT_BUS_PATH: &str = "/devices/ap"; + pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans"; + } } + // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 3f960790d..5c4c7d576 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -20,6 +20,7 @@ extern crate scopeguard; extern crate slog; use anyhow::{anyhow, Context, Result}; +use cfg_if::cfg_if; use clap::{AppSettings, Parser}; use nix::fcntl::OFlag; use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr}; @@ -34,8 +35,6 @@ use std::process::exit; use std::sync::Arc; use tracing::{instrument, span}; -#[cfg(target_arch = "s390x")] -mod ccw; mod config; mod console; mod device; @@ -75,6 +74,13 @@ mod image_rpc; mod rpc; mod tracer; +cfg_if! { + if #[cfg(target_arch = "s390x")] { + mod ap; + mod ccw; + } +} + const NAME: &str = "kata-agent"; lazy_static! { diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 17ea3859c..bc13a6896 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -240,6 +240,70 @@ async fn ephemeral_storage_handler( Ok("".to_string()) } +// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them +// with mount options passed by the caller +#[instrument] +pub async fn update_ephemeral_mounts( + logger: Logger, + storages: Vec, + sandbox: Arc>, +) -> Result<()> { + for (_, storage) in storages.iter().enumerate() { + let handler_name = storage.driver.clone(); + let logger = logger.new(o!( + "msg" => "updating tmpfs storage", + "subsystem" => "storage", + "storage-type" => handler_name.to_owned())); + + match handler_name.as_str() { + DRIVER_EPHEMERAL_TYPE => { + fs::create_dir_all(Path::new(&storage.mount_point))?; + + if storage.options.is_empty() { + continue; + } else { + // assume that fsGid has already been set + let mut opts = Vec::<&str>::new(); + for (_, opt) in storage.options.iter().enumerate() { + if opt.starts_with(FS_GID) { + continue; + } + opts.push(opt) + } + let mount_path = Path::new(&storage.mount_point); + let src_path = Path::new(&storage.source); + + let (flags, options) = parse_mount_flags_and_options(opts); + + info!(logger, "mounting storage"; + "mount-source" => src_path.display(), + "mount-destination" => mount_path.display(), + "mount-fstype" => storage.fstype.as_str(), + "mount-options" => options.as_str(), + ); + + return baremount( + src_path, + mount_path, + storage.fstype.as_str(), + flags, + options.as_str(), + &logger, + ); + } + } + _ => { + return Err(anyhow!( + "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported", + storage.driver.to_owned() + )); + } + }; + } + + Ok(()) +} + #[instrument] async fn overlayfs_storage_handler( logger: &Logger, diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 7edd7218f..a30ecb3d8 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -55,7 +55,7 @@ use crate::device::{ use crate::image_rpc; use crate::linux_abi::*; use crate::metrics::get_metrics; -use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST}; +use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST}; use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS}; use crate::network::setup_guest_dns; use crate::pci; @@ -1115,6 +1115,23 @@ impl agent_ttrpc::AgentService for AgentService { }) } + async fn update_ephemeral_mounts( + &self, + ctx: &TtrpcContext, + req: protocols::agent::UpdateEphemeralMountsRequest, + ) -> ttrpc::Result { + trace_rpc_call!(ctx, "update_mounts", req); + is_allowed!(req); + + match update_ephemeral_mounts(sl!(), req.storages.to_vec(), self.sandbox.clone()).await { + Ok(_) => Ok(Empty::new()), + Err(e) => Err(ttrpc_error!( + ttrpc::Code::INTERNAL, + format!("Failed to update mounts: {:?}", e), + )), + } + } + async fn get_ip_tables( &self, ctx: &TtrpcContext, diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 0493a818a..66890d7f9 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -776,3 +776,587 @@ impl Default for BlockDeviceMgr { } } } + +#[cfg(test)] +mod tests { + use test_utils::skip_if_not_root; + use vmm_sys_util::tempfile::TempFile; + + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + #[test] + fn test_block_device_type() { + let dev_type = BlockDeviceType::get_type("spool:/device1"); + assert_eq!(dev_type, BlockDeviceType::Spool); + let dev_type = BlockDeviceType::get_type("/device1"); + assert_eq!(dev_type, BlockDeviceType::RawBlock); + } + + #[test] + fn test_create_block_devices_configs() { + let mgr = BlockDeviceMgr::default(); + assert!(!mgr.has_root_block_device()); + assert!(!mgr.has_part_uuid_root()); + assert!(!mgr.is_read_only_root()); + assert_eq!(mgr.get_index_of_drive_id(""), None); + assert_eq!(mgr.info_list.len(), 0); + } + + #[test] + fn test_add_non_root_block_device() { + skip_if_not_root!(); + let dummy_file = TempFile::new().unwrap(); + let dummy_path = dummy_file.as_path().to_owned(); + let dummy_id = String::from("1"); + let dummy_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path.clone(), + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: dummy_id.clone(), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!(BlockDeviceMgr::insert_device( + vm.device_manager_mut(), + ctx, + dummy_block_device.clone(), + ) + .is_ok()); + + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + assert!(!vm.device_manager().block_manager.has_root_block_device()); + assert!(!vm.device_manager().block_manager.has_part_uuid_root()); + assert!(!vm.device_manager().block_manager.is_read_only_root()); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + assert_eq!( + vm.device_manager().block_manager.info_list[0] + .config + .device_type(), + BlockDeviceType::RawBlock + ); + assert_eq!( + vm.device_manager().block_manager.info_list[0] + .config + .queue_sizes(), + [128u16] + ); + + let dev_config = vm.device_manager().block_manager.iter().next().unwrap(); + assert_eq!(dev_config.config, dummy_block_device); + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_path(&dummy_path) + .is_some()); + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_id(&dummy_id) + .is_some()); + } + + #[test] + fn test_update_blk_device_ratelimiters() { + skip_if_not_root!(); + //Init vm for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + ); + + let dummy_file = TempFile::new().unwrap(); + let dummy_path = dummy_file.as_path().to_owned(); + + let dummy_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: true, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device) + .unwrap(); + + let cfg = BlockDeviceConfigUpdateInfo { + drive_id: String::from("1"), + rate_limiter: None, + }; + + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + ); + + vm.device_manager_mut() + .block_manager + .attach_devices(&mut device_op_ctx) + .unwrap(); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + + //Patch while the epoll handler is invalid. + let expected_error = "could not send patch message to the block epoll handler".to_string(); + + assert_eq!( + BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg) + .unwrap_err() + .to_string(), + expected_error + ); + + //Invalid drive id + let cfg2 = BlockDeviceConfigUpdateInfo { + drive_id: String::from("2"), + rate_limiter: None, + }; + + let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id); + + assert_eq!( + BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2) + .unwrap_err() + .to_string(), + expected_error + ); + } + + #[test] + fn test_add_one_root_block_device() { + skip_if_not_root!(); + let dummy_file = TempFile::new().unwrap(); + let dummy_path = dummy_file.as_path().to_owned(); + let dummy_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: true, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!(BlockDeviceMgr::insert_device( + vm.device_manager_mut(), + ctx, + dummy_block_device.clone(), + ) + .is_ok()); + + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + assert!(vm.device_manager().block_manager.has_root_block); + assert!(!vm.device_manager().block_manager.has_part_uuid_root); + assert!(vm.device_manager().block_manager.read_only_root); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + + let dev_config = vm.device_manager().block_manager.iter().next().unwrap(); + assert_eq!(dev_config.config, dummy_block_device); + assert!(vm.device_manager().block_manager.is_read_only_root()); + } + + #[test] + fn test_add_two_root_block_devices_configs() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device_1 = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let root_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!( + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2) + .is_err() + ); + } + + #[test] + // Test BlockDevicesConfigs::add when you first add the root device and then the other devices. + fn test_add_root_block_device_first() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let dummy_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_3 = TempFile::new().unwrap(); + let dummy_path_3 = dummy_file_3.as_path().to_owned(); + let dummy_block_device_3 = BlockDeviceConfigInfo { + path_on_host: dummy_path_3, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("3"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + vm.device_manager_mut() + .block_manager + .create(root_block_device.clone()) + .unwrap(); + vm.device_manager_mut() + .block_manager + .create(dummy_block_device_2.clone()) + .unwrap(); + vm.device_manager_mut() + .block_manager + .create(dummy_block_device_3.clone()) + .unwrap(); + + assert!(vm.device_manager().block_manager.has_root_block_device(),); + assert!(!vm.device_manager().block_manager.has_part_uuid_root()); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 3); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap(); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap(); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap(); + } + + #[test] + // Test BlockDevicesConfigs::add when you add other devices first and then the root device. + fn test_root_block_device_add_last() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let dummy_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_3 = TempFile::new().unwrap(); + let dummy_path_3 = dummy_file_3.as_path().to_owned(); + let dummy_block_device_3 = BlockDeviceConfigInfo { + path_on_host: dummy_path_3, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("3"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + .unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone()) + .unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone()) + .unwrap(); + + assert!(vm.device_manager().block_manager.has_root_block_device(),); + assert!(!vm.device_manager().block_manager.has_part_uuid_root()); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 3); + + let mut block_dev_iter = vm.device_manager().block_manager.iter(); + // The root device should be first in the list no matter of the order in + // which the devices were added. + assert_eq!( + block_dev_iter.next().unwrap().config.drive_id, + root_block_device.drive_id + ); + assert_eq!( + block_dev_iter.next().unwrap().config.drive_id, + dummy_block_device_2.drive_id + ); + assert_eq!( + block_dev_iter.next().unwrap().config.drive_id, + dummy_block_device_3.drive_id + ); + } + + #[test] + fn test_block_device_update() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path_1.clone(), + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let mut dummy_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2.clone(), + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + + // Add 2 block devices. + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + .unwrap(); + + // Get index zero. + assert_eq!( + vm.device_manager() + .block_manager + .get_index_of_drive_id(&String::from("1")) + .unwrap(), + 0 + ); + + // Get None. + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_id(&String::from("foo")) + .is_none()); + + // Test several update cases using dummy_block_device_2. + // Validate `dummy_block_device_2` is already in the list + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_id(&dummy_block_device_2.drive_id) + .is_some()); + // Update OK. + dummy_block_device_2.is_read_only = true; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + .unwrap(); + + let index = vm + .device_manager() + .block_manager + .get_index_of_drive_id(&dummy_block_device_2.drive_id) + .unwrap(); + // Validate update was successful. + assert!( + vm.device_manager().block_manager.info_list[index] + .config + .is_read_only + ); + + // Update with invalid path. + let dummy_filename_3 = String::from("test_update_3"); + let dummy_path_3 = PathBuf::from(dummy_filename_3); + dummy_block_device_2.path_on_host = dummy_path_3; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!(BlockDeviceMgr::insert_device( + vm.device_manager_mut(), + ctx, + dummy_block_device_2.clone(), + ) + .is_err()); + + // Update with 2 root block devices. + dummy_block_device_2.path_on_host = dummy_path_2.clone(); + dummy_block_device_2.is_root_device = true; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!( + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,) + .is_err(), + ); + + // Switch roots and add a PARTUUID for the new one. + let root_block_device_old = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + let root_block_device_new = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: Some("0eaa91a0-01".to_string()), + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap(); + assert!(vm.device_manager().block_manager.has_part_uuid_root); + } +} diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index e0c1296a2..16bdfda6f 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -110,9 +110,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index efcde317e..20497bfa6 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -12,7 +12,7 @@ edition = "2018" [dependencies] byteorder = "1.4.3" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } chrono = "0.4.0" common-path = "=1.0.0" fail = "0.5.0" diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 875e1835a..98ec142f4 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -9,14 +9,16 @@ use crate::config::{ConfigOps, TomlConfig}; pub use vendor::AgentVendor; -use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT}; +use super::default::{ + DEFAULT_AGENT_DIAL_TIMEOUT_MS, DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT, +}; use crate::eother; /// agent name of Kata agent. pub const AGENT_NAME_KATA: &str = "kata"; /// Kata agent configuration information. -#[derive(Debug, Default, Deserialize, Serialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct Agent { /// If enabled, the agent will log additional debug messages to the system log. #[serde(default, rename = "enable_debug")] @@ -81,6 +83,24 @@ pub struct Agent { pub container_pipe_size: u32, } +impl std::default::Default for Agent { + fn default() -> Self { + Self { + debug: true, + enable_tracing: false, + debug_console_enabled: false, + server_port: DEFAULT_AGENT_VSOCK_PORT, + log_port: DEFAULT_AGENT_LOG_PORT, + dial_timeout_ms: DEFAULT_AGENT_DIAL_TIMEOUT_MS, + reconnect_timeout_ms: 3_000, + request_timeout_ms: 30_000, + health_check_request_timeout_ms: 90_000, + kernel_modules: Default::default(), + container_pipe_size: 0, + } + } +} + fn default_server_port() -> u32 { DEFAULT_AGENT_VSOCK_PORT } diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 7108c4add..1d5e2b4d3 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -24,6 +24,7 @@ pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024; pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025; pub const DEFAULT_AGENT_DBG_CONSOLE_PORT: u32 = 1026; pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA; +pub const DEFAULT_AGENT_DIAL_TIMEOUT_MS: u32 = 10; pub const DEFAULT_RUNTIME_NAME: &str = RUNTIME_NAME_VIRTCONTAINER; pub const DEFAULT_HYPERVISOR: &str = HYPERVISOR_NAME_DRAGONBALL; diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index 504cae83a..da4377f0d 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -34,6 +34,7 @@ service AgentService { rpc SignalProcess(SignalProcessRequest) returns (google.protobuf.Empty); rpc WaitProcess(WaitProcessRequest) returns (WaitProcessResponse); // wait & reap like waitpid(2) rpc UpdateContainer(UpdateContainerRequest) returns (google.protobuf.Empty); + rpc UpdateEphemeralMounts(UpdateEphemeralMountsRequest) returns (google.protobuf.Empty); rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse); rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty); rpc ResumeContainer(ResumeContainerRequest) returns (google.protobuf.Empty); @@ -316,6 +317,10 @@ message UpdateRoutesRequest { Routes routes = 1; } +message UpdateEphemeralMountsRequest { + repeated Storage storages = 1; +} + message ListInterfacesRequest { } diff --git a/src/libs/shim-interface/src/shim_mgmt/client.rs b/src/libs/shim-interface/src/shim_mgmt/client.rs index ace72c1d6..f112903c5 100644 --- a/src/libs/shim-interface/src/shim_mgmt/client.rs +++ b/src/libs/shim-interface/src/shim_mgmt/client.rs @@ -53,7 +53,7 @@ impl MgmtClient { .method(Method::GET) .uri(url) .body(Body::empty())?; - return self.send_request(req).await; + self.send_request(req).await } /// The HTTP Post method for client @@ -72,7 +72,7 @@ impl MgmtClient { .uri(url) .header("content-type", content_type) .body(body)?; - return self.send_request(req).await; + self.send_request(req).await } /// The http PUT method for client @@ -82,7 +82,7 @@ impl MgmtClient { .method(Method::PUT) .uri(url) .body(Body::from(data))?; - return self.send_request(req).await; + self.send_request(req).await } async fn send_request(&self, req: Request) -> Result> { diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index e38ae3510..caff85873 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -402,9 +402,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", @@ -2452,6 +2452,7 @@ dependencies = [ name = "runtimes" version = "0.1.0" dependencies = [ + "agent", "anyhow", "common", "hyper", @@ -2470,6 +2471,7 @@ dependencies = [ "slog", "slog-scope", "tokio", + "url", "virt_container", "wasm_container", ] diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 4c17fe36c..599949bb0 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -203,7 +203,7 @@ ifneq (,$(DBCMD)) DEFNETWORKMODEL_DB := tcfilter KERNELPARAMS = console=ttyS1 agent.log_vport=1025 KERNELTYPE_DB = uncompressed - KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_DB)) + KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME_DB,$(KERNELTYPE_DB)) KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB) DEFSANDBOXCGROUPONLY = true RUNTIMENAME := virt_container @@ -371,8 +371,8 @@ endef # Returns the name of the kernel file to use based on the provided KERNELTYPE. # $1 : KERNELTYPE (compressed or uncompressed) -define MAKE_KERNEL_NAME -$(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container) +define MAKE_KERNEL_NAME_DB +$(if $(findstring uncompressed,$1),vmlinux-dragonball-experimental.container,vmlinuz-dragonball-experimental.container) endef .DEFAULT_GOAL := default diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index b403f0f44..aa0df0857 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -115,5 +115,7 @@ impl_agent!( copy_file | crate::CopyFileRequest | crate::Empty | None, get_oom_event | crate::Empty | crate::OomEventResponse | Some(0), get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None, - set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None + set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None, + get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None, + resize_volume | crate::ResizeVolumeRequest | crate::Empty | None ); diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 4ac0c45ec..7d33a0992 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -8,7 +8,7 @@ use std::convert::Into; use protocols::{ agent::{self, OOMEvent}, - empty, health, types, + csi, empty, health, types, }; use crate::{ @@ -20,11 +20,12 @@ use crate::{ GetIPTablesResponse, GuestDetailsResponse, HealthCheckResponse, HugetlbStats, IPAddress, IPFamily, Interface, Interfaces, KernelModule, MemHotplugByProbeRequest, MemoryData, MemoryStats, NetworkStats, OnlineCPUMemRequest, PidsStats, ReadStreamRequest, - ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, - SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, - StatsContainerResponse, Storage, StringUser, ThrottlingData, TtyWinResizeRequest, - UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, - WaitProcessRequest, WriteStreamRequest, + ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, ResizeVolumeRequest, + Route, Routes, SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, + SignalProcessRequest, StatsContainerResponse, Storage, StringUser, ThrottlingData, + TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, + VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, + WriteStreamRequest, }, OomEventResponse, WaitProcessResponse, WriteStreamResponse, }; @@ -846,3 +847,35 @@ impl From for OomEventResponse { } } } + +impl From for agent::VolumeStatsRequest { + fn from(from: VolumeStatsRequest) -> Self { + Self { + volume_guest_path: from.volume_guest_path, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for VolumeStatsResponse { + fn from(from: csi::VolumeStatsResponse) -> Self { + let result: String = format!( + "Usage: {:?} Volume Condition: {:?}", + from.get_usage(), + from.get_volume_condition() + ); + Self { data: result } + } +} + +impl From for agent::ResizeVolumeRequest { + fn from(from: ResizeVolumeRequest) -> Self { + Self { + volume_guest_path: from.volume_guest_path, + size: from.size, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index a3d1da72a..ea3bab78f 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -23,8 +23,8 @@ pub use types::{ ReseedRandomDevRequest, ResizeVolumeRequest, Route, Routes, SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, StatsContainerResponse, Storage, TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, - VersionCheckResponse, WaitProcessRequest, WaitProcessResponse, WriteStreamRequest, - WriteStreamResponse, + VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, + WaitProcessResponse, WriteStreamRequest, WriteStreamResponse, }; use anyhow::Result; @@ -88,4 +88,6 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { async fn get_oom_event(&self, req: Empty) -> Result; async fn get_ip_tables(&self, req: GetIPTablesRequest) -> Result; async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result; + async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result; + async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result; } diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index 0cd509ff5..de8822980 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -9,6 +9,8 @@ use std::convert::TryFrom; use serde::{Deserialize, Serialize}; +pub const DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT: u32 = 10; + #[derive(PartialEq, Clone, Default)] pub struct Empty {} @@ -164,7 +166,7 @@ impl ContainerProcessID { } } -#[derive(PartialEq, Clone, Debug, Default)] +#[derive(PartialEq, Clone, Debug)] pub struct RemoveContainerRequest { pub container_id: String, pub timeout: u32, @@ -179,6 +181,15 @@ impl RemoveContainerRequest { } } +impl std::default::Default for RemoveContainerRequest { + fn default() -> Self { + Self { + container_id: "".to_string(), + timeout: DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT, + } + } +} + #[derive(PartialEq, Clone, Default)] pub struct SignalProcessRequest { pub process_id: ContainerProcessID, @@ -568,6 +579,16 @@ pub struct ResizeVolumeRequest { pub size: u64, } +#[derive(PartialEq, Clone, Default, Debug)] +pub struct VolumeStatsRequest { + pub volume_guest_path: String, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct VolumeStatsResponse { + pub data: String, +} + #[cfg(test)] mod test { use std::convert::TryFrom; diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 9cd05de91..3a2631c35 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -473,7 +473,20 @@ impl CloudHypervisorInner { } pub(crate) async fn get_vmm_master_tid(&self) -> Result { - todo!() + if let Some(pid) = self.pid { + Ok(pid) + } else { + Err(anyhow!("could not get vmm master tid")) + } + } + + pub(crate) async fn get_ns_path(&self) -> Result { + if let Some(pid) = self.pid { + let ns_path = format!("/proc/{}/ns", pid); + Ok(ns_path) + } else { + Err(anyhow!("could not get ns path")) + } } pub(crate) async fn check(&self) -> Result<()> { diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 7805d2601..f8f44710e 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -123,6 +123,11 @@ impl Hypervisor for CloudHypervisor { inner.get_vmm_master_tid().await } + async fn get_ns_path(&self) -> Result { + let inner = self.inner.read().await; + inner.get_ns_path().await + } + async fn check(&self) -> Result<()> { let inner = self.inner.read().await; inner.check().await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs index fc3bea7a5..adc016807 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs @@ -132,6 +132,11 @@ impl DragonballInner { Ok(master_tid) } + pub(crate) async fn get_ns_path(&self) -> Result { + let ns_path = self.vmm_instance.get_ns_path(); + Ok(ns_path) + } + pub(crate) async fn check(&self) -> Result<()> { Ok(()) } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index d096abf54..951af2bc7 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -122,6 +122,11 @@ impl Hypervisor for Dragonball { inner.get_vmm_master_tid().await } + async fn get_ns_path(&self) -> Result { + let inner = self.inner.read().await; + inner.get_ns_path().await + } + async fn check(&self) -> Result<()> { let inner = self.inner.read().await; inner.check().await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 8e7d6332f..4c40b40c5 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -81,6 +81,13 @@ impl VmmInstance { result } + pub fn get_ns_path(&self) -> String { + let info_binding = self.vmm_shared_info.clone(); + let info = info_binding.read().unwrap(); + let result = format!("/proc/{}/task/{}/ns", info.pid, info.master_tid); + result + } + pub fn get_vcpu_tids(&self) -> Vec<(u8, u32)> { let info = self.vmm_shared_info.clone(); let result = info.read().unwrap().tids.clone(); diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index e7f18e513..364cce0f2 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -88,6 +88,7 @@ pub trait Hypervisor: Send + Sync { async fn get_thread_ids(&self) -> Result; async fn get_pids(&self) -> Result>; async fn get_vmm_master_tid(&self) -> Result; + async fn get_ns_path(&self) -> Result; async fn cleanup(&self) -> Result<()>; async fn check(&self) -> Result<()>; async fn get_jailer_root(&self) -> Result; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 995a9c590..0c0efbde6 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -94,6 +94,11 @@ impl QemuInner { todo!() } + pub(crate) async fn get_ns_path(&self) -> Result { + info!(sl!(), "QemuInner::get_ns_path()"); + todo!() + } + pub(crate) async fn cleanup(&self) -> Result<()> { info!(sl!(), "QemuInner::cleanup()"); todo!() diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 0192e2a8e..c73fb23d4 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -108,6 +108,11 @@ impl Hypervisor for Qemu { inner.get_vmm_master_tid().await } + async fn get_ns_path(&self) -> Result { + let inner = self.inner.read().await; + inner.get_ns_path().await + } + async fn cleanup(&self) -> Result<()> { let inner = self.inner.read().await; inner.cleanup().await diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 7ae1f799f..73b577c5a 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -14,7 +14,7 @@ anyhow = "^1.0" async-trait = "0.1.48" bitflags = "1.2.1" byte-unit = "4.0.14" -cgroups-rs = "0.3.1" +cgroups-rs = "0.3.2" futures = "0.3.11" lazy_static = "1.4.0" libc = ">=0.2.39" diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml index 142c44ed0..3a6ab0a1b 100644 --- a/src/runtime-rs/crates/runtimes/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/Cargo.toml @@ -15,7 +15,9 @@ hyper = { version = "0.14.20", features = ["stream", "server", "http1"] } hyperlocal = "0.8" serde_json = "1.0.88" nix = "0.25.0" +url = "2.3.1" +agent = { path = "../agent" } common = { path = "./common" } kata-types = { path = "../../../libs/kata-types" } kata-sys-util = { path = "../../../libs/kata-sys-util" } diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index 3fee8165d..0aee04922 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -26,4 +26,6 @@ pub trait Sandbox: Send + Sync { // utils async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result>; async fn get_iptables(&self, is_ipv6: bool) -> Result>; + async fn direct_volume_stats(&self, volume_path: &str) -> Result; + async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()>; } diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 84c1fa4a9..d8aad3a0a 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -353,9 +353,10 @@ impl RuntimeHandlerManager { /// Config override ordering(high to low): /// 1. podsandbox annotation -/// 2. shimv2 create task option -/// TODO: https://github.com/kata-containers/kata-containers/issues/3961 -/// 3. environment +/// 2. environment variable +/// 3. shimv2 create task option +/// 4. If above three are not set, then get default path from DEFAULT_RUNTIME_CONFIGURATIONS +/// in kata-containers/src/libs/kata-types/src/config/default.rs, in array order. fn load_config(spec: &oci::Spec, option: &Option>) -> Result { const KATA_CONF_FILE: &str = "KATA_CONF_FILE"; let annotation = Annotation::new(spec.annotations.clone()); diff --git a/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs b/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs index dea9725a5..b0c79cd40 100644 --- a/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs +++ b/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs @@ -7,12 +7,17 @@ // This defines the handlers corresponding to the url when a request is sent to destined url, // the handler function should be invoked, and the corresponding data will be in the response -use anyhow::{anyhow, Result}; +use agent::ResizeVolumeRequest; +use anyhow::{anyhow, Context, Result}; use common::Sandbox; use hyper::{Body, Method, Request, Response, StatusCode}; use std::sync::Arc; +use url::Url; -use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL}; +use shim_interface::shim_mgmt::{ + AGENT_URL, DIRECT_VOLUME_PATH_KEY, DIRECT_VOLUME_RESIZE_URL, DIRECT_VOLUME_STATS_URL, + IP6_TABLE_URL, IP_TABLE_URL, +}; // main router for response, this works as a multiplexer on // http arrival which invokes the corresponding handler function @@ -34,6 +39,10 @@ pub(crate) async fn handler_mux( (&Method::PUT, IP6_TABLE_URL) | (&Method::GET, IP6_TABLE_URL) => { ipv6_table_handler(sandbox, req).await } + (&Method::POST, DIRECT_VOLUME_STATS_URL) => direct_volume_stats_handler(sandbox, req).await, + (&Method::POST, DIRECT_VOLUME_RESIZE_URL) => { + direct_volume_resize_handler(sandbox, req).await + } _ => Ok(not_found(req).await), } } @@ -101,3 +110,39 @@ async fn generic_ip_table_handler( _ => Err(anyhow!("IP Tables only takes PUT and GET")), } } + +async fn direct_volume_stats_handler( + sandbox: Arc, + req: Request, +) -> Result> { + let params = Url::parse(&req.uri().to_string()) + .map_err(|e| anyhow!(e))? + .query_pairs() + .into_owned() + .collect::>(); + let volume_path = params + .get(DIRECT_VOLUME_PATH_KEY) + .context("shim-mgmt: volume path key not found in request params")?; + let result = sandbox.direct_volume_stats(volume_path).await; + match result { + Ok(stats) => Ok(Response::new(Body::from(stats))), + _ => Err(anyhow!("handler: Failed to get volume stats")), + } +} + +async fn direct_volume_resize_handler( + sandbox: Arc, + req: Request, +) -> Result> { + let body = hyper::body::to_bytes(req.into_body()).await?; + + // unserialize json body into resizeRequest struct + let resize_req: ResizeVolumeRequest = + serde_json::from_slice(&body).context("shim-mgmt: deserialize resizeRequest failed")?; + let result = sandbox.direct_volume_resize(resize_req).await; + + match result { + Ok(_) => Ok(Response::new(Body::from(""))), + _ => Err(anyhow!("handler: Failed to resize volume")), + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 501002e42..2f9f03b0e 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -344,20 +344,33 @@ impl Container { height: u32, ) -> Result<()> { let logger = logger_with_process(process); - let inner = self.inner.read().await; + let mut inner = self.inner.write().await; if inner.init_process.get_status().await != ProcessStatus::Running { warn!(logger, "container is not running"); return Ok(()); } - self.agent - .tty_win_resize(agent::TtyWinResizeRequest { - process_id: process.clone().into(), - row: height, - column: width, - }) - .await - .context("resize pty")?; - Ok(()) + + if process.exec_id.is_empty() { + inner.init_process.height = height; + inner.init_process.width = width; + } else if let Some(exec) = inner.exec_processes.get_mut(&process.exec_id) { + exec.process.height = height; + exec.process.width = width; + + // for some case, resize_pty request should be handled while the process has not been started in agent + // just return here, and truly resize_pty will happen in start_process + if exec.process.get_status().await != ProcessStatus::Running { + return Ok(()); + } + } else { + return Err(anyhow!( + "could not find process {} in container {}", + process.exec_id(), + process.container_id() + )); + } + + inner.win_resize_process(process, height, width).await } pub async fn stats(&self) -> Result> { diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index b041af076..bb6c2ed07 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -164,9 +164,12 @@ impl ContainerInner { let exit_status = self.get_exit_status().await; let _locked_exit_status = exit_status.read().await; info!(self.logger, "container terminated"); - let timeout: u32 = 10; + let remove_request = agent::RemoveContainerRequest { + container_id: cid.to_string(), + ..Default::default() + }; self.agent - .remove_container(agent::RemoveContainerRequest::new(cid, timeout)) + .remove_container(remove_request) .await .or_else(|e| { if force { diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index 53bd36894..f85e7901a 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -73,7 +73,8 @@ impl ContainerManager for VirtContainerManager { // * should be run after the vm is started, before container is created, and after CreateRuntime Hooks // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createcontainer-hooks let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?; - let vmm_netns_path = format!("/proc/{}/task/{}/ns/{}", self.pid, vmm_master_tid, "net"); + let vmm_ns_path = self.hypervisor.get_ns_path().await?; + let vmm_netns_path = format!("{}/{}", vmm_ns_path, "net"); let state = oci::State { version: spec.version.clone(), id: config.container_id.clone(), diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 0d6e4765e..f996c5747 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use agent::{ self, kata::KataAgent, types::KernelModule, Agent, GetIPTablesRequest, SetIPTablesRequest, + VolumeStatsRequest, }; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -18,10 +19,7 @@ use common::{ use containerd_shim_protos::events::task::TaskOOM; use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL}; use kata_sys_util::hooks::HookStates; -use kata_types::config::{ - default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT}, - TomlConfig, -}; +use kata_types::config::TomlConfig; use resource::{ manager::ManagerArgs, network::{NetworkConfig, NetworkWithNetNsConfig}, @@ -218,7 +216,7 @@ impl Sandbox for VirtSandbox { let agent_config = self.agent.agent_config().await; let kernel_modules = KernelModule::set_kernel_modules(agent_config.kernel_modules)?; let req = agent::CreateSandboxRequest { - hostname: "".to_string(), + hostname: spec.hostname.clone(), dns, storages: self .resource_manager @@ -329,6 +327,26 @@ impl Sandbox for VirtSandbox { self.agent.agent_sock().await } + async fn direct_volume_stats(&self, volume_guest_path: &str) -> Result { + let req: agent::VolumeStatsRequest = VolumeStatsRequest { + volume_guest_path: volume_guest_path.to_string(), + }; + let result = self + .agent + .get_volume_stats(req) + .await + .context("sandbox: failed to process direct volume stats query")?; + Ok(result.data) + } + + async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()> { + self.agent + .resize_volume(resize_req) + .await + .context("sandbox: failed to resize direct-volume")?; + Ok(()) + } + async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result> { info!(sl!(), "sb: set_iptables invoked"); let req = SetIPTablesRequest { is_ipv6, data }; @@ -380,19 +398,7 @@ impl Persist for VirtSandbox { HYPERVISOR_DRAGONBALL => Ok(Arc::new(Dragonball::restore((), h).await?)), _ => Err(anyhow!("Unsupported hypervisor {}", &h.hypervisor_type)), }?; - let agent = Arc::new(KataAgent::new(kata_types::config::Agent { - debug: true, - enable_tracing: false, - server_port: DEFAULT_AGENT_VSOCK_PORT, - log_port: DEFAULT_AGENT_LOG_PORT, - dial_timeout_ms: 10, - reconnect_timeout_ms: 3_000, - request_timeout_ms: 30_000, - health_check_request_timeout_ms: 90_000, - kernel_modules: Default::default(), - container_pipe_size: 0, - debug_console_enabled: false, - })); + let agent = Arc::new(KataAgent::new(kata_types::config::Agent::default())); let sid = sandbox_args.sid; let args = ManagerArgs { sid: sid.clone(), diff --git a/src/runtime-rs/crates/shim/src/args.rs b/src/runtime-rs/crates/shim/src/args.rs index e759c458c..1ab5b8afa 100644 --- a/src/runtime-rs/crates/shim/src/args.rs +++ b/src/runtime-rs/crates/shim/src/args.rs @@ -117,12 +117,11 @@ mod tests { result: Result<()>, } - let default_id = "1dfc0567".to_string(); - let default_namespace = "ns1".to_string(); + let default_id = "default_id".to_string(); + let default_namespace = "default_namespace".to_string(); let default_address = bind_address.to_string(); let default_publish_binary = "containerd".to_string(); let default_bundle = path.to_string(); - let default_debug = false; let mut arg = Args { id: default_id.clone(), @@ -130,7 +129,7 @@ mod tests { address: default_address.clone(), publish_binary: default_publish_binary.clone(), bundle: default_bundle.clone(), - debug: default_debug, + ..Default::default() }; let tests = &[ diff --git a/src/runtime-rs/crates/shim/src/shim.rs b/src/runtime-rs/crates/shim/src/shim.rs index 71941aee8..acd76d6bf 100644 --- a/src/runtime-rs/crates/shim/src/shim.rs +++ b/src/runtime-rs/crates/shim/src/shim.rs @@ -93,12 +93,12 @@ mod tests { std::env::set_current_dir(bundle_path).unwrap(); let args = Args { - id: "1dfc0567".to_string(), - namespace: "test_namespace".into(), - address: "containerd_socket".into(), + id: "default_id".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let executor = ShimExecutor::new(args); diff --git a/src/runtime-rs/crates/shim/src/shim_start.rs b/src/runtime-rs/crates/shim/src/shim_start.rs index 67fa8c5be..06a7f3ae5 100644 --- a/src/runtime-rs/crates/shim/src/shim_start.rs +++ b/src/runtime-rs/crates/shim/src/shim_start.rs @@ -153,12 +153,12 @@ mod tests { std::env::set_current_dir(bundle_path).unwrap(); let args = Args { - id: "sandbox1".into(), - namespace: "ns".into(), - address: "address".into(), + id: "default_id".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let mut executor = ShimExecutor::new(args); @@ -185,11 +185,11 @@ mod tests { let args = Args { id: sandbox_id.to_owned(), - namespace: "ns1".into(), - address: "containerd_socket".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let executor = ShimExecutor::new(args); @@ -204,11 +204,11 @@ mod tests { let args = Args { id: container_id, - namespace: "ns1".into(), - address: "containerd_socket".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path2.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let executor2 = ShimExecutor::new(args); diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index 35af6afec..dee6291ed 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -258,15 +258,24 @@ const ( // VFIODeviceErrorType is the error type of VFIO device VFIODeviceErrorType VFIODeviceType = iota - // VFIODeviceNormalType is a normal VFIO device type - VFIODeviceNormalType + // VFIOPCIDeviceNormalType is a normal VFIO PCI device type + VFIOPCIDeviceNormalType - // VFIODeviceMediatedType is a VFIO mediated device type - VFIODeviceMediatedType + // VFIOPCIDeviceMediatedType is a VFIO PCI mediated device type + VFIOPCIDeviceMediatedType + + // VFIOAPDeviceMediatedType is a VFIO AP mediated device type + VFIOAPDeviceMediatedType ) -// VFIODev represents a VFIO drive used for hotplugging -type VFIODev struct { +type VFIODev interface { + GetID() *string + GetType() VFIODeviceType + GetSysfsDev() *string +} + +// VFIOPCIDev represents a VFIO PCI device used for hotplugging +type VFIOPCIDev struct { // ID is used to identify this drive in the hypervisor options. ID string @@ -298,6 +307,44 @@ type VFIODev struct { IsPCIe bool } +func (d VFIOPCIDev) GetID() *string { + return &d.ID +} + +func (d VFIOPCIDev) GetType() VFIODeviceType { + return d.Type +} + +func (d VFIOPCIDev) GetSysfsDev() *string { + return &d.SysfsDev +} + +type VFIOAPDev struct { + // ID is used to identify this drive in the hypervisor options. + ID string + + // sysfsdev of VFIO mediated device + SysfsDev string + + // APDevices are the Adjunct Processor devices assigned to the mdev + APDevices []string + + // Type of VFIO device + Type VFIODeviceType +} + +func (d VFIOAPDev) GetID() *string { + return &d.ID +} + +func (d VFIOAPDev) GetType() VFIODeviceType { + return d.Type +} + +func (d VFIOAPDev) GetSysfsDev() *string { + return &d.SysfsDev +} + // RNGDev represents a random number generator device type RNGDev struct { // ID is used to identify the device in the hypervisor options. diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index 25f021eda..79e00adbd 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -89,18 +89,47 @@ func readPCIProperty(propertyPath string) (string, error) { return strings.Split(string(buf), "\n")[0], nil } -func GetVFIODeviceType(deviceFileName string) config.VFIODeviceType { +func GetVFIODeviceType(deviceFilePath string) (config.VFIODeviceType, error) { + deviceFileName := filepath.Base(deviceFilePath) + //For example, 0000:04:00.0 tokens := strings.Split(deviceFileName, ":") - vfioDeviceType := config.VFIODeviceErrorType if len(tokens) == 3 { - vfioDeviceType = config.VFIODeviceNormalType - } else { - //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 - tokens = strings.Split(deviceFileName, "-") - if len(tokens) == 5 { - vfioDeviceType = config.VFIODeviceMediatedType - } + return config.VFIOPCIDeviceNormalType, nil } - return vfioDeviceType + + //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 + tokens = strings.Split(deviceFileName, "-") + if len(tokens) != 5 { + return config.VFIODeviceErrorType, fmt.Errorf("Incorrect tokens found while parsing VFIO details: %s", deviceFileName) + } + + deviceSysfsDev, err := GetSysfsDev(deviceFilePath) + if err != nil { + return config.VFIODeviceErrorType, err + } + + if strings.HasPrefix(deviceSysfsDev, vfioAPSysfsDir) { + return config.VFIOAPDeviceMediatedType, nil + } + + return config.VFIOPCIDeviceMediatedType, nil +} + +// GetSysfsDev returns the sysfsdev of mediated device +// Expected input string format is absolute path to the sysfs dev node +// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 +func GetSysfsDev(sysfsDevStr string) (string, error) { + return filepath.EvalSymlinks(sysfsDevStr) +} + +// GetAPVFIODevices retrieves all APQNs associated with a mediated VFIO-AP +// device +func GetAPVFIODevices(sysfsdev string) ([]string, error) { + data, err := os.ReadFile(filepath.Join(sysfsdev, "matrix")) + if err != nil { + return []string{}, err + } + // Split by newlines, omitting final newline + return strings.Split(string(data[:len(data)-1]), "\n"), nil } diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 58658b0b8..94139aaa2 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -30,6 +30,7 @@ const ( iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group" vfioDevPath = "/dev/vfio/%s" pcieRootPortPrefix = "rp" + vfioAPSysfsDir = "/sys/devices/vfio_ap" ) var ( @@ -85,19 +86,42 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece if err != nil { return err } - vfio := &config.VFIODev{ - ID: utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize), - Type: vfioDeviceType, - BDF: deviceBDF, - SysfsDev: deviceSysfsDev, - IsPCIe: isPCIeDevice(deviceBDF), - Class: getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass), - } - device.VfioDevs = append(device.VfioDevs, vfio) - if vfio.IsPCIe { - vfio.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) - AllPCIeDevs[vfio.BDF] = true + id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize) + + var vfio config.VFIODev + + switch vfioDeviceType { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + isPCIe := isPCIeDevice(deviceBDF) + // Do not directly assign to `vfio` -- need to access field still + vfioPCI := config.VFIOPCIDev{ + ID: id, + Type: vfioDeviceType, + BDF: deviceBDF, + SysfsDev: deviceSysfsDev, + IsPCIe: isPCIe, + Class: getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass), + } + if isPCIe { + vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) + AllPCIeDevs[deviceBDF] = true + } + vfio = vfioPCI + case config.VFIOAPDeviceMediatedType: + devices, err := GetAPVFIODevices(deviceSysfsDev) + if err != nil { + return err + } + vfio = config.VFIOAPDev{ + ID: id, + SysfsDev: deviceSysfsDev, + Type: config.VFIOAPDeviceMediatedType, + APDevices: devices, + } + default: + return fmt.Errorf("Failed to append device: VFIO device type unrecognized") } + device.VfioDevs = append(device.VfioDevs, &vfio) } coldPlug := device.DeviceInfo.ColdPlug @@ -192,31 +216,60 @@ func (device *VFIODevice) Load(ds config.DeviceState) { device.GenericDevice.Load(ds) for _, dev := range ds.VFIODevs { - device.VfioDevs = append(device.VfioDevs, &config.VFIODev{ - ID: dev.ID, - Type: config.VFIODeviceType(dev.Type), - BDF: dev.BDF, - SysfsDev: dev.SysfsDev, - }) + var vfio config.VFIODev + + vfioDeviceType := (*device.VfioDevs[0]).GetType() + switch vfioDeviceType { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + bdf := "" + if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDev.BDF + } + vfio = config.VFIOPCIDev{ + ID: *(*dev).GetID(), + Type: config.VFIODeviceType((*dev).GetType()), + BDF: bdf, + SysfsDev: *(*dev).GetSysfsDev(), + } + case config.VFIOAPDeviceMediatedType: + vfio = config.VFIOAPDev{ + ID: *(*dev).GetID(), + SysfsDev: *(*dev).GetSysfsDev(), + } + default: + deviceLogger().WithError( + fmt.Errorf("VFIO device type unrecognized"), + ).Error("Failed to append device") + return + } + + device.VfioDevs = append(device.VfioDevs, &vfio) } } // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) { - vfioDeviceType = GetVFIODeviceType(deviceFileName) + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + vfioDeviceType, err = GetVFIODeviceType(sysfsDevStr) + if err != nil { + return deviceBDF, deviceSysfsDev, vfioDeviceType, err + } switch vfioDeviceType { - case config.VFIODeviceNormalType: + case config.VFIOPCIDeviceNormalType: // Get bdf of device eg. 0000:00:1c.0 deviceBDF = getBDF(deviceFileName) // Get sysfs path used by cloud-hypervisor deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName) - case config.VFIODeviceMediatedType: + case config.VFIOPCIDeviceMediatedType: // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) - deviceSysfsDev, err = getSysfsDev(sysfsDevStr) + deviceSysfsDev, err = GetSysfsDev(sysfsDevStr) deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev)) + case config.VFIOAPDeviceMediatedType: + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + deviceSysfsDev, err = GetSysfsDev(sysfsDevStr) default: err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName) } @@ -244,13 +297,6 @@ func getBDF(deviceSysStr string) string { return tokens[1] } -// getSysfsDev returns the sysfsdev of mediated device -// Expected input string format is absolute path to the sysfs dev node -// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 -func getSysfsDev(sysfsDevStr string) (string, error) { - return filepath.EvalSymlinks(sysfsDevStr) -} - // BindDevicetoVFIO binds the device to vfio driver after unbinding from host. // Will be called by a network interface or a generic pcie device. func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) { diff --git a/src/runtime/pkg/device/drivers/vfio_test.go b/src/runtime/pkg/device/drivers/vfio_test.go index 3c25a64c3..6a1ab61eb 100644 --- a/src/runtime/pkg/device/drivers/vfio_test.go +++ b/src/runtime/pkg/device/drivers/vfio_test.go @@ -32,9 +32,9 @@ func TestGetVFIODetails(t *testing.T) { deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "") switch vfioDeviceType { - case config.VFIODeviceNormalType: + case config.VFIOPCIDeviceNormalType: assert.Equal(t, d.expectedStr, deviceBDF) - case config.VFIODeviceMediatedType: + case config.VFIOPCIDeviceMediatedType, config.VFIOAPDeviceMediatedType: assert.Equal(t, d.expectedStr, deviceSysfsDev) default: assert.NotNil(t, err) diff --git a/src/runtime/pkg/resourcecontrol/cgroups.go b/src/runtime/pkg/resourcecontrol/cgroups.go index da88820ef..be8e9dc97 100644 --- a/src/runtime/pkg/resourcecontrol/cgroups.go +++ b/src/runtime/pkg/resourcecontrol/cgroups.go @@ -67,7 +67,8 @@ func sandboxDevices() []specs.LinuxDeviceCgroup { // In order to run Virtual Machines and create virtqueues, hypervisors // need access to certain character devices in the host, like kvm and vhost-net. hypervisorDevices := []string{ - "/dev/kvm", // To run virtual machines + "/dev/kvm", // To run virtual machines with KVM + "/dev/mshv", // To run virtual machines with Hyper-V "/dev/vhost-net", // To create virtqueues "/dev/vfio/vfio", // To access VFIO devices "/dev/vhost-vsock", // To interact with vsock if diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index 40aaf0bfa..6355f791f 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -154,6 +154,9 @@ type agent interface { // listInterfaces will tell the agent to list interfaces of an existed Sandbox listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error) + // updateEphemeralMounts will tell the agent to update tmpfs mounts in the Sandbox. + updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error + // updateRoutes will tell the agent to update route table for an existed Sandbox. updateRoutes(ctx context.Context, routes []*pbTypes.Route) ([]*pbTypes.Route, error) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index b0860fce0..f23393b97 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -857,12 +857,12 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { defer cancel() // Create the clh device config via the constructor to ensure default values are properly assigned - clhDevice := *chclient.NewDeviceConfig(device.SysfsDev) + clhDevice := *chclient.NewDeviceConfig(*(*device).GetSysfsDev()) pciInfo, _, err := cl.VmAddDevicePut(ctx, clhDevice) if err != nil { return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } - clh.devicesIds[device.ID] = pciInfo.GetId() + clh.devicesIds[*(*device).GetID()] = pciInfo.GetId() // clh doesn't use bridges, so the PCI path is simply the slot // number of the device. This will break if clh starts using @@ -879,7 +879,14 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { return fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } - device.GuestPciPath, err = types.PciPathFromString(tokens[0]) + guestPciPath, err := types.PciPathFromString(tokens[0]) + + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device) + } + pciDevice.GuestPciPath = guestPciPath + *device = pciDevice return err } @@ -923,7 +930,7 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int case BlockDev: deviceID = clhDriveIndexToID(devInfo.(*config.BlockDrive).Index) case VfioDev: - deviceID = devInfo.(*config.VFIODev).ID + deviceID = *devInfo.(config.VFIODev).GetID() default: clh.Logger().WithFields(log.Fields{"devInfo": devInfo, "deviceType": devType}).Error("HotplugRemoveDevice: unsupported device") diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 20e6935d8..a1c3c4b64 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -624,7 +624,7 @@ func TestCloudHypervisorHotplugRemoveDevice(t *testing.T) { _, err = clh.HotplugRemoveDevice(context.Background(), &config.BlockDrive{}, BlockDev) assert.NoError(err, "Hotplug remove block device expected no error") - _, err = clh.HotplugRemoveDevice(context.Background(), &config.VFIODev{}, VfioDev) + _, err = clh.HotplugRemoveDevice(context.Background(), &config.VFIOPCIDev{}, VfioDev) assert.NoError(err, "Hotplug remove vfio block device expected no error") _, err = clh.HotplugRemoveDevice(context.Background(), nil, NetDev) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 3925ceaa2..f13828f36 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -80,78 +80,80 @@ const ( type customRequestTimeoutKeyType struct{} var ( - checkRequestTimeout = 30 * time.Second - defaultRequestTimeout = 60 * time.Second - remoteRequestTimeout = 300 * time.Second - customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" - defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" - defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" - mountGuestTag = "kataShared" - defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" - type9pFs = "9p" - typeVirtioFS = "virtiofs" - typeOverlayFS = "overlay" - kata9pDevType = "9p" - kataMmioBlkDevType = "mmioblk" - kataBlkDevType = "blk" - kataBlkCCWDevType = "blk-ccw" - kataSCSIDevType = "scsi" - kataNvdimmDevType = "nvdimm" - kataVirtioFSDevType = "virtio-fs" - kataOverlayDevType = "overlayfs" - kataWatchableBindDevType = "watchable-bind" - kataVfioDevType = "vfio" // VFIO device to used as VFIO in the container - kataVfioGuestKernelDevType = "vfio-gk" // VFIO device for consumption by the guest kernel - sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} - sharedDirVirtioFSOptions = []string{} - sharedDirVirtioFSDaxOptions = "dax" - shmDir = "shm" - kataEphemeralDevType = "ephemeral" - defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) - grpcMaxDataSize = int64(1024 * 1024) - localDirOptions = []string{"mode=0777"} - maxHostnameLen = 64 - GuestDNSFile = "/etc/resolv.conf" + checkRequestTimeout = 30 * time.Second + defaultRequestTimeout = 60 * time.Second + remoteRequestTimeout = 300 * time.Second + customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" + defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" + defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" + mountGuestTag = "kataShared" + defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" + type9pFs = "9p" + typeVirtioFS = "virtiofs" + typeOverlayFS = "overlay" + kata9pDevType = "9p" + kataMmioBlkDevType = "mmioblk" + kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" + kataSCSIDevType = "scsi" + kataNvdimmDevType = "nvdimm" + kataVirtioFSDevType = "virtio-fs" + kataOverlayDevType = "overlayfs" + kataWatchableBindDevType = "watchable-bind" + kataVfioPciDevType = "vfio-pci" // VFIO device to used as VFIO in the container + kataVfioPciGuestKernelDevType = "vfio-pci-gk" // VFIO device for consumption by the guest kernel + kataVfioApDevType = "vfio-ap" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} + sharedDirVirtioFSOptions = []string{} + sharedDirVirtioFSDaxOptions = "dax" + shmDir = "shm" + kataEphemeralDevType = "ephemeral" + defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) + grpcMaxDataSize = int64(1024 * 1024) + localDirOptions = []string{"mode=0777"} + maxHostnameLen = 64 + GuestDNSFile = "/etc/resolv.conf" ) const ( - grpcCheckRequest = "grpc.CheckRequest" - grpcExecProcessRequest = "grpc.ExecProcessRequest" - grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" - grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" - grpcCreateContainerRequest = "grpc.CreateContainerRequest" - grpcStartContainerRequest = "grpc.StartContainerRequest" - grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" - grpcSignalProcessRequest = "grpc.SignalProcessRequest" - grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" - grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" - grpcListInterfacesRequest = "grpc.ListInterfacesRequest" - grpcListRoutesRequest = "grpc.ListRoutesRequest" - grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" - grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" - grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" - grpcWaitProcessRequest = "grpc.WaitProcessRequest" - grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" - grpcWriteStreamRequest = "grpc.WriteStreamRequest" - grpcCloseStdinRequest = "grpc.CloseStdinRequest" - grpcStatsContainerRequest = "grpc.StatsContainerRequest" - grpcPauseContainerRequest = "grpc.PauseContainerRequest" - grpcResumeContainerRequest = "grpc.ResumeContainerRequest" - grpcPullImageRequest = "grpc.PullImageRequest" - grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" - grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" - grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" - grpcCopyFileRequest = "grpc.CopyFileRequest" - grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" - grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" - grpcGetMetricsRequest = "grpc.GetMetricsRequest" - grpcAddSwapRequest = "grpc.AddSwapRequest" - grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" - grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" - grpcGetIPTablesRequest = "grpc.GetIPTablesRequest" - grpcSetIPTablesRequest = "grpc.SetIPTablesRequest" + grpcCheckRequest = "grpc.CheckRequest" + grpcExecProcessRequest = "grpc.ExecProcessRequest" + grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" + grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" + grpcCreateContainerRequest = "grpc.CreateContainerRequest" + grpcStartContainerRequest = "grpc.StartContainerRequest" + grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" + grpcSignalProcessRequest = "grpc.SignalProcessRequest" + grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" + grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" + grpcListInterfacesRequest = "grpc.ListInterfacesRequest" + grpcListRoutesRequest = "grpc.ListRoutesRequest" + grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" + grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" + grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" + grpcUpdateEphemeralMountsRequest = "grpc.UpdateEphemeralMountsRequest" + grpcWaitProcessRequest = "grpc.WaitProcessRequest" + grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" + grpcWriteStreamRequest = "grpc.WriteStreamRequest" + grpcCloseStdinRequest = "grpc.CloseStdinRequest" + grpcStatsContainerRequest = "grpc.StatsContainerRequest" + grpcPauseContainerRequest = "grpc.PauseContainerRequest" + grpcResumeContainerRequest = "grpc.ResumeContainerRequest" + grpcPullImageRequest = "grpc.PullImageRequest" + grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" + grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" + grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" + grpcCopyFileRequest = "grpc.CopyFileRequest" + grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" + grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" + grpcGetMetricsRequest = "grpc.GetMetricsRequest" + grpcAddSwapRequest = "grpc.AddSwapRequest" + grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" + grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" + grpcGetIPTablesRequest = "grpc.GetIPTablesRequest" + grpcSetIPTablesRequest = "grpc.SetIPTablesRequest" ) // newKataAgent returns an agent from an agent type. @@ -610,6 +612,21 @@ func (k *kataAgent) updateRoutes(ctx context.Context, routes []*pbTypes.Route) ( return nil, nil } +func (k *kataAgent) updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error { + if storages != nil { + storagesReq := &grpc.UpdateEphemeralMountsRequest{ + Storages: storages, + } + + if _, err := k.sendReq(ctx, storagesReq); err != nil { + k.Logger().WithError(err).Error("update mounts request failed") + return err + } + return nil + } + return nil +} + func (k *kataAgent) addARPNeighbors(ctx context.Context, neighs []*pbTypes.ARPNeighbor) error { if neighs != nil { neighsReq := &grpc.AddARPNeighborsRequest{ @@ -1112,20 +1129,25 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * groupNum := filepath.Base(dev.ContainerPath) - // Each /dev/vfio/NN device represents a VFIO group, which - // could include several PCI devices. So we give group - // information in the main structure, then list each - // individual PCI device in the Options array. + // For VFIO-PCI, each /dev/vfio/NN device represents a VFIO group, + // which could include several PCI devices. So we give group + // information in the main structure, then list each individual PCI + // device in the Options array. // // Each option is formatted as "DDDD:BB:DD.F=" // DDDD:BB:DD.F is the device's PCI address on the // *host*. is the device's PCI path in the guest // (see qomGetPciPath() for details). + // + // For VFIO-AP, one VFIO group could include several queue devices. They are + // identified by APQNs (Adjunct Processor Queue Numbers), which do not differ + // between host and guest. They are passed as options so they can be awaited + // by the agent. kataDevice := &grpc.Device{ ContainerPath: dev.ContainerPath, - Type: kataVfioDevType, + Type: kataVfioPciDevType, Id: groupNum, - Options: make([]string, len(devList)), + Options: nil, } // We always pass the device information to the agent, since @@ -1133,11 +1155,18 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * // on the vfio_mode, we need to use a different device type so // the agent can handle it properly if c.sandbox.config.VfioMode == config.VFIOModeGuestKernel { - kataDevice.Type = kataVfioGuestKernelDevType + kataDevice.Type = kataVfioPciGuestKernelDevType } - for i, pciDev := range devList { - kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDev.BDF, pciDev.GuestPciPath) + if (*devList[0]).GetType() == config.VFIOAPDeviceMediatedType { + kataDevice.Type = kataVfioApDevType + kataDevice.Options = (*devList[0]).(config.VFIOAPDev).APDevices + } else { + kataDevice.Options = make([]string, len(devList)) + for i, device := range devList { + pciDevice := (*device).(config.VFIOPCIDev) + kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath) + } } return kataDevice @@ -1963,6 +1992,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest)) } + k.reqHandlers[grpcUpdateEphemeralMountsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.UpdateEphemeralMounts(ctx, req.(*grpc.UpdateEphemeralMountsRequest)) + } k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest)) } diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index d588ceab3..b71a52f7e 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -122,6 +122,11 @@ func (n *mockAgent) listRoutes(ctx context.Context) ([]*pbTypes.Route, error) { return nil, nil } +// updateEphemeralMounts is the Noop agent updateEphemeralMounts implementation. It does nothing. +func (n *mockAgent) updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error { + return nil +} + // check is the Noop agent health checker. It does nothing. func (n *mockAgent) check(ctx context.Context) error { return nil diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index 747eca792..36af23886 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1527,6 +1527,45 @@ func (m *UpdateRoutesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_UpdateRoutesRequest proto.InternalMessageInfo +type UpdateEphemeralMountsRequest struct { + Storages []*Storage `protobuf:"bytes,1,rep,name=storages,proto3" json:"storages,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UpdateEphemeralMountsRequest) Reset() { *m = UpdateEphemeralMountsRequest{} } +func (*UpdateEphemeralMountsRequest) ProtoMessage() {} +func (*UpdateEphemeralMountsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{36} +} +func (m *UpdateEphemeralMountsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *UpdateEphemeralMountsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_UpdateEphemeralMountsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *UpdateEphemeralMountsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UpdateEphemeralMountsRequest.Merge(m, src) +} +func (m *UpdateEphemeralMountsRequest) XXX_Size() int { + return m.Size() +} +func (m *UpdateEphemeralMountsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_UpdateEphemeralMountsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_UpdateEphemeralMountsRequest proto.InternalMessageInfo + type ListInterfacesRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1536,7 +1575,7 @@ type ListInterfacesRequest struct { func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } func (*ListInterfacesRequest) ProtoMessage() {} func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{36} + return fileDescriptor_712ce9a559fda969, []int{37} } func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1574,7 +1613,7 @@ type ListRoutesRequest struct { func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } func (*ListRoutesRequest) ProtoMessage() {} func (*ListRoutesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{37} + return fileDescriptor_712ce9a559fda969, []int{38} } func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1613,7 +1652,7 @@ type ARPNeighbors struct { func (m *ARPNeighbors) Reset() { *m = ARPNeighbors{} } func (*ARPNeighbors) ProtoMessage() {} func (*ARPNeighbors) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{38} + return fileDescriptor_712ce9a559fda969, []int{39} } func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1652,7 +1691,7 @@ type AddARPNeighborsRequest struct { func (m *AddARPNeighborsRequest) Reset() { *m = AddARPNeighborsRequest{} } func (*AddARPNeighborsRequest) ProtoMessage() {} func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{39} + return fileDescriptor_712ce9a559fda969, []int{40} } func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1691,7 +1730,7 @@ type GetIPTablesRequest struct { func (m *GetIPTablesRequest) Reset() { *m = GetIPTablesRequest{} } func (*GetIPTablesRequest) ProtoMessage() {} func (*GetIPTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{40} + return fileDescriptor_712ce9a559fda969, []int{41} } func (m *GetIPTablesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1731,7 +1770,7 @@ type GetIPTablesResponse struct { func (m *GetIPTablesResponse) Reset() { *m = GetIPTablesResponse{} } func (*GetIPTablesResponse) ProtoMessage() {} func (*GetIPTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{41} + return fileDescriptor_712ce9a559fda969, []int{42} } func (m *GetIPTablesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1773,7 +1812,7 @@ type SetIPTablesRequest struct { func (m *SetIPTablesRequest) Reset() { *m = SetIPTablesRequest{} } func (*SetIPTablesRequest) ProtoMessage() {} func (*SetIPTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{42} + return fileDescriptor_712ce9a559fda969, []int{43} } func (m *SetIPTablesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1803,7 +1842,7 @@ func (m *SetIPTablesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_SetIPTablesRequest proto.InternalMessageInfo type SetIPTablesResponse struct { - // raw stdout from iptables-restore or ip6tables-restore ? + // raw stdout from iptables-restore or ip6tables-restore Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1813,7 +1852,7 @@ type SetIPTablesResponse struct { func (m *SetIPTablesResponse) Reset() { *m = SetIPTablesResponse{} } func (*SetIPTablesResponse) ProtoMessage() {} func (*SetIPTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{43} + return fileDescriptor_712ce9a559fda969, []int{44} } func (m *SetIPTablesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1859,7 +1898,7 @@ type OnlineCPUMemRequest struct { func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } func (*OnlineCPUMemRequest) ProtoMessage() {} func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{44} + return fileDescriptor_712ce9a559fda969, []int{45} } func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1899,7 +1938,7 @@ type ReseedRandomDevRequest struct { func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } func (*ReseedRandomDevRequest) ProtoMessage() {} func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{45} + return fileDescriptor_712ce9a559fda969, []int{46} } func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1949,7 +1988,7 @@ type AgentDetails struct { func (m *AgentDetails) Reset() { *m = AgentDetails{} } func (*AgentDetails) ProtoMessage() {} func (*AgentDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{46} + return fileDescriptor_712ce9a559fda969, []int{47} } func (m *AgentDetails) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1995,7 +2034,7 @@ type GuestDetailsRequest struct { func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } func (*GuestDetailsRequest) ProtoMessage() {} func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{47} + return fileDescriptor_712ce9a559fda969, []int{48} } func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2037,7 +2076,7 @@ type GuestDetailsResponse struct { func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } func (*GuestDetailsResponse) ProtoMessage() {} func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{48} + return fileDescriptor_712ce9a559fda969, []int{49} } func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2078,7 +2117,7 @@ type MemHotplugByProbeRequest struct { func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } func (*MemHotplugByProbeRequest) ProtoMessage() {} func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{49} + return fileDescriptor_712ce9a559fda969, []int{50} } func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2120,7 +2159,7 @@ type SetGuestDateTimeRequest struct { func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } func (*SetGuestDateTimeRequest) ProtoMessage() {} func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{50} + return fileDescriptor_712ce9a559fda969, []int{51} } func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2166,7 +2205,7 @@ type FSGroup struct { func (m *FSGroup) Reset() { *m = FSGroup{} } func (*FSGroup) ProtoMessage() {} func (*FSGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{51} + return fileDescriptor_712ce9a559fda969, []int{52} } func (m *FSGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2234,7 +2273,7 @@ type Storage struct { func (m *Storage) Reset() { *m = Storage{} } func (*Storage) ProtoMessage() {} func (*Storage) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{52} + return fileDescriptor_712ce9a559fda969, []int{53} } func (m *Storage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2305,7 +2344,7 @@ type Device struct { func (m *Device) Reset() { *m = Device{} } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{53} + return fileDescriptor_712ce9a559fda969, []int{54} } func (m *Device) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2346,7 +2385,7 @@ type StringUser struct { func (m *StringUser) Reset() { *m = StringUser{} } func (*StringUser) ProtoMessage() {} func (*StringUser) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{54} + return fileDescriptor_712ce9a559fda969, []int{55} } func (m *StringUser) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2403,7 +2442,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (*CopyFileRequest) ProtoMessage() {} func (*CopyFileRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{55} + return fileDescriptor_712ce9a559fda969, []int{56} } func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2441,7 +2480,7 @@ type GetOOMEventRequest struct { func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } func (*GetOOMEventRequest) ProtoMessage() {} func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{56} + return fileDescriptor_712ce9a559fda969, []int{57} } func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2480,7 +2519,7 @@ type OOMEvent struct { func (m *OOMEvent) Reset() { *m = OOMEvent{} } func (*OOMEvent) ProtoMessage() {} func (*OOMEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{57} + return fileDescriptor_712ce9a559fda969, []int{58} } func (m *OOMEvent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2519,7 +2558,7 @@ type AddSwapRequest struct { func (m *AddSwapRequest) Reset() { *m = AddSwapRequest{} } func (*AddSwapRequest) ProtoMessage() {} func (*AddSwapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{58} + return fileDescriptor_712ce9a559fda969, []int{59} } func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2557,7 +2596,7 @@ type GetMetricsRequest struct { func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } func (*GetMetricsRequest) ProtoMessage() {} func (*GetMetricsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{59} + return fileDescriptor_712ce9a559fda969, []int{60} } func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2596,7 +2635,7 @@ type Metrics struct { func (m *Metrics) Reset() { *m = Metrics{} } func (*Metrics) ProtoMessage() {} func (*Metrics) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{60} + return fileDescriptor_712ce9a559fda969, []int{61} } func (m *Metrics) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2636,7 +2675,7 @@ type VolumeStatsRequest struct { func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } func (*VolumeStatsRequest) ProtoMessage() {} func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{61} + return fileDescriptor_712ce9a559fda969, []int{62} } func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2677,7 +2716,7 @@ type ResizeVolumeRequest struct { func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } func (*ResizeVolumeRequest) ProtoMessage() {} func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{62} + return fileDescriptor_712ce9a559fda969, []int{63} } func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2745,6 +2784,7 @@ func init() { proto.RegisterType((*Routes)(nil), "grpc.Routes") proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") + proto.RegisterType((*UpdateEphemeralMountsRequest)(nil), "grpc.UpdateEphemeralMountsRequest") proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") proto.RegisterType((*ARPNeighbors)(nil), "grpc.ARPNeighbors") @@ -2779,208 +2819,211 @@ func init() { } var fileDescriptor_712ce9a559fda969 = []byte{ - // 3207 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x73, 0x1b, 0xc7, - 0xd1, 0xc6, 0x83, 0x04, 0xd0, 0x78, 0x11, 0x4b, 0x8a, 0x82, 0x60, 0x9b, 0x96, 0x57, 0xb6, 0x4c, - 0xd9, 0x9f, 0x48, 0x5b, 0x76, 0x59, 0x7e, 0x94, 0x3f, 0x7d, 0x24, 0x45, 0x93, 0xb4, 0x4d, 0x0b, - 0xdf, 0x42, 0x8c, 0x53, 0x49, 0x25, 0x5b, 0xcb, 0xdd, 0x21, 0x38, 0x26, 0x76, 0x67, 0x3d, 0x3b, - 0x0b, 0x91, 0x4e, 0x55, 0x2a, 0xa7, 0xe4, 0x96, 0x63, 0x6e, 0xf9, 0x03, 0xa9, 0xfc, 0x83, 0x5c, - 0x73, 0x50, 0xe5, 0x94, 0x63, 0x2e, 0x49, 0xc5, 0xfa, 0x09, 0xf9, 0x05, 0xa9, 0x79, 0xed, 0x03, - 0x0f, 0xda, 0x61, 0xa9, 0x2a, 0x17, 0xd4, 0x76, 0x4f, 0x4f, 0xbf, 0xa6, 0xa7, 0xa7, 0x7b, 0x06, - 0xd0, 0x1f, 0x62, 0x76, 0x1a, 0x1f, 0x6f, 0xb8, 0xc4, 0xdf, 0x3c, 0x73, 0x98, 0x73, 0xd7, 0x25, - 0x01, 0x73, 0x70, 0x80, 0x68, 0x34, 0x05, 0x47, 0xd4, 0xdd, 0x1c, 0xe1, 0xe3, 0x68, 0x33, 0xa4, - 0x84, 0x11, 0x97, 0x8c, 0xd4, 0x57, 0xb4, 0xe9, 0x0c, 0x51, 0xc0, 0x36, 0x04, 0x60, 0x94, 0x87, - 0x34, 0x74, 0x7b, 0x35, 0xe2, 0x62, 0x89, 0xe8, 0xd5, 0xdc, 0x48, 0x7f, 0xd6, 0xd9, 0x45, 0x88, - 0x22, 0x05, 0xbc, 0x38, 0x24, 0x64, 0x38, 0x42, 0x92, 0xc7, 0x71, 0x7c, 0xb2, 0x89, 0xfc, 0x90, - 0x5d, 0xc8, 0x41, 0xf3, 0xf7, 0x45, 0x58, 0xdd, 0xa1, 0xc8, 0x61, 0x68, 0x47, 0x2b, 0x60, 0xa1, - 0x6f, 0x62, 0x14, 0x31, 0xe3, 0x55, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x16, 0x6e, 0x16, 0xd6, - 0x6b, 0x56, 0x3d, 0xc1, 0x1d, 0x78, 0xc6, 0x75, 0xa8, 0xa0, 0x73, 0xe4, 0xf2, 0xd1, 0xa2, 0x18, - 0x5d, 0xe4, 0xe0, 0x81, 0x67, 0xbc, 0x03, 0xf5, 0x88, 0x51, 0x1c, 0x0c, 0xed, 0x38, 0x42, 0xb4, - 0x5b, 0xba, 0x59, 0x58, 0xaf, 0xdf, 0x5b, 0xda, 0xe0, 0x2a, 0x6f, 0x0c, 0xc4, 0xc0, 0x51, 0x84, - 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x86, 0x8a, 0x87, 0xc6, 0xd8, 0x45, 0x51, 0xb7, 0x7c, 0xb3, - 0xb4, 0x5e, 0xbf, 0xd7, 0x90, 0xe4, 0x0f, 0x05, 0xd2, 0xd2, 0x83, 0xc6, 0x1d, 0xa8, 0x46, 0x8c, - 0x50, 0x67, 0x88, 0xa2, 0xee, 0x82, 0x20, 0x6c, 0x6a, 0xbe, 0x02, 0x6b, 0x25, 0xc3, 0xc6, 0x4b, - 0x50, 0x7a, 0xb4, 0x73, 0xd0, 0x5d, 0x14, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8, - 0x05, 0xcd, 0xc8, 0x09, 0xbc, 0x63, 0x72, 0x6e, 0x87, 0xd8, 0x0b, 0xa2, 0x6e, 0xe5, 0x66, 0x61, - 0xbd, 0x6a, 0x35, 0x14, 0xb2, 0xcf, 0x71, 0xe6, 0x47, 0x70, 0x6d, 0xc0, 0x1c, 0xca, 0xae, 0xe0, - 0x1d, 0xf3, 0x08, 0x56, 0x2d, 0xe4, 0x93, 0xf1, 0x95, 0x5c, 0xdb, 0x85, 0x0a, 0xc3, 0x3e, 0x22, - 0x31, 0x13, 0xae, 0x6d, 0x5a, 0x1a, 0x34, 0xff, 0x58, 0x00, 0x63, 0xf7, 0x1c, 0xb9, 0x7d, 0x4a, - 0x5c, 0x14, 0x45, 0xff, 0xa5, 0xe5, 0x7a, 0x03, 0x2a, 0xa1, 0x54, 0xa0, 0x5b, 0x16, 0xe4, 0x6a, - 0x15, 0xb4, 0x56, 0x7a, 0xd4, 0xfc, 0x1a, 0x56, 0x06, 0x78, 0x18, 0x38, 0xa3, 0xe7, 0xa8, 0xef, - 0x2a, 0x2c, 0x46, 0x82, 0xa7, 0x50, 0xb5, 0x69, 0x29, 0xc8, 0xec, 0x83, 0xf1, 0x95, 0x83, 0xd9, - 0xf3, 0x93, 0x64, 0xde, 0x85, 0xe5, 0x1c, 0xc7, 0x28, 0x24, 0x41, 0x84, 0x84, 0x02, 0xcc, 0x61, - 0x71, 0x24, 0x98, 0x2d, 0x58, 0x0a, 0x32, 0x09, 0xac, 0x1e, 0x85, 0xde, 0x15, 0x77, 0xd3, 0x3d, - 0xa8, 0x51, 0x14, 0x91, 0x98, 0xf2, 0x3d, 0x50, 0x14, 0x4e, 0x5d, 0x91, 0x4e, 0xfd, 0x02, 0x07, - 0xf1, 0xb9, 0xa5, 0xc7, 0xac, 0x94, 0x4c, 0xc5, 0x27, 0x8b, 0xae, 0x12, 0x9f, 0x1f, 0xc1, 0xb5, - 0xbe, 0x13, 0x47, 0x57, 0xd1, 0xd5, 0xfc, 0x98, 0xc7, 0x76, 0x14, 0xfb, 0x57, 0x9a, 0xfc, 0x87, - 0x02, 0x54, 0x77, 0xc2, 0xf8, 0x28, 0x72, 0x86, 0xc8, 0x78, 0x05, 0xea, 0x8c, 0x30, 0x67, 0x64, - 0xc7, 0x1c, 0x14, 0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x0a, 0x8d, 0x10, 0x51, 0x37, 0x8c, - 0x15, 0x45, 0xf1, 0x66, 0x69, 0xbd, 0x6c, 0xd5, 0x25, 0x4e, 0x92, 0x6c, 0xc0, 0xb2, 0x18, 0xb3, - 0x71, 0x60, 0x9f, 0x21, 0x1a, 0xa0, 0x91, 0x4f, 0x3c, 0x24, 0x82, 0xa3, 0x6c, 0x75, 0xc4, 0xd0, - 0x41, 0xf0, 0x79, 0x32, 0x60, 0xbc, 0x09, 0x9d, 0x84, 0x9e, 0x47, 0xbc, 0xa0, 0x2e, 0x0b, 0xea, - 0xb6, 0xa2, 0x3e, 0x52, 0x68, 0xf3, 0x97, 0xd0, 0x7a, 0x7c, 0x4a, 0x09, 0x63, 0x23, 0x1c, 0x0c, - 0x1f, 0x3a, 0xcc, 0xe1, 0x5b, 0x33, 0x44, 0x14, 0x13, 0x2f, 0x52, 0xda, 0x6a, 0xd0, 0x78, 0x0b, - 0x3a, 0x4c, 0xd2, 0x22, 0xcf, 0xd6, 0x34, 0x45, 0x41, 0xb3, 0x94, 0x0c, 0xf4, 0x15, 0xf1, 0xeb, - 0xd0, 0x4a, 0x89, 0xf9, 0xe6, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x63, 0xec, 0x23, 0x73, 0x2c, 0x7c, - 0x25, 0x16, 0xd9, 0x78, 0x0b, 0x6a, 0xa9, 0x1f, 0x0a, 0x22, 0x42, 0x5a, 0x32, 0x42, 0xb4, 0x3b, - 0xad, 0x6a, 0xe2, 0x94, 0x4f, 0xa0, 0xcd, 0x12, 0xc5, 0x6d, 0xcf, 0x61, 0x4e, 0x3e, 0xa8, 0xf2, - 0x56, 0x59, 0x2d, 0x96, 0x83, 0xcd, 0x8f, 0xa1, 0xd6, 0xc7, 0x5e, 0x24, 0x05, 0x77, 0xa1, 0xe2, - 0xc6, 0x94, 0xa2, 0x80, 0x69, 0x93, 0x15, 0x68, 0xac, 0xc0, 0xc2, 0x08, 0xfb, 0x98, 0x29, 0x33, - 0x25, 0x60, 0x12, 0x80, 0x43, 0xe4, 0x13, 0x7a, 0x21, 0x1c, 0xb6, 0x02, 0x0b, 0xd9, 0xc5, 0x95, - 0x80, 0xf1, 0x22, 0xd4, 0x7c, 0xe7, 0x3c, 0x59, 0x54, 0x3e, 0x52, 0xf5, 0x9d, 0x73, 0xa9, 0x7c, - 0x17, 0x2a, 0x27, 0x0e, 0x1e, 0xb9, 0x01, 0x53, 0x5e, 0xd1, 0x60, 0x2a, 0xb0, 0x9c, 0x15, 0xf8, - 0xe7, 0x22, 0xd4, 0xa5, 0x44, 0xa9, 0xf0, 0x0a, 0x2c, 0xb8, 0x8e, 0x7b, 0x9a, 0x88, 0x14, 0x80, - 0x71, 0x5b, 0x2b, 0x52, 0xcc, 0x66, 0xb8, 0x54, 0x53, 0xad, 0xda, 0x26, 0x40, 0xf4, 0xc4, 0x09, - 0x95, 0x6e, 0xa5, 0x39, 0xc4, 0x35, 0x4e, 0x23, 0xd5, 0x7d, 0x17, 0x1a, 0x32, 0xee, 0xd4, 0x94, - 0xf2, 0x9c, 0x29, 0x75, 0x49, 0x25, 0x27, 0xdd, 0x82, 0x66, 0x1c, 0x21, 0xfb, 0x14, 0x23, 0xea, - 0x50, 0xf7, 0xf4, 0xa2, 0xbb, 0x20, 0x0f, 0xa0, 0x38, 0x42, 0xfb, 0x1a, 0x67, 0xdc, 0x83, 0x05, - 0x9e, 0x5b, 0xa2, 0xee, 0xa2, 0x38, 0xeb, 0x5e, 0xca, 0xb2, 0x14, 0xa6, 0x6e, 0x88, 0xdf, 0xdd, - 0x80, 0xd1, 0x0b, 0x4b, 0x92, 0xf6, 0x3e, 0x00, 0x48, 0x91, 0xc6, 0x12, 0x94, 0xce, 0xd0, 0x85, - 0xda, 0x87, 0xfc, 0x93, 0x3b, 0x67, 0xec, 0x8c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x54, 0xfc, 0xa0, - 0x60, 0xba, 0xd0, 0xde, 0x1e, 0x9d, 0x61, 0x92, 0x99, 0xbe, 0x02, 0x0b, 0xbe, 0xf3, 0x35, 0xa1, - 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84, 0xc2, - 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x47, 0x19, 0x20, 0x95, 0x62, - 0x58, 0xd0, 0xc3, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x7c, 0xc1, 0x50, 0x64, 0x53, 0xe4, - 0xc6, 0x34, 0xc2, 0x63, 0xbe, 0x7e, 0xdc, 0xec, 0x6b, 0xd2, 0xec, 0x09, 0xdd, 0xac, 0xeb, 0x98, - 0x0c, 0xe4, 0xbc, 0x6d, 0x3e, 0xcd, 0xd2, 0xb3, 0x8c, 0x03, 0xb8, 0x96, 0xf2, 0xf4, 0x32, 0xec, - 0x8a, 0x97, 0xb1, 0x5b, 0x4e, 0xd8, 0x79, 0x29, 0xab, 0x5d, 0x58, 0xc6, 0xc4, 0xfe, 0x26, 0x46, - 0x71, 0x8e, 0x51, 0xe9, 0x32, 0x46, 0x1d, 0x4c, 0xfe, 0x5f, 0x4c, 0x48, 0xd9, 0xf4, 0xe1, 0x46, - 0xc6, 0x4a, 0xbe, 0xdd, 0x33, 0xcc, 0xca, 0x97, 0x31, 0x5b, 0x4d, 0xb4, 0xe2, 0xf9, 0x20, 0xe5, - 0xf8, 0x19, 0xac, 0x62, 0x62, 0x3f, 0x71, 0x30, 0x9b, 0x64, 0xb7, 0xf0, 0x3d, 0x46, 0xf2, 0x13, - 0x2d, 0xcf, 0x4b, 0x1a, 0xe9, 0x23, 0x3a, 0xcc, 0x19, 0xb9, 0xf8, 0x3d, 0x46, 0x1e, 0x8a, 0x09, - 0x29, 0x9b, 0x2d, 0xe8, 0x60, 0x32, 0xa9, 0x4d, 0xe5, 0x32, 0x26, 0x6d, 0x4c, 0xf2, 0x9a, 0x6c, - 0x43, 0x27, 0x42, 0x2e, 0x23, 0x34, 0x1b, 0x04, 0xd5, 0xcb, 0x58, 0x2c, 0x29, 0xfa, 0x84, 0x87, - 0xf9, 0x53, 0x68, 0xec, 0xc7, 0x43, 0xc4, 0x46, 0xc7, 0x49, 0x32, 0x78, 0x6e, 0xf9, 0xc7, 0xfc, - 0x57, 0x11, 0xea, 0x3b, 0x43, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xcc, 0xc9, 0x82, - 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x1e, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d, 0xa9, - 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x00, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d, 0xb5, - 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x0e, 0xd4, 0x8f, 0xb9, 0x93, 0xd4, - 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x71, 0xba, 0x09, 0xf7, 0xa1, 0x79, 0x2a, 0x5d, 0xa6, - 0x26, 0xc9, 0x18, 0xba, 0xa5, 0x2c, 0x49, 0xed, 0xdd, 0xc8, 0x7a, 0x56, 0x2e, 0x40, 0xe3, 0x34, - 0x83, 0xea, 0x0d, 0xa0, 0x33, 0x45, 0x32, 0x23, 0x07, 0xad, 0x67, 0x73, 0x50, 0xfd, 0x9e, 0x21, - 0x05, 0x65, 0x67, 0x66, 0xf3, 0xd2, 0x6f, 0x8b, 0xd0, 0xf8, 0x12, 0xb1, 0x27, 0x84, 0x9e, 0x49, - 0x7d, 0x0d, 0x28, 0x07, 0x8e, 0x8f, 0x14, 0x47, 0xf1, 0x6d, 0xdc, 0x80, 0x2a, 0x3d, 0x97, 0x09, - 0x44, 0xad, 0x67, 0x85, 0x9e, 0x8b, 0xc4, 0x60, 0xbc, 0x0c, 0x40, 0xcf, 0xed, 0xd0, 0x71, 0xcf, - 0x90, 0xf2, 0x60, 0xd9, 0xaa, 0xd1, 0xf3, 0xbe, 0x44, 0xf0, 0x50, 0xa0, 0xe7, 0x36, 0xa2, 0x94, - 0xd0, 0x48, 0xe5, 0xaa, 0x2a, 0x3d, 0xdf, 0x15, 0xb0, 0x9a, 0xeb, 0x51, 0x12, 0x86, 0xc8, 0x13, - 0x39, 0x5a, 0xcc, 0x7d, 0x28, 0x11, 0x5c, 0x2a, 0xd3, 0x52, 0x17, 0xa5, 0x54, 0x96, 0x4a, 0x65, - 0xa9, 0xd4, 0x8a, 0x9c, 0xc9, 0xb2, 0x52, 0x59, 0x22, 0xb5, 0x2a, 0xa5, 0xb2, 0x8c, 0x54, 0x96, - 0x4a, 0xad, 0xe9, 0xb9, 0x4a, 0xaa, 0xf9, 0x9b, 0x02, 0xac, 0x4e, 0x16, 0x7e, 0xaa, 0x36, 0x7d, - 0x0f, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0xa6, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30, 0xbe, - 0x0f, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, 0x03, - 0x99, 0x1e, 0x18, 0x5f, 0x51, 0xcc, 0xd0, 0x80, 0x51, 0xe4, 0xf8, 0xcf, 0xa3, 0xba, 0x37, 0xa0, - 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0xc0, 0x72, 0x4e, 0x8a, 0xb2, 0x75, - 0x09, 0x4a, 0x23, 0x14, 0x08, 0xee, 0x4d, 0x8b, 0x7f, 0x9a, 0x0e, 0x74, 0x2c, 0xe4, 0x78, 0xcf, - 0x4f, 0x1b, 0x25, 0xa2, 0x94, 0x8a, 0x58, 0x07, 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x64, - 0xb4, 0x7e, 0x04, 0x9d, 0x9d, 0x11, 0x89, 0xd0, 0x80, 0x79, 0x38, 0x78, 0x1e, 0xed, 0xc8, 0x2f, - 0x60, 0xf9, 0x31, 0xbb, 0xf8, 0x8a, 0x33, 0x8b, 0xf0, 0xb7, 0xe8, 0x39, 0xd9, 0x47, 0xc9, 0x13, - 0x6d, 0x1f, 0x25, 0x4f, 0x78, 0x73, 0xe3, 0x92, 0x51, 0xec, 0x07, 0x62, 0x2b, 0x34, 0x2d, 0x05, - 0x99, 0xdb, 0xd0, 0x90, 0x35, 0xf4, 0x21, 0xf1, 0xe2, 0x11, 0x9a, 0xb9, 0x07, 0xd7, 0x00, 0x42, - 0x87, 0x3a, 0x3e, 0x62, 0x88, 0xca, 0x18, 0xaa, 0x59, 0x19, 0x8c, 0xf9, 0xbb, 0x22, 0xac, 0xc8, - 0xfb, 0x86, 0x81, 0x6c, 0xb3, 0xb5, 0x09, 0x3d, 0xa8, 0x9e, 0x92, 0x88, 0x65, 0x18, 0x26, 0x30, - 0x57, 0x91, 0xf7, 0xe7, 0x92, 0x1b, 0xff, 0xcc, 0x5d, 0x02, 0x94, 0x2e, 0xbf, 0x04, 0x98, 0x6a, - 0xf3, 0xcb, 0xd3, 0x6d, 0x3e, 0xdf, 0x6d, 0x9a, 0x08, 0xcb, 0x3d, 0x5e, 0xb3, 0x6a, 0x0a, 0x73, - 0xe0, 0x19, 0xb7, 0xa1, 0x3d, 0xe4, 0x5a, 0xda, 0xa7, 0x84, 0x9c, 0xd9, 0xa1, 0xc3, 0x4e, 0xc5, - 0x56, 0xaf, 0x59, 0x4d, 0x81, 0xde, 0x27, 0xe4, 0xac, 0xef, 0xb0, 0x53, 0xe3, 0x43, 0x68, 0xa9, - 0x32, 0xd0, 0x17, 0x2e, 0x8a, 0xd4, 0xe1, 0xa7, 0x76, 0x51, 0xd6, 0x7b, 0x56, 0xf3, 0x2c, 0x03, - 0x45, 0xe6, 0x75, 0xb8, 0xf6, 0x10, 0x45, 0x8c, 0x92, 0x8b, 0xbc, 0x63, 0xcc, 0xff, 0x05, 0x38, - 0x08, 0x18, 0xa2, 0x27, 0x8e, 0x8b, 0x22, 0xe3, 0xed, 0x2c, 0xa4, 0x8a, 0xa3, 0xa5, 0x0d, 0x79, - 0xdd, 0x93, 0x0c, 0x58, 0x19, 0x1a, 0x73, 0x03, 0x16, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0x9a, 0xfe, - 0x52, 0xf3, 0x1a, 0x6a, 0x9e, 0x40, 0x5a, 0x6a, 0xcc, 0xdc, 0xd7, 0x2d, 0x6c, 0xca, 0x4e, 0x2d, - 0xd1, 0x06, 0xd4, 0xb0, 0xc6, 0xa9, 0xac, 0x32, 0x2d, 0x3a, 0x25, 0x31, 0x3f, 0x86, 0x65, 0xc9, - 0x49, 0x72, 0xd6, 0x6c, 0x5e, 0x83, 0x45, 0xaa, 0xd5, 0x28, 0xa4, 0xf7, 0x3c, 0x8a, 0x48, 0x8d, - 0x71, 0x7f, 0x7c, 0x81, 0x23, 0x96, 0x1a, 0xa2, 0xfd, 0xb1, 0x0c, 0x1d, 0x3e, 0x90, 0xe3, 0x69, - 0x7e, 0x0a, 0x8d, 0x2d, 0xab, 0xff, 0x25, 0xc2, 0xc3, 0xd3, 0x63, 0x9e, 0x3d, 0xdf, 0xcf, 0xc3, - 0xca, 0x60, 0x43, 0x69, 0x9b, 0x19, 0xb2, 0x72, 0x74, 0xe6, 0x67, 0xb0, 0xba, 0xe5, 0x79, 0x59, - 0x94, 0xd6, 0xfa, 0x6d, 0xa8, 0x05, 0x19, 0x76, 0x99, 0x33, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x77, - 0xc1, 0xd8, 0x43, 0xec, 0xa0, 0xff, 0xd8, 0x39, 0x1e, 0xa5, 0xd6, 0x5f, 0x87, 0x0a, 0x8e, 0x6c, - 0x1c, 0x8e, 0xdf, 0x17, 0x5c, 0xaa, 0xd6, 0x22, 0x8e, 0x0e, 0xc2, 0xf1, 0xfb, 0xe6, 0x1d, 0x58, - 0xce, 0x91, 0x5f, 0x92, 0x56, 0xb6, 0xc0, 0x18, 0xfc, 0x70, 0xce, 0x09, 0x8b, 0x62, 0x86, 0xc5, - 0x1d, 0x58, 0x1e, 0xfc, 0x40, 0x69, 0x3f, 0x83, 0xe5, 0x47, 0xc1, 0x08, 0x07, 0x68, 0xa7, 0x7f, - 0x74, 0x88, 0x92, 0x9c, 0x6a, 0x40, 0x99, 0xd7, 0x9e, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0xc7, - 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xb4, 0x5a, 0x0c, 0x8e, 0x77, 0xc2, 0x38, 0xe2, 0x87, 0x24, 0x2f, - 0x92, 0x48, 0x30, 0xba, 0x10, 0x99, 0xa6, 0x6a, 0x55, 0xdc, 0x30, 0x7e, 0x14, 0x8c, 0x2e, 0xcc, - 0xff, 0x11, 0x37, 0x09, 0x08, 0x79, 0x96, 0x13, 0x78, 0xc4, 0x7f, 0x88, 0xc6, 0x19, 0x09, 0x53, - 0x7a, 0x3f, 0x2d, 0x40, 0x63, 0x6b, 0x88, 0x02, 0xf6, 0x10, 0x31, 0x07, 0x8f, 0x44, 0x67, 0x3a, - 0x46, 0x34, 0xc2, 0x24, 0x50, 0x69, 0x43, 0x83, 0xc6, 0x2b, 0x50, 0xc7, 0x01, 0x66, 0xb6, 0xe7, - 0x20, 0x9f, 0x04, 0x82, 0x4b, 0xd5, 0x02, 0x8e, 0x7a, 0x28, 0x30, 0xc6, 0x1b, 0xd0, 0x96, 0x97, - 0x8a, 0xf6, 0xa9, 0x13, 0x78, 0x23, 0x9e, 0xb0, 0x4a, 0x22, 0xc5, 0xb4, 0x24, 0x7a, 0x5f, 0x61, - 0x8d, 0x3b, 0xb0, 0xa4, 0xd2, 0x49, 0x4a, 0x59, 0x16, 0x94, 0x6d, 0x85, 0xcf, 0x91, 0xc6, 0x61, - 0x48, 0x28, 0x8b, 0xec, 0x08, 0xb9, 0x2e, 0xf1, 0x43, 0xd5, 0xd6, 0xb5, 0x35, 0x7e, 0x20, 0xd1, - 0xe6, 0x10, 0x96, 0xf7, 0xb8, 0x9d, 0xca, 0x92, 0x74, 0x7b, 0xb4, 0x7c, 0xe4, 0xdb, 0xc7, 0x23, - 0xe2, 0x9e, 0xd9, 0x3c, 0xc9, 0x2b, 0x0f, 0xf3, 0xc2, 0x71, 0x9b, 0x23, 0x07, 0xf8, 0x5b, 0x71, - 0x83, 0xc1, 0xa9, 0x4e, 0x09, 0x0b, 0x47, 0xf1, 0xd0, 0x0e, 0x29, 0x39, 0x46, 0xca, 0xc4, 0xb6, - 0x8f, 0xfc, 0x7d, 0x89, 0xef, 0x73, 0xb4, 0xf9, 0xa7, 0x02, 0xac, 0xe4, 0x25, 0xa9, 0xd5, 0xde, - 0x84, 0x95, 0xbc, 0x28, 0x55, 0xc6, 0xc8, 0x32, 0xb9, 0x93, 0x15, 0x28, 0x0b, 0x9a, 0xfb, 0xd0, - 0x14, 0x57, 0xd0, 0xb6, 0x27, 0x39, 0xe5, 0x8b, 0xb7, 0xec, 0xba, 0x58, 0x0d, 0x27, 0xbb, 0x4a, - 0x1f, 0xc2, 0x0d, 0x65, 0xbe, 0x3d, 0xad, 0xb6, 0x0c, 0x88, 0x55, 0x45, 0x70, 0x38, 0xa1, 0xfd, - 0x17, 0xd0, 0x4d, 0x51, 0xdb, 0x17, 0x02, 0x99, 0x6e, 0xca, 0xe5, 0x09, 0x63, 0xb7, 0x3c, 0x8f, - 0x8a, 0xdd, 0x5e, 0xb6, 0x66, 0x0d, 0x99, 0x0f, 0xe0, 0xfa, 0x00, 0x31, 0xe9, 0x0d, 0x87, 0xa9, - 0x8e, 0x4a, 0x32, 0x5b, 0x82, 0xd2, 0x00, 0xb9, 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, - 0x45, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d, 0x86, 0x50, 0xf9, 0x74, 0xb0, 0xc7, 0xeb, 0x26, - 0x1e, 0xd4, 0xb2, 0xce, 0x52, 0x67, 0x6a, 0xd3, 0xaa, 0x08, 0xf8, 0xc0, 0x33, 0x3e, 0x83, 0x65, - 0x39, 0xe4, 0x9e, 0x3a, 0xc1, 0x10, 0xd9, 0x21, 0x19, 0x61, 0x57, 0x86, 0x7e, 0xeb, 0x5e, 0x4f, - 0xa5, 0x21, 0xc5, 0x67, 0x47, 0x90, 0xf4, 0x05, 0x85, 0xd5, 0x19, 0x4e, 0xa2, 0xcc, 0xbf, 0x17, - 0xa0, 0xa2, 0x8e, 0x35, 0x7e, 0x34, 0x7b, 0x14, 0x8f, 0x11, 0x55, 0xc1, 0xae, 0x20, 0xe3, 0x75, - 0x68, 0xc9, 0x2f, 0x9b, 0x84, 0x0c, 0x93, 0xe4, 0xb0, 0x6c, 0x4a, 0xec, 0x23, 0x89, 0x14, 0xd7, - 0x96, 0xe2, 0xe2, 0x50, 0xf5, 0xe8, 0x0a, 0xe2, 0xf8, 0x93, 0x88, 0x2b, 0x25, 0x0e, 0xc7, 0x9a, - 0xa5, 0x20, 0xbe, 0xb9, 0x34, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, 0x73, 0xf9, 0x24, 0x0e, 0x98, - 0x1d, 0x12, 0x1c, 0x30, 0x75, 0x1a, 0x82, 0x40, 0xf5, 0x39, 0xc6, 0x58, 0x87, 0xea, 0x49, 0x64, - 0x0b, 0x6b, 0x44, 0xe5, 0x9b, 0x9c, 0xd0, 0xca, 0x6a, 0xab, 0x72, 0x12, 0x89, 0x0f, 0xf3, 0xd7, - 0x05, 0x58, 0x94, 0x97, 0xfc, 0x46, 0x0b, 0x8a, 0x49, 0xf5, 0x52, 0xc4, 0xa2, 0x12, 0x14, 0x5a, - 0xc9, 0x8a, 0x45, 0x7c, 0xf3, 0x1c, 0x33, 0xf6, 0xe5, 0x19, 0xac, 0x8c, 0x18, 0xfb, 0xe2, 0xf0, - 0x7d, 0x1d, 0x5a, 0x69, 0x11, 0x24, 0xc6, 0xa5, 0x31, 0xcd, 0x04, 0x2b, 0xc8, 0xe6, 0xda, 0x64, - 0xfe, 0x18, 0x20, 0xbd, 0xec, 0xe6, 0xe1, 0x10, 0x27, 0xca, 0xf0, 0x4f, 0x8e, 0x19, 0x26, 0xe5, - 0x13, 0xff, 0x34, 0x6e, 0x43, 0xcb, 0xf1, 0x3c, 0xcc, 0xa7, 0x3b, 0xa3, 0x3d, 0xec, 0x25, 0x09, - 0x24, 0x8f, 0x35, 0xff, 0x52, 0x80, 0xf6, 0x0e, 0x09, 0x2f, 0x3e, 0xc5, 0x23, 0x94, 0xc9, 0x6e, - 0x42, 0x49, 0x55, 0x3d, 0xf1, 0x6f, 0xde, 0x11, 0x9c, 0xe0, 0x11, 0x92, 0xdb, 0x5e, 0x46, 0x5d, - 0x95, 0x23, 0xc4, 0x96, 0xd7, 0x83, 0xc9, 0xd5, 0x66, 0x53, 0x0e, 0x1e, 0x12, 0x4f, 0xf4, 0x3e, - 0x1e, 0xa6, 0x76, 0x72, 0x91, 0xd9, 0xb4, 0x2a, 0x1e, 0xa6, 0x62, 0x48, 0x19, 0xb2, 0x20, 0x2e, - 0xaa, 0xb3, 0x86, 0x2c, 0x4a, 0x0c, 0x37, 0x64, 0x15, 0x16, 0xc9, 0xc9, 0x49, 0x84, 0x98, 0x58, - 0xab, 0x92, 0xa5, 0xa0, 0x24, 0x05, 0x57, 0x33, 0x29, 0x78, 0x45, 0x9c, 0x6b, 0x8f, 0x1e, 0x1d, - 0xee, 0x8e, 0x51, 0xc0, 0xf4, 0x09, 0x7c, 0x17, 0xaa, 0x1a, 0xf5, 0x43, 0xae, 0x80, 0xdf, 0x84, - 0xd6, 0x96, 0xe7, 0x0d, 0x9e, 0x38, 0xa1, 0xf6, 0x47, 0x17, 0x2a, 0xfd, 0x9d, 0x83, 0xbe, 0x74, - 0x49, 0x89, 0x1b, 0xa0, 0x40, 0x7e, 0xe2, 0xef, 0x21, 0x76, 0x88, 0x18, 0xc5, 0x6e, 0x72, 0xe2, - 0xdf, 0x82, 0x8a, 0xc2, 0xf0, 0x99, 0xbe, 0xfc, 0xd4, 0x47, 0x80, 0x02, 0xcd, 0xff, 0x03, 0xe3, - 0x47, 0xbc, 0x76, 0x45, 0xb2, 0x71, 0x51, 0x92, 0xde, 0x84, 0xce, 0x58, 0x60, 0x6d, 0x59, 0xd4, - 0x65, 0x96, 0xa1, 0x2d, 0x07, 0x44, 0x7e, 0x10, 0xb2, 0x8f, 0x60, 0x59, 0x96, 0xda, 0x92, 0xcf, - 0x15, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6, 0xc4, 0xf7, 0xbd, 0xa7, 0x86, 0x3a, 0xc6, 0xd4, - 0xcd, 0x8e, 0xb1, 0x07, 0xed, 0x89, 0x67, 0x38, 0x43, 0x5d, 0xf5, 0xcd, 0x7e, 0x9d, 0xeb, 0xad, - 0x6e, 0xc8, 0x67, 0xbd, 0x0d, 0xfd, 0xac, 0xb7, 0xb1, 0xeb, 0x87, 0xec, 0xc2, 0xd8, 0x85, 0x56, - 0xfe, 0xc1, 0xca, 0x78, 0x51, 0x57, 0xc6, 0x33, 0x9e, 0xb1, 0xe6, 0xb2, 0xd9, 0x83, 0xf6, 0xc4, - 0xdb, 0x95, 0xd6, 0x67, 0xf6, 0x93, 0xd6, 0x5c, 0x46, 0x0f, 0xa0, 0x9e, 0x79, 0xac, 0x32, 0xba, - 0x92, 0xc9, 0xf4, 0xfb, 0xd5, 0x5c, 0x06, 0x3b, 0xd0, 0xcc, 0xbd, 0x1f, 0x19, 0x3d, 0x65, 0xcf, - 0x8c, 0x47, 0xa5, 0xb9, 0x4c, 0xb6, 0xa1, 0x9e, 0x79, 0xc6, 0xd1, 0x5a, 0x4c, 0xbf, 0x15, 0xf5, - 0x6e, 0xcc, 0x18, 0x51, 0xa7, 0xe5, 0x1e, 0xb4, 0x27, 0xde, 0x76, 0xb4, 0x4b, 0x66, 0x3f, 0xf9, - 0xcc, 0x55, 0xe6, 0x73, 0xb1, 0x44, 0x99, 0xd6, 0x3d, 0xb3, 0x44, 0xd3, 0x2f, 0x39, 0xbd, 0x97, - 0x66, 0x0f, 0x2a, 0xad, 0x76, 0xa1, 0x95, 0x7f, 0xc4, 0xd1, 0xcc, 0x66, 0x3e, 0xed, 0x5c, 0xbe, - 0xde, 0xb9, 0xf7, 0x9c, 0x74, 0xbd, 0x67, 0x3d, 0xf3, 0xcc, 0x65, 0xb4, 0x05, 0xa0, 0x1a, 0x75, - 0x0f, 0x07, 0x89, 0xa3, 0xa7, 0x2e, 0x08, 0x12, 0x47, 0xcf, 0x68, 0xea, 0x1f, 0x00, 0xc8, 0xfe, - 0xda, 0x23, 0x31, 0x33, 0xae, 0x6b, 0x35, 0x26, 0x9a, 0xfa, 0x5e, 0x77, 0x7a, 0x60, 0x8a, 0x01, - 0xa2, 0xf4, 0x2a, 0x0c, 0x3e, 0x01, 0x48, 0xfb, 0x76, 0xcd, 0x60, 0xaa, 0x93, 0xbf, 0xc4, 0x07, - 0x8d, 0x6c, 0x97, 0x6e, 0x28, 0x5b, 0x67, 0x74, 0xee, 0x97, 0xb0, 0x68, 0x4f, 0x74, 0x61, 0xf9, - 0x60, 0x9b, 0x6c, 0xce, 0x7a, 0x53, 0x9d, 0x98, 0x71, 0x1f, 0x1a, 0xd9, 0xf6, 0x4b, 0x6b, 0x31, - 0xa3, 0x25, 0xeb, 0xe5, 0x5a, 0x30, 0xe3, 0x01, 0xb4, 0xf2, 0xad, 0x97, 0x0e, 0xa9, 0x99, 0x0d, - 0x59, 0x4f, 0x5d, 0x2c, 0x66, 0xc8, 0xdf, 0x05, 0x48, 0x5b, 0x34, 0xed, 0xbe, 0xa9, 0xa6, 0x6d, - 0x42, 0xea, 0x1e, 0xb4, 0x27, 0x5a, 0x2f, 0x6d, 0xf1, 0xec, 0x8e, 0xec, 0xb2, 0xbd, 0x9e, 0x69, - 0xa4, 0x74, 0x08, 0x4e, 0xb7, 0x62, 0x3a, 0x04, 0x67, 0x75, 0x5d, 0xdb, 0x50, 0x1f, 0x4c, 0xf3, - 0x18, 0xcc, 0xe5, 0x31, 0xab, 0x97, 0x7a, 0x0f, 0x20, 0x3d, 0xb6, 0xb4, 0x17, 0xa6, 0x0e, 0xb2, - 0x5e, 0x53, 0x5f, 0xfe, 0x4a, 0xba, 0x1d, 0x68, 0xe6, 0xee, 0x47, 0x74, 0xba, 0x9b, 0x75, 0x69, - 0x72, 0xd9, 0x21, 0x90, 0xbf, 0x4c, 0xd0, 0x2b, 0x38, 0xf3, 0x8a, 0xe1, 0xb2, 0x38, 0xce, 0x76, - 0x7e, 0x3a, 0x82, 0x66, 0x74, 0x83, 0xdf, 0x93, 0x57, 0xb2, 0xdd, 0x5d, 0x26, 0xaf, 0xcc, 0x68, - 0xfa, 0xe6, 0x32, 0xda, 0x87, 0xf6, 0x9e, 0x2e, 0xdc, 0x55, 0x53, 0xa1, 0xd7, 0x6f, 0xba, 0x89, - 0xea, 0xf5, 0x66, 0x0d, 0xa9, 0x75, 0xf9, 0x1c, 0x3a, 0x53, 0x0d, 0x85, 0xb1, 0x96, 0x5c, 0xc1, - 0xcf, 0xec, 0x34, 0xe6, 0xaa, 0x75, 0x00, 0x4b, 0x93, 0xfd, 0x84, 0xf1, 0x72, 0x12, 0x13, 0xb3, - 0xfa, 0x8c, 0xb9, 0xac, 0x3e, 0x84, 0xaa, 0xae, 0x11, 0x0d, 0xf5, 0xd4, 0x31, 0x51, 0x33, 0xce, - 0x9d, 0x7a, 0x5f, 0x84, 0x7c, 0x52, 0x7f, 0xa5, 0x21, 0x3f, 0x51, 0xa5, 0xf5, 0xd4, 0xcb, 0x44, - 0x42, 0x79, 0x1f, 0x2a, 0xaa, 0x0c, 0x33, 0x56, 0x92, 0xcd, 0x96, 0xa9, 0xca, 0x2e, 0x8b, 0xb0, - 0x3d, 0xc4, 0x32, 0xc5, 0x95, 0x16, 0x3a, 0x5d, 0x6f, 0xe9, 0x3d, 0x92, 0x1b, 0x51, 0x6b, 0xb1, - 0x05, 0x8d, 0x6c, 0x79, 0xa5, 0x97, 0x74, 0x46, 0xc9, 0x35, 0x4f, 0x93, 0xed, 0xf3, 0xa7, 0xdf, - 0xad, 0xbd, 0xf0, 0xb7, 0xef, 0xd6, 0x5e, 0xf8, 0xd5, 0xb3, 0xb5, 0xc2, 0xd3, 0x67, 0x6b, 0x85, - 0xbf, 0x3e, 0x5b, 0x2b, 0xfc, 0xf3, 0xd9, 0x5a, 0xe1, 0x27, 0x3f, 0xff, 0x0f, 0xff, 0x73, 0x45, - 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x8e, 0x31, 0x65, 0x99, 0xa1, 0xf0, 0x6c, 0x28, 0xff, 0x78, 0x95, - 0xf9, 0x3f, 0x16, 0xd7, 0xf2, 0x78, 0x51, 0xc0, 0xef, 0xfe, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x77, - 0xb9, 0x27, 0x67, 0xdc, 0x25, 0x00, 0x00, + // 3249 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0xcb, 0x72, 0x1c, 0x47, + 0x72, 0x9a, 0x07, 0x30, 0x33, 0x39, 0x2f, 0x4c, 0x0f, 0x08, 0x0e, 0x47, 0x14, 0x44, 0x35, 0x25, + 0x0a, 0x94, 0x4c, 0x40, 0xa2, 0x14, 0xa2, 0x1e, 0x21, 0xd3, 0x00, 0x08, 0x01, 0x90, 0x04, 0x71, + 0xdc, 0x43, 0x58, 0x0e, 0x3b, 0xec, 0x8e, 0x46, 0x77, 0x61, 0xa6, 0x84, 0xe9, 0xae, 0x56, 0x75, + 0x35, 0x08, 0xc8, 0x11, 0x0e, 0x9f, 0xec, 0x9b, 0x8f, 0xbe, 0xe9, 0x07, 0x1c, 0xfe, 0x03, 0x5f, + 0x7d, 0x50, 0xf8, 0xb4, 0xc7, 0xbd, 0xec, 0xc6, 0x8a, 0x9f, 0xb0, 0x5f, 0xb0, 0x51, 0xaf, 0x7e, + 0xcc, 0x03, 0xd2, 0x22, 0x18, 0xb1, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x56, 0x56, 0x66, + 0xd5, 0xc0, 0x60, 0x84, 0xd9, 0x38, 0x3e, 0xd9, 0x74, 0x89, 0xbf, 0x75, 0xe6, 0x30, 0xe7, 0x81, + 0x4b, 0x02, 0xe6, 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x35, 0xc1, 0x27, 0xd1, 0x56, + 0x48, 0x09, 0x23, 0x2e, 0x99, 0xa8, 0xaf, 0x68, 0xcb, 0x19, 0xa1, 0x80, 0x6d, 0x0a, 0xc0, 0x28, + 0x8f, 0x68, 0xe8, 0xf6, 0x6b, 0xc4, 0xc5, 0x12, 0xd1, 0xaf, 0xb9, 0x91, 0xfe, 0xac, 0xb3, 0xcb, + 0x10, 0x45, 0x0a, 0x78, 0x75, 0x44, 0xc8, 0x68, 0x82, 0x24, 0x8f, 0x93, 0xf8, 0x74, 0x0b, 0xf9, + 0x21, 0xbb, 0x94, 0x83, 0xe6, 0x8f, 0x45, 0x58, 0xdb, 0xa5, 0xc8, 0x61, 0x68, 0x57, 0x2b, 0x60, + 0xa1, 0xef, 0x63, 0x14, 0x31, 0xe3, 0x0d, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x15, 0xee, 0x14, + 0x36, 0x6a, 0x56, 0x3d, 0xc1, 0x1d, 0x7a, 0xc6, 0x4d, 0xa8, 0xa0, 0x0b, 0xe4, 0xf2, 0xd1, 0xa2, + 0x18, 0x5d, 0xe6, 0xe0, 0xa1, 0x67, 0xbc, 0x0f, 0xf5, 0x88, 0x51, 0x1c, 0x8c, 0xec, 0x38, 0x42, + 0xb4, 0x57, 0xba, 0x53, 0xd8, 0xa8, 0x3f, 0x5c, 0xd9, 0xe4, 0x2a, 0x6f, 0x0e, 0xc5, 0xc0, 0x71, + 0x84, 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x83, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x5e, 0xf9, + 0x4e, 0x69, 0xa3, 0xfe, 0xb0, 0x21, 0xc9, 0x9f, 0x08, 0xa4, 0xa5, 0x07, 0x8d, 0xfb, 0x50, 0x8d, + 0x18, 0xa1, 0xce, 0x08, 0x45, 0xbd, 0x25, 0x41, 0xd8, 0xd4, 0x7c, 0x05, 0xd6, 0x4a, 0x86, 0x8d, + 0xdb, 0x50, 0x7a, 0xba, 0x7b, 0xd8, 0x5b, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, + 0xb8, 0x0b, 0xcd, 0xc8, 0x09, 0xbc, 0x13, 0x72, 0x61, 0x87, 0xd8, 0x0b, 0xa2, 0x5e, 0xe5, 0x4e, + 0x61, 0xa3, 0x6a, 0x35, 0x14, 0x72, 0xc0, 0x71, 0xe6, 0xa7, 0x70, 0x63, 0xc8, 0x1c, 0xca, 0xae, + 0xe1, 0x1d, 0xf3, 0x18, 0xd6, 0x2c, 0xe4, 0x93, 0xf3, 0x6b, 0xb9, 0xb6, 0x07, 0x15, 0x86, 0x7d, + 0x44, 0x62, 0x26, 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0x4f, 0x01, 0x8c, 0xbd, 0x0b, 0xe4, 0x0e, + 0x28, 0x71, 0x51, 0x14, 0xfd, 0x85, 0x96, 0xeb, 0x6d, 0xa8, 0x84, 0x52, 0x81, 0x5e, 0x59, 0x90, + 0xab, 0x55, 0xd0, 0x5a, 0xe9, 0x51, 0xf3, 0x3b, 0x58, 0x1d, 0xe2, 0x51, 0xe0, 0x4c, 0x5e, 0xa2, + 0xbe, 0x6b, 0xb0, 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0x73, 0x00, 0xc6, 0xb7, 0x0e, + 0x66, 0x2f, 0x4f, 0x92, 0xf9, 0x00, 0xba, 0x39, 0x8e, 0x51, 0x48, 0x82, 0x08, 0x09, 0x05, 0x98, + 0xc3, 0xe2, 0x48, 0x30, 0x5b, 0xb2, 0x14, 0x64, 0x12, 0x58, 0x3b, 0x0e, 0xbd, 0x6b, 0xee, 0xa6, + 0x87, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x7b, 0xa0, 0x28, 0x9c, 0xba, 0x2a, 0x9d, 0xfa, 0x35, + 0x0e, 0xe2, 0x0b, 0x4b, 0x8f, 0x59, 0x29, 0x99, 0x8a, 0x4f, 0x16, 0x5d, 0x27, 0x3e, 0x3f, 0x85, + 0x1b, 0x03, 0x27, 0x8e, 0xae, 0xa3, 0xab, 0xf9, 0x19, 0x8f, 0xed, 0x28, 0xf6, 0xaf, 0x35, 0xf9, + 0xbf, 0x0b, 0x50, 0xdd, 0x0d, 0xe3, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0x75, 0xa8, 0x33, 0xc2, 0x9c, + 0x89, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, 0x49, 0xf0, 0x06, 0x34, 0x42, 0x44, 0xdd, + 0x30, 0x56, 0x14, 0xc5, 0x3b, 0xa5, 0x8d, 0xb2, 0x55, 0x97, 0x38, 0x49, 0xb2, 0x09, 0x5d, 0x31, + 0x66, 0xe3, 0xc0, 0x3e, 0x43, 0x34, 0x40, 0x13, 0x9f, 0x78, 0x48, 0x04, 0x47, 0xd9, 0xea, 0x88, + 0xa1, 0xc3, 0xe0, 0xab, 0x64, 0xc0, 0x78, 0x07, 0x3a, 0x09, 0x3d, 0x8f, 0x78, 0x41, 0x5d, 0x16, + 0xd4, 0x6d, 0x45, 0x7d, 0xac, 0xd0, 0xe6, 0xbf, 0x42, 0xeb, 0xd9, 0x98, 0x12, 0xc6, 0x26, 0x38, + 0x18, 0x3d, 0x71, 0x98, 0xc3, 0xb7, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, + 0x2e, 0x74, 0x98, 0xa4, 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x25, 0x19, 0x18, 0x28, 0xe2, + 0xb7, 0xa0, 0x95, 0x12, 0xf3, 0xcd, 0xad, 0xf4, 0x6d, 0x26, 0xd8, 0x67, 0xd8, 0x47, 0xe6, 0xb9, + 0xf0, 0x95, 0x58, 0x64, 0xe3, 0x5d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x08, 0x69, 0xc9, 0x08, 0xd1, + 0xee, 0xb4, 0xaa, 0x89, 0x53, 0x3e, 0x87, 0x36, 0x4b, 0x14, 0xb7, 0x3d, 0x87, 0x39, 0xf9, 0xa0, + 0xca, 0x5b, 0x65, 0xb5, 0x58, 0x0e, 0x36, 0x3f, 0x83, 0xda, 0x00, 0x7b, 0x91, 0x14, 0xdc, 0x83, + 0x8a, 0x1b, 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x0a, 0x4b, 0x13, 0xec, 0x63, 0xa6, + 0xcc, 0x94, 0x80, 0x49, 0x00, 0x8e, 0x90, 0x4f, 0xe8, 0xa5, 0x70, 0xd8, 0x2a, 0x2c, 0x65, 0x17, + 0x57, 0x02, 0xc6, 0xab, 0x50, 0xf3, 0x9d, 0x8b, 0x64, 0x51, 0xf9, 0x48, 0xd5, 0x77, 0x2e, 0xa4, + 0xf2, 0x3d, 0xa8, 0x9c, 0x3a, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x72, 0x56, + 0xe0, 0xff, 0x15, 0xa1, 0x2e, 0x25, 0x4a, 0x85, 0x57, 0x61, 0xc9, 0x75, 0xdc, 0x71, 0x22, 0x52, + 0x00, 0xc6, 0x3d, 0xad, 0x48, 0x31, 0x9b, 0xe1, 0x52, 0x4d, 0xb5, 0x6a, 0x5b, 0x00, 0xd1, 0x73, + 0x27, 0x54, 0xba, 0x95, 0x16, 0x10, 0xd7, 0x38, 0x8d, 0x54, 0xf7, 0x03, 0x68, 0xc8, 0xb8, 0x53, + 0x53, 0xca, 0x0b, 0xa6, 0xd4, 0x25, 0x95, 0x9c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x31, 0x46, + 0xd4, 0xa1, 0xee, 0xf8, 0xb2, 0xb7, 0x24, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0x3c, 0x84, + 0x25, 0x9e, 0x5b, 0xa2, 0xde, 0xb2, 0x38, 0xeb, 0x6e, 0x67, 0x59, 0x0a, 0x53, 0x37, 0xc5, 0xef, + 0x5e, 0xc0, 0xe8, 0xa5, 0x25, 0x49, 0xfb, 0x1f, 0x03, 0xa4, 0x48, 0x63, 0x05, 0x4a, 0x67, 0xe8, + 0x52, 0xed, 0x43, 0xfe, 0xc9, 0x9d, 0x73, 0xee, 0x4c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x5a, 0xfc, + 0xb8, 0x60, 0xba, 0xd0, 0xde, 0x99, 0x9c, 0x61, 0x92, 0x99, 0xbe, 0x0a, 0x4b, 0xbe, 0xf3, 0x1d, + 0xa1, 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84, + 0xc2, 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x7d, 0x19, 0x20, 0x95, + 0x62, 0x58, 0xd0, 0xc7, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x72, 0xc9, 0x50, 0x64, 0x53, + 0xe4, 0xc6, 0x34, 0xc2, 0xe7, 0x7c, 0xfd, 0xb8, 0xd9, 0x37, 0xa4, 0xd9, 0x53, 0xba, 0x59, 0x37, + 0x31, 0x19, 0xca, 0x79, 0x3b, 0x7c, 0x9a, 0xa5, 0x67, 0x19, 0x87, 0x70, 0x23, 0xe5, 0xe9, 0x65, + 0xd8, 0x15, 0xaf, 0x62, 0xd7, 0x4d, 0xd8, 0x79, 0x29, 0xab, 0x3d, 0xe8, 0x62, 0x62, 0x7f, 0x1f, + 0xa3, 0x38, 0xc7, 0xa8, 0x74, 0x15, 0xa3, 0x0e, 0x26, 0x7f, 0x2b, 0x26, 0xa4, 0x6c, 0x06, 0x70, + 0x2b, 0x63, 0x25, 0xdf, 0xee, 0x19, 0x66, 0xe5, 0xab, 0x98, 0xad, 0x25, 0x5a, 0xf1, 0x7c, 0x90, + 0x72, 0xfc, 0x12, 0xd6, 0x30, 0xb1, 0x9f, 0x3b, 0x98, 0x4d, 0xb3, 0x5b, 0xfa, 0x05, 0x23, 0xf9, + 0x89, 0x96, 0xe7, 0x25, 0x8d, 0xf4, 0x11, 0x1d, 0xe5, 0x8c, 0x5c, 0xfe, 0x05, 0x23, 0x8f, 0xc4, + 0x84, 0x94, 0xcd, 0x36, 0x74, 0x30, 0x99, 0xd6, 0xa6, 0x72, 0x15, 0x93, 0x36, 0x26, 0x79, 0x4d, + 0x76, 0xa0, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x0d, 0x82, 0xea, 0x55, 0x2c, 0x56, 0x14, 0x7d, 0xc2, + 0xc3, 0xfc, 0x47, 0x68, 0x1c, 0xc4, 0x23, 0xc4, 0x26, 0x27, 0x49, 0x32, 0x78, 0x69, 0xf9, 0xc7, + 0xfc, 0x63, 0x11, 0xea, 0xbb, 0x23, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xce, 0xc9, + 0x82, 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x21, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d, + 0x99, 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x04, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d, + 0xb5, 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x3e, 0xd4, 0x4f, 0xb8, 0x93, + 0xd4, 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x49, 0xba, 0x09, 0x0f, 0xa0, 0x39, 0x96, 0x2e, + 0x53, 0x93, 0x64, 0x0c, 0xdd, 0x55, 0x96, 0xa4, 0xf6, 0x6e, 0x66, 0x3d, 0x2b, 0x17, 0xa0, 0x31, + 0xce, 0xa0, 0xfa, 0x43, 0xe8, 0xcc, 0x90, 0xcc, 0xc9, 0x41, 0x1b, 0xd9, 0x1c, 0x54, 0x7f, 0x68, + 0x48, 0x41, 0xd9, 0x99, 0xd9, 0xbc, 0xf4, 0x9f, 0x45, 0x68, 0x7c, 0x83, 0xd8, 0x73, 0x42, 0xcf, + 0xa4, 0xbe, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa3, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e, 0xc8, + 0x04, 0xa2, 0xd6, 0xb3, 0x42, 0x2f, 0x44, 0x62, 0x30, 0x5e, 0x03, 0xa0, 0x17, 0x76, 0xe8, 0xb8, + 0x67, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xc5, 0x40, 0x22, 0x78, 0x28, 0xd0, 0x0b, 0x1b, 0x51, + 0x4a, 0x68, 0xa4, 0x72, 0x55, 0x95, 0x5e, 0xec, 0x09, 0x58, 0xcd, 0xf5, 0x28, 0x09, 0x43, 0xe4, + 0x89, 0x1c, 0x2d, 0xe6, 0x3e, 0x91, 0x08, 0x2e, 0x95, 0x69, 0xa9, 0xcb, 0x52, 0x2a, 0x4b, 0xa5, + 0xb2, 0x54, 0x6a, 0x45, 0xce, 0x64, 0x59, 0xa9, 0x2c, 0x91, 0x5a, 0x95, 0x52, 0x59, 0x46, 0x2a, + 0x4b, 0xa5, 0xd6, 0xf4, 0x5c, 0x25, 0xd5, 0xfc, 0x8f, 0x02, 0xac, 0x4d, 0x17, 0x7e, 0xaa, 0x36, + 0xfd, 0x10, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0x66, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30, + 0x7e, 0x04, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, + 0x03, 0x99, 0x1e, 0x18, 0xdf, 0x52, 0xcc, 0xd0, 0x90, 0x51, 0xe4, 0xf8, 0x2f, 0xa3, 0xba, 0x37, + 0xa0, 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0x43, 0x37, 0x27, 0x45, 0xd9, + 0xba, 0x02, 0xa5, 0x09, 0x0a, 0x04, 0xf7, 0xa6, 0xc5, 0x3f, 0x4d, 0x07, 0x3a, 0x16, 0x72, 0xbc, + 0x97, 0xa7, 0x8d, 0x12, 0x51, 0x4a, 0x45, 0x6c, 0x80, 0x91, 0x15, 0xa1, 0x54, 0xd1, 0x5a, 0x17, + 0x32, 0x5a, 0x3f, 0x85, 0xce, 0xee, 0x84, 0x44, 0x68, 0xc8, 0x3c, 0x1c, 0xbc, 0x8c, 0x76, 0xe4, + 0x5f, 0xa0, 0xfb, 0x8c, 0x5d, 0x7e, 0xcb, 0x99, 0x45, 0xf8, 0x07, 0xf4, 0x92, 0xec, 0xa3, 0xe4, + 0xb9, 0xb6, 0x8f, 0x92, 0xe7, 0xbc, 0xb9, 0x71, 0xc9, 0x24, 0xf6, 0x03, 0xb1, 0x15, 0x9a, 0x96, + 0x82, 0xcc, 0x1d, 0x68, 0xc8, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0xcd, 0xdd, 0x83, 0xeb, 0x00, + 0xa1, 0x43, 0x1d, 0x1f, 0x31, 0x44, 0x65, 0x0c, 0xd5, 0xac, 0x0c, 0xc6, 0xfc, 0xaf, 0x22, 0xac, + 0xca, 0xfb, 0x86, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x7d, 0xa8, 0x8e, 0x49, 0xc4, 0x32, 0x0c, 0x13, + 0x98, 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0xe6, 0x2e, 0x01, 0x4a, 0x57, 0x5f, 0x02, 0xcc, + 0xb4, 0xf9, 0xe5, 0xd9, 0x36, 0x9f, 0xef, 0x36, 0x4d, 0x84, 0xe5, 0x1e, 0xaf, 0x59, 0x35, 0x85, + 0x39, 0xf4, 0x8c, 0x7b, 0xd0, 0x1e, 0x71, 0x2d, 0xed, 0x31, 0x21, 0x67, 0x76, 0xe8, 0xb0, 0xb1, + 0xd8, 0xea, 0x35, 0xab, 0x29, 0xd0, 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0x3e, 0x81, 0x96, + 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, 0x1d, 0x7e, 0x6a, 0x17, 0x65, 0xbd, 0x67, 0x35, 0xcf, 0x32, + 0x50, 0x64, 0xde, 0x84, 0x1b, 0x4f, 0x50, 0xc4, 0x28, 0xb9, 0xcc, 0x3b, 0xc6, 0xfc, 0x6b, 0x80, + 0xc3, 0x80, 0x21, 0x7a, 0xea, 0xb8, 0x28, 0x32, 0xde, 0xcb, 0x42, 0xaa, 0x38, 0x5a, 0xd9, 0x94, + 0xd7, 0x3d, 0xc9, 0x80, 0x95, 0xa1, 0x31, 0x37, 0x61, 0xd9, 0x22, 0x31, 0x4f, 0x47, 0x6f, 0xea, + 0x2f, 0x35, 0xaf, 0xa1, 0xe6, 0x09, 0xa4, 0xa5, 0xc6, 0xcc, 0x03, 0xdd, 0xc2, 0xa6, 0xec, 0xd4, + 0x12, 0x6d, 0x42, 0x0d, 0x6b, 0x9c, 0xca, 0x2a, 0xb3, 0xa2, 0x53, 0x12, 0xf3, 0x33, 0xe8, 0x4a, + 0x4e, 0x92, 0xb3, 0x66, 0xf3, 0x26, 0x2c, 0x53, 0xad, 0x46, 0x21, 0xbd, 0xe7, 0x51, 0x44, 0x6a, + 0xcc, 0x3c, 0x84, 0xdb, 0x72, 0xf2, 0x5e, 0x38, 0x46, 0x3e, 0xa2, 0xce, 0xe4, 0x88, 0xc4, 0x01, + 0x4b, 0xb8, 0x64, 0x23, 0xa0, 0x70, 0x65, 0x04, 0x70, 0xd7, 0x7e, 0x8d, 0x23, 0x96, 0xfa, 0x44, + 0xbb, 0xb6, 0x0b, 0x1d, 0x3e, 0x90, 0x53, 0xcf, 0xfc, 0x02, 0x1a, 0xdb, 0xd6, 0xe0, 0x1b, 0x84, + 0x47, 0xe3, 0x13, 0x9e, 0x88, 0x3f, 0xca, 0xc3, 0x4a, 0x98, 0xa1, 0x0c, 0xcf, 0x0c, 0x59, 0x39, + 0x3a, 0xf3, 0x4b, 0x58, 0xdb, 0xf6, 0xbc, 0x2c, 0x4a, 0xab, 0xfe, 0x1e, 0xd4, 0x82, 0x0c, 0xbb, + 0xcc, 0xf1, 0x97, 0xa3, 0x4e, 0x89, 0xcc, 0x07, 0x60, 0xec, 0x23, 0x76, 0x38, 0x78, 0xe6, 0x9c, + 0x4c, 0x52, 0x47, 0xde, 0x84, 0x0a, 0x8e, 0x6c, 0x1c, 0x9e, 0x7f, 0x24, 0xb8, 0x54, 0xad, 0x65, + 0x1c, 0x1d, 0x86, 0xe7, 0x1f, 0x99, 0xf7, 0xa1, 0x9b, 0x23, 0xbf, 0x22, 0x43, 0x6d, 0x83, 0x31, + 0xfc, 0xf5, 0x9c, 0x13, 0x16, 0xc5, 0x0c, 0x8b, 0xfb, 0xd0, 0x1d, 0xfe, 0x4a, 0x69, 0xff, 0x04, + 0xdd, 0xa7, 0xc1, 0x04, 0x07, 0x68, 0x77, 0x70, 0x7c, 0x84, 0x92, 0xf4, 0x6c, 0x40, 0x99, 0x97, + 0xb1, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0x27, 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xff, 0x5a, 0x0e, + 0x4e, 0x76, 0xc3, 0x38, 0xe2, 0xe7, 0x2d, 0xaf, 0xb7, 0x48, 0x30, 0xb9, 0x14, 0x49, 0xab, 0x6a, + 0x55, 0xdc, 0x30, 0x7e, 0x1a, 0x4c, 0x2e, 0xcd, 0xbf, 0x12, 0x97, 0x12, 0x08, 0x79, 0x96, 0x13, + 0x78, 0xc4, 0x7f, 0x82, 0xce, 0x33, 0x12, 0x66, 0xf4, 0xfe, 0xa9, 0x00, 0x8d, 0xed, 0x11, 0x0a, + 0xd8, 0x13, 0xc4, 0x1c, 0x3c, 0x11, 0x4d, 0xee, 0x39, 0xa2, 0x11, 0x26, 0x81, 0xca, 0x40, 0x1a, + 0x34, 0x5e, 0x87, 0x3a, 0x0e, 0x30, 0xb3, 0x3d, 0x07, 0xf9, 0x24, 0x10, 0x5c, 0xaa, 0x16, 0x70, + 0xd4, 0x13, 0x81, 0x31, 0xde, 0x86, 0xb6, 0xbc, 0x9f, 0xb4, 0xc7, 0x4e, 0xe0, 0x4d, 0x78, 0xee, + 0x2b, 0x89, 0x6c, 0xd5, 0x92, 0xe8, 0x03, 0x85, 0x35, 0xee, 0xc3, 0x8a, 0x8a, 0xcb, 0x94, 0xb2, + 0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2, + 0x87, 0xaa, 0x43, 0x6c, 0x6b, 0xfc, 0x50, 0xa2, 0xcd, 0x11, 0x74, 0xf7, 0xb9, 0x9d, 0xca, 0x92, + 0x74, 0xa7, 0xb5, 0x7c, 0xe4, 0xdb, 0x27, 0x13, 0xe2, 0x9e, 0xd9, 0xfc, 0xbc, 0x50, 0x1e, 0xe6, + 0x35, 0xe8, 0x0e, 0x47, 0x0e, 0xf1, 0x0f, 0xe2, 0x32, 0x84, 0x53, 0x8d, 0x09, 0x0b, 0x27, 0xf1, + 0xc8, 0x0e, 0x29, 0x39, 0x41, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, + 0xbf, 0x05, 0x58, 0xcd, 0x4b, 0x52, 0xab, 0xbd, 0x05, 0xab, 0x79, 0x51, 0xaa, 0x22, 0x92, 0x15, + 0x77, 0x27, 0x2b, 0x50, 0xd6, 0x46, 0x8f, 0xa0, 0x29, 0x6e, 0xb3, 0x6d, 0x4f, 0x72, 0xca, 0xd7, + 0x81, 0xd9, 0x75, 0xb1, 0x1a, 0x4e, 0x76, 0x95, 0x3e, 0x81, 0x5b, 0xca, 0x7c, 0x7b, 0x56, 0x6d, + 0x19, 0x10, 0x6b, 0x8a, 0xe0, 0x68, 0x4a, 0xfb, 0xaf, 0xa1, 0x97, 0xa2, 0x76, 0x2e, 0x05, 0x32, + 0xdd, 0x94, 0xdd, 0x29, 0x63, 0xb7, 0x3d, 0x8f, 0x8a, 0xdd, 0x5e, 0xb6, 0xe6, 0x0d, 0x99, 0x8f, + 0xe1, 0xe6, 0x10, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xe6, 0x4c, 0x32, 0x5b, 0x81, 0xd2, 0x10, 0xb9, + 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, 0x47, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d, + 0x86, 0x50, 0xf9, 0x62, 0xb8, 0xcf, 0x4b, 0x30, 0x1e, 0xd4, 0xb2, 0x64, 0x53, 0xc7, 0x73, 0xd3, + 0xaa, 0x08, 0xf8, 0xd0, 0x33, 0xbe, 0x84, 0xae, 0x1c, 0x72, 0xc7, 0x4e, 0x30, 0x42, 0x76, 0x48, + 0x26, 0xd8, 0x95, 0xa1, 0xdf, 0x7a, 0xd8, 0x57, 0x69, 0x48, 0xf1, 0xd9, 0x15, 0x24, 0x03, 0x41, + 0x61, 0x75, 0x46, 0xd3, 0x28, 0xf3, 0x77, 0x05, 0xa8, 0xa8, 0xfc, 0xc8, 0x4f, 0x79, 0x8f, 0xe2, + 0x73, 0x44, 0x55, 0xb0, 0x2b, 0xc8, 0x78, 0x0b, 0x5a, 0xf2, 0xcb, 0x26, 0x21, 0xc3, 0x24, 0x39, + 0x77, 0x9b, 0x12, 0xfb, 0x54, 0x22, 0xc5, 0x0d, 0xa8, 0xb8, 0x83, 0x54, 0xed, 0xbe, 0x82, 0x38, + 0xfe, 0x34, 0xe2, 0x4a, 0x89, 0x73, 0xb6, 0x66, 0x29, 0x88, 0x6f, 0x2e, 0xcd, 0x6f, 0x49, 0xf0, + 0xd3, 0x20, 0xdf, 0x5c, 0x3e, 0x4f, 0xed, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0xc1, 0x0a, 0x02, 0x35, + 0xe0, 0x18, 0x63, 0x03, 0xaa, 0xa7, 0x91, 0x2d, 0xac, 0x11, 0x45, 0x74, 0x92, 0xea, 0x95, 0xd5, + 0x56, 0xe5, 0x34, 0x12, 0x1f, 0xe6, 0xbf, 0x17, 0x60, 0x59, 0xbe, 0x17, 0x18, 0x2d, 0x28, 0x26, + 0x85, 0x50, 0x11, 0x8b, 0xa2, 0x52, 0x68, 0x25, 0x8b, 0x1f, 0xf1, 0xcd, 0x73, 0xcc, 0xb9, 0x2f, + 0x8f, 0x73, 0x65, 0xc4, 0xb9, 0x2f, 0xce, 0xf1, 0xb7, 0xa0, 0x95, 0xd6, 0x53, 0x62, 0x5c, 0x1a, + 0xd3, 0x4c, 0xb0, 0x82, 0x6c, 0xa1, 0x4d, 0xe6, 0xdf, 0x03, 0xa4, 0xf7, 0xe6, 0x3c, 0x1c, 0xe2, + 0x44, 0x19, 0xfe, 0xc9, 0x31, 0xa3, 0xa4, 0x12, 0xe3, 0x9f, 0xc6, 0x3d, 0x68, 0x39, 0x9e, 0x87, + 0xf9, 0x74, 0x67, 0xb2, 0x8f, 0xbd, 0x24, 0x81, 0xe4, 0xb1, 0xe6, 0xff, 0x17, 0xa0, 0xbd, 0x4b, + 0xc2, 0xcb, 0x2f, 0xf0, 0x04, 0x65, 0xb2, 0x9b, 0x50, 0x52, 0x15, 0x62, 0xfc, 0x9b, 0x37, 0x17, + 0xa7, 0x78, 0x82, 0xe4, 0xb6, 0x97, 0x51, 0x57, 0xe5, 0x08, 0xb1, 0xe5, 0xf5, 0x60, 0x72, 0x4b, + 0xda, 0x94, 0x83, 0x47, 0xc4, 0x13, 0x6d, 0x94, 0x87, 0xa9, 0x9d, 0xdc, 0x89, 0x36, 0xad, 0x8a, + 0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x3b, 0xef, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59, + 0x83, 0x65, 0x72, 0x7a, 0x1a, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c, + 0x0a, 0x5e, 0x15, 0xe7, 0xda, 0xd3, 0xa7, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0xfa, 0x04, 0x7e, 0x00, + 0x55, 0x8d, 0xfa, 0x35, 0xb7, 0xc9, 0xef, 0x40, 0x6b, 0xdb, 0xf3, 0x86, 0xcf, 0x9d, 0x50, 0xfb, + 0xa3, 0x07, 0x95, 0xc1, 0xee, 0xe1, 0x40, 0xba, 0xa4, 0xc4, 0x0d, 0x50, 0x20, 0x3f, 0xf1, 0xf7, + 0x11, 0x3b, 0x42, 0x8c, 0x62, 0x37, 0x39, 0xf1, 0xef, 0x42, 0x45, 0x61, 0xf8, 0x4c, 0x5f, 0x7e, + 0xea, 0x23, 0x40, 0x81, 0xe6, 0xdf, 0x80, 0xf1, 0x77, 0xbc, 0x0c, 0x46, 0xb2, 0x07, 0x52, 0x92, + 0xde, 0x81, 0xce, 0xb9, 0xc0, 0xda, 0xb2, 0x3e, 0xcc, 0x2c, 0x43, 0x5b, 0x0e, 0x88, 0xfc, 0x20, + 0x64, 0x1f, 0x43, 0x57, 0x56, 0xed, 0x92, 0xcf, 0x35, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6, + 0xc4, 0xf7, 0xc3, 0x1f, 0xbb, 0xea, 0x18, 0x53, 0x97, 0x44, 0xc6, 0x3e, 0xb4, 0xa7, 0x5e, 0xf4, + 0x0c, 0x75, 0x6b, 0x38, 0xff, 0xa1, 0xaf, 0xbf, 0xb6, 0x29, 0x5f, 0x08, 0x37, 0xf5, 0x0b, 0xe1, + 0xe6, 0x9e, 0x1f, 0xb2, 0x4b, 0x63, 0x0f, 0x5a, 0xf9, 0xb7, 0x2f, 0xe3, 0x55, 0x5d, 0x62, 0xcd, + 0x79, 0x11, 0x5b, 0xc8, 0x66, 0x1f, 0xda, 0x53, 0xcf, 0x60, 0x5a, 0x9f, 0xf9, 0xaf, 0x63, 0x0b, + 0x19, 0x3d, 0x86, 0x7a, 0xe6, 0xdd, 0xcb, 0xe8, 0x49, 0x26, 0xb3, 0x4f, 0x61, 0x0b, 0x19, 0xec, + 0x42, 0x33, 0xf7, 0x14, 0x65, 0xf4, 0x95, 0x3d, 0x73, 0xde, 0xa7, 0x16, 0x32, 0xd9, 0x81, 0x7a, + 0xe6, 0x45, 0x48, 0x6b, 0x31, 0xfb, 0xec, 0xd4, 0xbf, 0x35, 0x67, 0x44, 0x9d, 0x96, 0xfb, 0xd0, + 0x9e, 0x7a, 0x26, 0xd2, 0x2e, 0x99, 0xff, 0x7a, 0xb4, 0x50, 0x99, 0x21, 0xdc, 0x98, 0x5b, 0x25, + 0x1b, 0x66, 0x96, 0xdd, 0xfc, 0x12, 0x7a, 0x21, 0xd3, 0xaf, 0xc4, 0xba, 0x67, 0xae, 0x16, 0x32, + 0xeb, 0x3e, 0xfb, 0xd2, 0xd4, 0xbf, 0x3d, 0x7f, 0x50, 0x99, 0xba, 0x07, 0xad, 0xfc, 0x23, 0x93, + 0x66, 0x36, 0xf7, 0xe9, 0xe9, 0xea, 0x20, 0xca, 0xbd, 0x37, 0xa5, 0x41, 0x34, 0xef, 0x19, 0x6a, + 0x21, 0xa3, 0x6d, 0x00, 0x75, 0x91, 0xe0, 0xe1, 0x20, 0x59, 0xbd, 0x99, 0x0b, 0x8c, 0x64, 0xf5, + 0xe6, 0x5c, 0x3a, 0x3c, 0x06, 0x90, 0xfd, 0xbf, 0x47, 0x62, 0x66, 0xdc, 0xd4, 0x6a, 0x4c, 0x5d, + 0x3a, 0xf4, 0x7b, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x39, 0x40, 0x7a, 0xaf, + 0xa0, 0x19, 0xcc, 0xdc, 0x34, 0x5c, 0xe1, 0x83, 0x46, 0xf6, 0x16, 0xc1, 0x50, 0xb6, 0xce, 0xb9, + 0x59, 0xb8, 0x82, 0x45, 0x7b, 0xaa, 0x4b, 0xcc, 0x47, 0xf0, 0x74, 0xf3, 0xd8, 0x9f, 0xe9, 0x14, + 0x8d, 0x47, 0xd0, 0xc8, 0xb6, 0x87, 0x5a, 0x8b, 0x39, 0x2d, 0x63, 0x3f, 0xd7, 0x22, 0x1a, 0x8f, + 0xa1, 0x95, 0xef, 0xe7, 0x74, 0x48, 0xcd, 0xed, 0xf2, 0xfa, 0xea, 0xe2, 0x33, 0x43, 0xfe, 0x01, + 0x40, 0xda, 0xf7, 0x69, 0xf7, 0xcd, 0x74, 0x82, 0x53, 0x52, 0xf7, 0xa1, 0x3d, 0xd5, 0xcf, 0x69, + 0x8b, 0xe7, 0xb7, 0x79, 0x57, 0x25, 0x90, 0x4c, 0x77, 0xa6, 0x43, 0x70, 0xb6, 0xbf, 0xd3, 0x21, + 0x38, 0xaf, 0x95, 0xdb, 0x81, 0xfa, 0x70, 0x96, 0xc7, 0x70, 0x21, 0x8f, 0x79, 0x0d, 0xda, 0x87, + 0x00, 0xe9, 0x59, 0xa8, 0xbd, 0x30, 0x73, 0x3a, 0xf6, 0x9b, 0xfa, 0x72, 0x5a, 0xd2, 0xed, 0x42, + 0x33, 0x77, 0x7f, 0xa3, 0x73, 0xe8, 0xbc, 0x4b, 0x9d, 0xab, 0x4e, 0x96, 0xfc, 0x65, 0x87, 0x5e, + 0xc1, 0xb9, 0x57, 0x20, 0x57, 0xc5, 0x71, 0xb6, 0x9d, 0xd4, 0x11, 0x34, 0xa7, 0xc5, 0xfc, 0x85, + 0xbc, 0x92, 0x6d, 0x19, 0x33, 0x79, 0x65, 0x4e, 0x27, 0xb9, 0x90, 0xd1, 0x01, 0xb4, 0xf7, 0x75, + 0x37, 0xa0, 0x3a, 0x15, 0xbd, 0x7e, 0xb3, 0x9d, 0x59, 0xbf, 0x3f, 0x6f, 0x48, 0xad, 0xcb, 0x57, + 0xd0, 0x99, 0xe9, 0x52, 0x8c, 0xf5, 0xe4, 0x89, 0x60, 0x6e, 0xfb, 0xb2, 0x50, 0xad, 0x43, 0x58, + 0x99, 0x6e, 0x52, 0x8c, 0xd7, 0x92, 0x98, 0x98, 0xd7, 0xbc, 0x2c, 0x64, 0xf5, 0x09, 0x54, 0x75, + 0xe1, 0x69, 0xa8, 0xa7, 0x98, 0xa9, 0x42, 0x74, 0xe1, 0xd4, 0x47, 0x22, 0xe4, 0x93, 0xa2, 0x2e, + 0x0d, 0xf9, 0xa9, 0xd2, 0xaf, 0xaf, 0x5e, 0x4e, 0x12, 0xca, 0x47, 0x50, 0x51, 0xb5, 0x9d, 0xb1, + 0x9a, 0x6c, 0xb6, 0x4c, 0xa9, 0x77, 0x55, 0x84, 0xed, 0x23, 0x96, 0xa9, 0xd8, 0xb4, 0xd0, 0xd9, + 0x22, 0x4e, 0xef, 0x91, 0xdc, 0x88, 0x5a, 0x8b, 0x6d, 0x68, 0x64, 0x6b, 0x36, 0xbd, 0xa4, 0x73, + 0xea, 0xb8, 0x45, 0x9a, 0xec, 0x5c, 0xfc, 0xf4, 0xf3, 0xfa, 0x2b, 0xbf, 0xfd, 0x79, 0xfd, 0x95, + 0x7f, 0x7b, 0xb1, 0x5e, 0xf8, 0xe9, 0xc5, 0x7a, 0xe1, 0x37, 0x2f, 0xd6, 0x0b, 0x7f, 0x78, 0xb1, + 0x5e, 0xf8, 0x87, 0x7f, 0xfe, 0x33, 0xff, 0x13, 0x46, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x9d, 0x63, + 0xca, 0x32, 0x43, 0xe1, 0xd9, 0x48, 0xfe, 0x31, 0x2c, 0xf3, 0x7f, 0x31, 0xae, 0xe5, 0xc9, 0xb2, + 0x80, 0x3f, 0xf8, 0x53, 0x00, 0x00, 0x00, 0xff, 0xff, 0xeb, 0xb0, 0x39, 0x1d, 0x7c, 0x26, 0x00, + 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -4858,6 +4901,47 @@ func (m *UpdateRoutesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *UpdateEphemeralMountsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UpdateEphemeralMountsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *UpdateEphemeralMountsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Storages) > 0 { + for iNdEx := len(m.Storages) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Storages[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintAgent(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *ListInterfacesRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6901,6 +6985,24 @@ func (m *UpdateRoutesRequest) Size() (n int) { return n } +func (m *UpdateEphemeralMountsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Storages) > 0 { + for _, e := range m.Storages { + l = e.Size() + n += 1 + l + sovAgent(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *ListInterfacesRequest) Size() (n int) { if m == nil { return 0 @@ -7664,7 +7766,7 @@ func (this *MemoryStats) String() string { return "nil" } keysForStats := make([]string, 0, len(this.Stats)) - for k := range this.Stats { + for k, _ := range this.Stats { keysForStats = append(keysForStats, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForStats) @@ -7775,7 +7877,7 @@ func (this *CgroupStats) String() string { return "nil" } keysForHugetlbStats := make([]string, 0, len(this.HugetlbStats)) - for k := range this.HugetlbStats { + for k, _ := range this.HugetlbStats { keysForHugetlbStats = append(keysForHugetlbStats, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForHugetlbStats) @@ -8008,6 +8110,22 @@ func (this *UpdateRoutesRequest) String() string { }, "") return s } +func (this *UpdateEphemeralMountsRequest) String() string { + if this == nil { + return "nil" + } + repeatedStringForStorages := "[]*Storage{" + for _, f := range this.Storages { + repeatedStringForStorages += strings.Replace(f.String(), "Storage", "Storage", 1) + "," + } + repeatedStringForStorages += "}" + s := strings.Join([]string{`&UpdateEphemeralMountsRequest{`, + `Storages:` + repeatedStringForStorages + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func (this *ListInterfacesRequest) String() string { if this == nil { return "nil" @@ -8355,6 +8473,7 @@ type AgentServiceService interface { SignalProcess(ctx context.Context, req *SignalProcessRequest) (*types.Empty, error) WaitProcess(ctx context.Context, req *WaitProcessRequest) (*WaitProcessResponse, error) UpdateContainer(ctx context.Context, req *UpdateContainerRequest) (*types.Empty, error) + UpdateEphemeralMounts(ctx context.Context, req *UpdateEphemeralMountsRequest) (*types.Empty, error) StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error) PauseContainer(ctx context.Context, req *PauseContainerRequest) (*types.Empty, error) ResumeContainer(ctx context.Context, req *ResumeContainerRequest) (*types.Empty, error) @@ -8436,6 +8555,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag } return svc.UpdateContainer(ctx, &req) }, + "UpdateEphemeralMounts": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req UpdateEphemeralMountsRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.UpdateEphemeralMounts(ctx, &req) + }, "StatsContainer": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { var req StatsContainerRequest if err := unmarshal(&req); err != nil { @@ -8701,6 +8827,14 @@ func (c *agentServiceClient) UpdateContainer(ctx context.Context, req *UpdateCon return &resp, nil } +func (c *agentServiceClient) UpdateEphemeralMounts(ctx context.Context, req *UpdateEphemeralMountsRequest) (*types.Empty, error) { + var resp types.Empty + if err := c.client.Call(ctx, "grpc.AgentService", "UpdateEphemeralMounts", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + func (c *agentServiceClient) StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error) { var resp StatsContainerResponse if err := c.client.Call(ctx, "grpc.AgentService", "StatsContainer", req, &resp); err != nil { @@ -13934,6 +14068,91 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *UpdateEphemeralMountsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UpdateEphemeralMountsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UpdateEphemeralMountsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Storages", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Storages = append(m.Storages, &Storage{}) + if err := m.Storages[len(m.Storages)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ListInterfacesRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go index f06524697..255f5ce45 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go @@ -423,7 +423,13 @@ type Hooks struct { // Poststart is a list of hooks to be run after the container process is started. Poststart []Hook `protobuf:"bytes,2,rep,name=Poststart,proto3" json:"Poststart"` // Poststop is a list of hooks to be run after the container process exits. - Poststop []Hook `protobuf:"bytes,3,rep,name=Poststop,proto3" json:"Poststop"` + Poststop []Hook `protobuf:"bytes,3,rep,name=Poststop,proto3" json:"Poststop"` + // Createruntime is a list of hooks to be run during the creation of runtime(sandbox). + CreateRuntime []Hook `protobuf:"bytes,4,rep,name=CreateRuntime,proto3" json:"CreateRuntime"` + // CreateContainer is a list of hooks to be run after VM is started, and before container is created. + CreateContainer []Hook `protobuf:"bytes,5,rep,name=CreateContainer,proto3" json:"CreateContainer"` + // StartContainer is a list of hooks to be run after container is created, but before it is started. + StartContainer []Hook `protobuf:"bytes,6,rep,name=StartContainer,proto3" json:"StartContainer"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1390,6 +1396,7 @@ type LinuxSyscall struct { Names []string `protobuf:"bytes,1,rep,name=Names,proto3" json:"Names,omitempty"` Action string `protobuf:"bytes,2,opt,name=Action,proto3" json:"Action,omitempty"` // Types that are valid to be assigned to ErrnoRet: + // // *LinuxSyscall_Errnoret ErrnoRet isLinuxSyscall_ErrnoRet `protobuf_oneof:"ErrnoRet"` Args []LinuxSeccompArg `protobuf:"bytes,4,rep,name=Args,proto3" json:"Args"` @@ -1546,140 +1553,143 @@ func init() { } var fileDescriptor_82a9ef0098ca0b24 = []byte{ - // 2121 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0xcb, 0x73, 0x5b, 0x49, - 0xd5, 0xcf, 0x95, 0x64, 0x59, 0x6a, 0xc5, 0x79, 0xf4, 0x64, 0x32, 0xf7, 0xcb, 0x97, 0xd2, 0x78, - 0x2e, 0x29, 0x30, 0x10, 0xac, 0x22, 0xe1, 0x31, 0x0c, 0x8f, 0x2a, 0xd9, 0x4e, 0x62, 0xd5, 0xc4, - 0xb1, 0x68, 0xd9, 0x13, 0x98, 0xc5, 0x54, 0xb5, 0xaf, 0xda, 0x52, 0x8f, 0xaf, 0x6e, 0xdf, 0xea, - 0xdb, 0xb2, 0x63, 0x56, 0xb0, 0x63, 0xcf, 0x82, 0x35, 0x1b, 0x1e, 0xff, 0x01, 0xc5, 0x8a, 0x1d, - 0x29, 0x56, 0x2c, 0xa9, 0xa2, 0x0a, 0x88, 0xf7, 0xec, 0x59, 0x52, 0xa7, 0xfb, 0xdc, 0xab, 0x96, - 0xe4, 0xc0, 0x04, 0x56, 0xea, 0xf3, 0x3b, 0x8f, 0x7e, 0x9c, 0xe7, 0x15, 0x79, 0x36, 0x92, 0x66, - 0x3c, 0x3d, 0xda, 0x8c, 0xd5, 0xa4, 0x73, 0xc2, 0x0d, 0xff, 0x4a, 0xac, 0x52, 0xc3, 0x65, 0x2a, - 0x74, 0xbe, 0x44, 0xe7, 0x3a, 0xee, 0x24, 0xf2, 0x28, 0xef, 0x64, 0x5a, 0x19, 0x15, 0xab, 0x04, - 0x57, 0x79, 0x47, 0xc5, 0x72, 0xd3, 0x2e, 0x69, 0x6d, 0xa4, 0xb3, 0xf8, 0x4e, 0x34, 0x52, 0x23, - 0xe5, 0x98, 0x47, 0xd3, 0xe3, 0x0e, 0x50, 0x96, 0xb0, 0x2b, 0x27, 0x19, 0xfd, 0xa1, 0x4a, 0x6a, - 0x83, 0x4c, 0xc4, 0x34, 0x24, 0xab, 0x1f, 0x09, 0x9d, 0x4b, 0x95, 0x86, 0xc1, 0x7a, 0xb0, 0xd1, - 0x64, 0x05, 0x49, 0xbf, 0x40, 0x56, 0xfb, 0x5a, 0xc5, 0x22, 0xcf, 0xc3, 0xca, 0x7a, 0xb0, 0xd1, - 0x7a, 0xb0, 0xb6, 0x09, 0xe6, 0x37, 0x11, 0x64, 0x05, 0x97, 0xb6, 0x49, 0x8d, 0x29, 0x65, 0xc2, - 0xaa, 0x95, 0x22, 0x4e, 0x0a, 0x10, 0x66, 0x71, 0x7a, 0x87, 0x34, 0x76, 0x55, 0x6e, 0x52, 0x3e, - 0x11, 0x61, 0xcd, 0xee, 0x51, 0xd2, 0xf4, 0x8b, 0xa4, 0xbe, 0xa7, 0xa6, 0xa9, 0xc9, 0xc3, 0x95, - 0xf5, 0xea, 0x46, 0xeb, 0x41, 0xcb, 0x69, 0x5b, 0x6c, 0xab, 0xf6, 0xf2, 0xaf, 0xef, 0x5e, 0x61, - 0x28, 0x40, 0xdf, 0x23, 0x2b, 0xbb, 0x4a, 0x9d, 0xe4, 0x61, 0xdd, 0xee, 0x83, 0x92, 0x16, 0x62, - 0x8e, 0x43, 0xbf, 0x4b, 0x5a, 0xdd, 0x34, 0x55, 0x86, 0x1b, 0xa9, 0xd2, 0x3c, 0x5c, 0xb5, 0x26, - 0xff, 0xdf, 0x09, 0xc2, 0x6d, 0x37, 0x3d, 0xee, 0xa3, 0xd4, 0xe8, 0x73, 0xe6, 0xcb, 0xc3, 0x0e, - 0x4f, 0x65, 0x3a, 0x7d, 0x11, 0x36, 0xfc, 0x1d, 0x2c, 0xc4, 0x1c, 0x07, 0x1e, 0x65, 0xa0, 0x12, - 0xae, 0x65, 0x1e, 0x36, 0xfd, 0x47, 0x41, 0x90, 0x15, 0x5c, 0x10, 0x7c, 0x2e, 0xd3, 0xa1, 0x3a, - 0xcb, 0x43, 0xe2, 0x0b, 0x22, 0xc8, 0x0a, 0xee, 0x9d, 0xef, 0x91, 0x1b, 0x8b, 0xa7, 0xa2, 0x37, - 0x48, 0xf5, 0x44, 0x9c, 0xa3, 0x43, 0x60, 0x49, 0x6f, 0x91, 0x95, 0x53, 0x9e, 0x4c, 0x85, 0x75, - 0x45, 0x93, 0x39, 0xe2, 0x83, 0xca, 0xfb, 0x41, 0xf4, 0xbb, 0x6a, 0xe9, 0x27, 0x78, 0xe9, 0x03, - 0xa1, 0x27, 0x32, 0xe5, 0x89, 0x55, 0x6e, 0xb0, 0x92, 0xa6, 0x5f, 0x26, 0xad, 0x6d, 0x95, 0xe6, - 0x2a, 0x11, 0x03, 0xf9, 0x23, 0x81, 0x2e, 0x6d, 0xba, 0x43, 0x6d, 0xa9, 0x17, 0xcc, 0xe7, 0xd2, - 0x7b, 0xa4, 0x76, 0x98, 0x0b, 0x3d, 0xef, 0x52, 0x40, 0xd0, 0x27, 0x96, 0x4b, 0x29, 0xa9, 0x75, - 0xf5, 0x28, 0x0f, 0x6b, 0xeb, 0xd5, 0x8d, 0x26, 0xb3, 0x6b, 0x38, 0xfa, 0xa3, 0xf4, 0xd4, 0x7a, - 0xb3, 0xc9, 0x60, 0x09, 0xc8, 0xf6, 0xd9, 0xd0, 0x7a, 0xad, 0xc9, 0x60, 0x49, 0xbf, 0x4d, 0xae, - 0x6e, 0xf3, 0x8c, 0x1f, 0xc9, 0x44, 0x1a, 0x29, 0xc0, 0x4f, 0xb0, 0xcb, 0x3b, 0xde, 0x73, 0xfb, - 0x6c, 0x36, 0x27, 0x4c, 0xbf, 0x4a, 0x56, 0x59, 0x22, 0x27, 0xd2, 0xe4, 0x61, 0xc3, 0xfa, 0xf7, - 0x26, 0x86, 0xe5, 0xfe, 0xa0, 0xf7, 0x03, 0xc7, 0xc1, 0x43, 0x16, 0x72, 0x74, 0x83, 0x5c, 0x7f, - 0xa6, 0x9e, 0x89, 0xb3, 0xbe, 0x96, 0xa7, 0x32, 0x11, 0x23, 0xe1, 0x9c, 0xd7, 0x60, 0x8b, 0x30, - 0x48, 0x76, 0xb3, 0x8c, 0xeb, 0x89, 0xd2, 0x7d, 0xad, 0x8e, 0x65, 0x22, 0xac, 0xf7, 0x9a, 0x6c, - 0x11, 0xa6, 0xeb, 0xa4, 0xb5, 0xbf, 0xbf, 0x37, 0x88, 0x95, 0x16, 0xdd, 0xe1, 0xa7, 0x61, 0x6b, - 0x3d, 0xd8, 0xa8, 0x32, 0x1f, 0xa2, 0x11, 0xb9, 0x3a, 0x10, 0x09, 0xdc, 0xe6, 0x29, 0x3f, 0x12, - 0x49, 0x78, 0xd5, 0x1a, 0x9a, 0xc3, 0xa2, 0x87, 0xa4, 0xba, 0xa5, 0x5e, 0xd0, 0xdb, 0xa4, 0xbe, - 0x2b, 0xe4, 0x68, 0x6c, 0xac, 0xd7, 0xd6, 0x18, 0x52, 0xe0, 0xf5, 0xe7, 0x72, 0x68, 0xc6, 0xd6, - 0x5b, 0x6b, 0xcc, 0x11, 0x51, 0xea, 0x9c, 0x03, 0x0f, 0x7b, 0xd8, 0xdb, 0x41, 0x15, 0x58, 0x02, - 0xf2, 0xa4, 0xb7, 0x83, 0xd2, 0xb0, 0xa4, 0x9f, 0x27, 0xd7, 0xba, 0xc3, 0xa1, 0x84, 0xd8, 0xe2, - 0xc9, 0x13, 0x39, 0xcc, 0xc3, 0xea, 0x7a, 0x75, 0x63, 0x8d, 0x2d, 0xa0, 0x10, 0x39, 0x60, 0xd3, - 0xcf, 0xd1, 0x82, 0x8e, 0x7e, 0x15, 0x90, 0x9b, 0x4b, 0x5e, 0x01, 0x8d, 0x2d, 0x35, 0x4d, 0x87, - 0x32, 0x1d, 0x85, 0x81, 0xf5, 0x76, 0x49, 0xd3, 0xbb, 0xa4, 0xf9, 0xe8, 0xf8, 0x58, 0xc4, 0x46, - 0x9e, 0x42, 0xa4, 0x01, 0x73, 0x06, 0xc0, 0xd3, 0xf5, 0xd2, 0xb1, 0xd0, 0xd2, 0xf0, 0xa3, 0x44, - 0xd8, 0x03, 0x35, 0x99, 0x0f, 0x81, 0x7e, 0x1f, 0xe2, 0xd6, 0x18, 0x31, 0xc4, 0xe8, 0x9a, 0x01, - 0x50, 0xb2, 0xba, 0x93, 0x23, 0x29, 0x52, 0x83, 0x61, 0x56, 0x90, 0x51, 0x8f, 0xb4, 0xbc, 0x30, - 0x80, 0xf8, 0x3c, 0x38, 0xcf, 0x04, 0xe6, 0x91, 0x5d, 0x03, 0xb6, 0xcb, 0xf5, 0xd0, 0xbe, 0x51, - 0x8d, 0xd9, 0x35, 0x60, 0x03, 0x75, 0xec, 0x0a, 0x58, 0x8d, 0xd9, 0x75, 0xa4, 0xc8, 0x8a, 0xad, - 0x3b, 0x70, 0xda, 0xa1, 0xc8, 0x8d, 0x4c, 0x6d, 0x82, 0xa2, 0x2d, 0x1f, 0x02, 0xef, 0xe5, 0x6a, - 0xaa, 0xe3, 0x22, 0x39, 0x91, 0x02, 0xb3, 0x06, 0xb6, 0xaf, 0xba, 0xed, 0x61, 0x0d, 0x67, 0x57, - 0x99, 0xab, 0x4e, 0xee, 0x5e, 0x05, 0x19, 0x7d, 0xc3, 0x55, 0x51, 0xd0, 0xea, 0x73, 0x33, 0x2e, - 0x0e, 0x0d, 0x6b, 0x78, 0x6b, 0x26, 0xf8, 0x50, 0xa5, 0xc9, 0xb9, 0xdd, 0xa3, 0xc1, 0x4a, 0x3a, - 0xfa, 0x59, 0x80, 0x75, 0x91, 0xde, 0x27, 0x8d, 0xbe, 0x16, 0xb9, 0xe1, 0xda, 0x58, 0x8f, 0x94, - 0x89, 0x0b, 0x6c, 0xcc, 0x89, 0x52, 0x82, 0x6e, 0x92, 0x66, 0x5f, 0xe5, 0xc6, 0x89, 0x57, 0x5e, - 0x23, 0x3e, 0x13, 0xb1, 0xd6, 0x2d, 0xa1, 0x32, 0xeb, 0xb2, 0xcb, 0xad, 0xa3, 0x44, 0xf4, 0x31, - 0xa9, 0x01, 0x7e, 0xe9, 0x6d, 0x8a, 0xb2, 0x51, 0x59, 0x2e, 0x1b, 0xd5, 0x59, 0xd9, 0x08, 0xc9, - 0xea, 0x81, 0x9c, 0x08, 0x35, 0x35, 0x36, 0x20, 0xab, 0xac, 0x20, 0xa3, 0xdf, 0xac, 0x60, 0x9d, - 0xa6, 0xdf, 0x21, 0xad, 0xc3, 0xde, 0xce, 0x1e, 0xcf, 0x32, 0x99, 0x8e, 0x72, 0xbc, 0xf4, 0x2d, - 0xaf, 0x8e, 0x94, 0x4c, 0x3c, 0xa0, 0x2f, 0x0e, 0xda, 0x4f, 0x3c, 0xed, 0xca, 0x7f, 0xd6, 0xf6, - 0xc4, 0x69, 0x87, 0xd4, 0x07, 0xe7, 0x79, 0x6c, 0x12, 0x7c, 0x0d, 0xbf, 0x7c, 0x6d, 0x3a, 0x8e, - 0x6b, 0x31, 0x28, 0x46, 0x1f, 0x90, 0x26, 0x13, 0x2e, 0x34, 0x72, 0x7b, 0xa5, 0xf9, 0xcd, 0x4a, - 0x1e, 0x9b, 0x89, 0x41, 0xf0, 0x6d, 0x8f, 0xb4, 0x9a, 0x66, 0xb9, 0x7d, 0xc5, 0x15, 0x17, 0x7c, - 0x1e, 0x44, 0x3f, 0x20, 0xe4, 0x19, 0x9f, 0x88, 0x3c, 0xe3, 0x60, 0xb6, 0xbe, 0x74, 0x87, 0x92, - 0x89, 0x77, 0xf0, 0xa4, 0xa1, 0x94, 0xee, 0x88, 0x53, 0x19, 0x8b, 0xa2, 0x55, 0xde, 0xf4, 0x14, - 0x1d, 0xa7, 0x28, 0xa5, 0x28, 0x47, 0xef, 0x93, 0xd5, 0x81, 0x88, 0x63, 0x35, 0xc9, 0xb0, 0x49, - 0x52, 0x4f, 0x05, 0x39, 0xac, 0x10, 0xa1, 0xf7, 0xc9, 0x4d, 0x88, 0xe9, 0xe3, 0xbc, 0xaf, 0x55, - 0xc6, 0x47, 0x2e, 0x83, 0x9a, 0xf6, 0x12, 0xcb, 0x0c, 0xb8, 0xec, 0x1e, 0xcf, 0x4f, 0xc4, 0x10, - 0x2e, 0x06, 0x6d, 0xd3, 0xd6, 0x05, 0x0f, 0xa2, 0xf7, 0xc8, 0x5a, 0x11, 0xf7, 0x4e, 0xa6, 0x65, - 0x65, 0xe6, 0x41, 0xda, 0x26, 0xc4, 0xa6, 0xae, 0x5f, 0x76, 0x3d, 0x84, 0x76, 0x48, 0xa3, 0x97, - 0x1a, 0x91, 0xb0, 0xa1, 0x09, 0xd7, 0xec, 0x25, 0xde, 0xf2, 0x9d, 0x8e, 0x2c, 0x56, 0x0a, 0xdd, - 0xf9, 0x16, 0x69, 0x79, 0x0e, 0x7d, 0xa3, 0xee, 0xfc, 0x6e, 0x39, 0x06, 0x80, 0xd0, 0x70, 0x3a, - 0x99, 0x14, 0x8a, 0x8e, 0x00, 0x81, 0x62, 0x64, 0xb8, 0x5c, 0xe0, 0x13, 0x72, 0x6d, 0x3e, 0x18, - 0x6d, 0xb7, 0x50, 0xb9, 0x29, 0x4b, 0x3f, 0x52, 0x36, 0x58, 0x8a, 0x79, 0xb1, 0xec, 0x02, 0x3e, - 0x64, 0x0b, 0x1d, 0x34, 0xff, 0xaa, 0x65, 0xd9, 0x75, 0xf4, 0x3e, 0xda, 0x2f, 0xe3, 0xe2, 0x75, - 0x65, 0xd3, 0x46, 0x60, 0x65, 0x96, 0xc7, 0xd1, 0x2f, 0x02, 0xd2, 0xf2, 0x42, 0xe5, 0x75, 0xb9, - 0x6e, 0x6d, 0x55, 0x3c, 0x5b, 0xb7, 0xc8, 0xca, 0x1e, 0xff, 0x54, 0xb9, 0xe9, 0xa2, 0xca, 0x1c, - 0x61, 0x51, 0x99, 0x2a, 0x8d, 0xd9, 0xee, 0x08, 0xa8, 0x7c, 0x8f, 0x65, 0x22, 0xf6, 0xd4, 0x50, - 0xd8, 0xe8, 0x5f, 0x63, 0x25, 0x5d, 0xf4, 0xbf, 0xfa, 0x52, 0xff, 0x5b, 0x2d, 0xfb, 0x5f, 0xf4, - 0xb7, 0x0a, 0x5e, 0x6f, 0x96, 0x53, 0xdf, 0x9c, 0x45, 0x7d, 0xb0, 0x94, 0xb9, 0x8e, 0xe3, 0x12, - 0x6c, 0x31, 0xf6, 0x61, 0x56, 0x15, 0x13, 0xa5, 0xcf, 0x71, 0x78, 0xf2, 0xb3, 0xc5, 0x31, 0x18, - 0x0a, 0xd0, 0x75, 0x52, 0xdd, 0xee, 0x1f, 0xe2, 0xf8, 0x74, 0xcd, 0x1f, 0x6c, 0xfa, 0x87, 0x0c, - 0x58, 0xf4, 0x73, 0xa4, 0xd6, 0x87, 0x76, 0xec, 0x0a, 0xc1, 0x75, 0x4f, 0x04, 0x60, 0x66, 0x99, - 0x90, 0x6d, 0x5b, 0x89, 0x8a, 0x4f, 0x7a, 0xfb, 0xf6, 0xf2, 0xf3, 0xd9, 0x86, 0x1c, 0x56, 0x88, - 0xd0, 0xc7, 0xe4, 0xda, 0xee, 0x74, 0x24, 0x32, 0x3e, 0x12, 0x4f, 0xdd, 0x80, 0xe4, 0xca, 0x41, - 0xe8, 0x29, 0xcd, 0x09, 0xe0, 0x05, 0x17, 0xb4, 0x60, 0xd7, 0x67, 0xc2, 0x9c, 0x29, 0x7d, 0x82, - 0x93, 0x99, 0xbf, 0x2b, 0x72, 0x58, 0x21, 0x12, 0xfd, 0xa5, 0x88, 0x02, 0xbc, 0xfa, 0x2d, 0x28, - 0xce, 0x13, 0xe9, 0x46, 0x99, 0x2a, 0x73, 0x04, 0xc4, 0x26, 0x13, 0xb9, 0xd0, 0xa7, 0xae, 0x06, - 0x54, 0xdc, 0xb8, 0xe4, 0x41, 0x36, 0x36, 0xcf, 0x78, 0x86, 0x41, 0x61, 0xd7, 0x10, 0xe9, 0x1f, - 0x0a, 0x9d, 0x8a, 0x04, 0x83, 0x02, 0x29, 0x98, 0x0f, 0xdc, 0xea, 0x60, 0xbb, 0x6f, 0x5f, 0xa6, - 0xca, 0x66, 0x00, 0xe4, 0x3f, 0x68, 0x67, 0x32, 0x85, 0x6f, 0x97, 0xba, 0x6d, 0xea, 0x1e, 0x42, - 0xbf, 0x44, 0x6e, 0xec, 0xc8, 0x1c, 0x06, 0x8d, 0xfd, 0xfd, 0xbd, 0x0f, 0x65, 0x92, 0x08, 0x6d, - 0x2f, 0xda, 0x60, 0x4b, 0x78, 0xf4, 0xc7, 0x80, 0x34, 0x0a, 0xc7, 0xc1, 0x71, 0x06, 0x63, 0xae, - 0x6d, 0xe0, 0x80, 0x51, 0xa4, 0xe0, 0xca, 0xdf, 0x9f, 0x2a, 0xc3, 0xf1, 0x5a, 0x8e, 0x00, 0xe9, - 0xbe, 0xd0, 0x52, 0x0d, 0x71, 0xae, 0x40, 0x0a, 0x66, 0x4c, 0x26, 0x78, 0x62, 0xe4, 0x44, 0xb0, - 0x69, 0x0a, 0x3f, 0x78, 0xbb, 0x45, 0x18, 0x86, 0xb7, 0x02, 0x42, 0x4b, 0x2b, 0xd6, 0xd2, 0x02, - 0x0a, 0x4f, 0xb7, 0x9d, 0x4d, 0x73, 0x1c, 0xb1, 0xed, 0x1a, 0xb0, 0x3d, 0x31, 0x71, 0xb3, 0x75, - 0x93, 0xd9, 0x75, 0x74, 0x86, 0x73, 0xdc, 0x73, 0x3b, 0x5d, 0x62, 0xd6, 0x96, 0xd9, 0x18, 0x5c, - 0x9a, 0x8d, 0x15, 0x3f, 0x1b, 0x6f, 0x93, 0xba, 0xd3, 0xc5, 0x0a, 0x82, 0x14, 0xbc, 0xf8, 0x53, - 0xc1, 0x8f, 0x91, 0x57, 0xb3, 0x3c, 0x0f, 0x89, 0x0e, 0xc9, 0x5b, 0x76, 0xe3, 0x83, 0xb1, 0x56, - 0xc6, 0x24, 0xe2, 0xbf, 0xd8, 0x9a, 0x92, 0x1a, 0xe3, 0x46, 0x14, 0x33, 0x1a, 0xac, 0xa3, 0x7f, - 0x54, 0xc9, 0x55, 0x3f, 0x15, 0xbc, 0xf3, 0x05, 0xff, 0xe6, 0x7c, 0x95, 0xc5, 0xf3, 0xd1, 0x2e, - 0xb9, 0xea, 0xbf, 0xc9, 0x25, 0x1d, 0xdd, 0x67, 0x63, 0xda, 0xcc, 0xa9, 0xd0, 0x43, 0xf2, 0x76, - 0x71, 0x3b, 0xe8, 0x46, 0x5b, 0x59, 0x8e, 0xb6, 0x6a, 0xd6, 0xd6, 0xff, 0x79, 0xb6, 0xe6, 0x5f, - 0x01, 0xad, 0x5d, 0xae, 0x4d, 0x9f, 0x93, 0xdb, 0x05, 0xe3, 0xb9, 0x96, 0x46, 0xcc, 0xec, 0xae, - 0x7c, 0x36, 0xbb, 0xaf, 0x51, 0xf7, 0x0d, 0xc3, 0x8e, 0xbd, 0xfd, 0xfe, 0x00, 0x0d, 0xd7, 0xdf, - 0xd0, 0xf0, 0xbc, 0x3a, 0xfd, 0x21, 0x79, 0x67, 0x6e, 0x4b, 0xcf, 0xf2, 0xea, 0x67, 0xb3, 0xfc, - 0x3a, 0xfd, 0xe8, 0x3d, 0xd2, 0x2c, 0x2b, 0xe4, 0xe5, 0x75, 0x26, 0xfa, 0x49, 0xf1, 0xad, 0xe2, - 0x17, 0x72, 0x90, 0xed, 0x26, 0x89, 0x3a, 0xc3, 0x8f, 0x62, 0x47, 0xfc, 0xcf, 0xbd, 0xe9, 0x36, - 0xa9, 0x77, 0x63, 0xfb, 0xff, 0x88, 0x9b, 0xcb, 0x90, 0x8a, 0x12, 0x8c, 0x4a, 0xac, 0x90, 0x30, - 0xc9, 0x6e, 0x27, 0x3c, 0xcf, 0xcb, 0x86, 0x5d, 0x90, 0x74, 0x8b, 0x90, 0xbe, 0x96, 0x4a, 0xbb, - 0xcf, 0x60, 0x37, 0x80, 0xde, 0x5d, 0x98, 0x45, 0xf4, 0x31, 0x8f, 0x05, 0x4a, 0x9d, 0x17, 0x43, - 0xdc, 0x4c, 0x2b, 0x7a, 0x4c, 0xe8, 0x72, 0x65, 0x87, 0xbe, 0xd9, 0xe7, 0x23, 0x91, 0x43, 0xb7, - 0x77, 0xfd, 0xb8, 0xa4, 0x67, 0x2f, 0xe7, 0xbe, 0x81, 0xf0, 0xe5, 0x76, 0xc9, 0xed, 0xcb, 0xf7, - 0x84, 0x77, 0x82, 0xe1, 0xa0, 0xe8, 0xeb, 0xb0, 0xb6, 0xf6, 0x91, 0x8f, 0xf9, 0x54, 0xd2, 0xd1, - 0x2f, 0x03, 0x7c, 0x80, 0x62, 0x0c, 0xbc, 0x47, 0xd6, 0x76, 0xc4, 0x31, 0x9f, 0x26, 0xa6, 0x1b, - 0x7b, 0x1f, 0x51, 0xf3, 0x20, 0x48, 0x75, 0x75, 0x3c, 0x96, 0x46, 0xc4, 0x66, 0xaa, 0x45, 0xf1, - 0x7d, 0x30, 0x0f, 0xc2, 0xe1, 0x1f, 0x27, 0x7c, 0x94, 0xe3, 0xa7, 0x82, 0x23, 0xe8, 0xd7, 0x48, - 0x03, 0x26, 0x34, 0x9e, 0x24, 0x39, 0x26, 0xdc, 0xdc, 0x5c, 0xea, 0x58, 0xc5, 0x47, 0x4a, 0x21, - 0x19, 0x49, 0x72, 0xdd, 0x3f, 0x67, 0x57, 0x8f, 0xc0, 0x7c, 0x2f, 0x1d, 0x8a, 0x17, 0x58, 0xe1, - 0x1d, 0x01, 0xe8, 0x47, 0xe5, 0x7c, 0x57, 0x63, 0x8e, 0x80, 0x37, 0xb0, 0x8b, 0x83, 0x33, 0x85, - 0x65, 0xa9, 0xa4, 0xe9, 0x35, 0x52, 0xd9, 0xcf, 0xf0, 0x4b, 0xba, 0xb2, 0x9f, 0x45, 0x3f, 0x2f, - 0xdf, 0xc4, 0x6d, 0x0e, 0x26, 0xed, 0xc4, 0x85, 0xdf, 0xce, 0x8e, 0x70, 0x21, 0x55, 0x76, 0x48, - 0x1b, 0x52, 0xf6, 0x6d, 0xee, 0x92, 0x86, 0xd0, 0x3a, 0x55, 0x5a, 0x60, 0xe9, 0xdd, 0xbd, 0xc2, - 0x4a, 0x84, 0x76, 0xbc, 0xff, 0x61, 0x5a, 0x0f, 0xde, 0x5e, 0x9e, 0xc8, 0xbb, 0xba, 0xf8, 0x84, - 0xb1, 0x82, 0x5b, 0x84, 0x34, 0x1e, 0x81, 0x32, 0x13, 0x26, 0xfa, 0x3a, 0x59, 0x9b, 0x9b, 0x7b, - 0xc1, 0x0f, 0x4f, 0x1f, 0x6e, 0xf3, 0x78, 0x2c, 0x06, 0xf1, 0x58, 0x4c, 0x78, 0xe1, 0xad, 0x39, - 0x70, 0xeb, 0xa7, 0xc1, 0xcb, 0x57, 0xed, 0x2b, 0x7f, 0x7e, 0xd5, 0xbe, 0xf2, 0xcf, 0x57, 0xed, - 0xe0, 0xc7, 0x17, 0xed, 0xe0, 0xd7, 0x17, 0xed, 0xe0, 0xb7, 0x17, 0xed, 0xe0, 0xf7, 0x17, 0xed, - 0xe0, 0xe5, 0x45, 0x3b, 0xf8, 0xd3, 0x45, 0x3b, 0xf8, 0xfb, 0x45, 0x3b, 0xf8, 0xf8, 0x93, 0x37, - 0xfc, 0x97, 0x53, 0xbb, 0xf6, 0xd7, 0x39, 0x95, 0xda, 0x78, 0xac, 0xec, 0x64, 0xd4, 0xe1, 0x23, - 0x91, 0x1a, 0xef, 0x1f, 0x50, 0xb8, 0xe9, 0x51, 0xdd, 0xd2, 0x0f, 0xff, 0x15, 0x00, 0x00, 0xff, - 0xff, 0x10, 0x2c, 0x26, 0x0c, 0x4e, 0x15, 0x00, 0x00, + // 2163 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0xcd, 0x73, 0x1b, 0x49, + 0x15, 0xcf, 0x48, 0xb2, 0x2c, 0xb5, 0x22, 0x67, 0xd3, 0x9b, 0xcd, 0x0e, 0x21, 0xa5, 0xf5, 0x0e, + 0x29, 0x30, 0x10, 0xac, 0x22, 0x81, 0x25, 0x84, 0x8f, 0x2a, 0x59, 0x4e, 0x62, 0xd5, 0xc6, 0xb1, + 0x68, 0xd9, 0x6b, 0xd8, 0xc3, 0x56, 0xb5, 0x47, 0x6d, 0xa9, 0xd7, 0xa3, 0xe9, 0xa9, 0x9e, 0x96, + 0x1d, 0x73, 0x82, 0x1b, 0xff, 0x01, 0x67, 0x2e, 0x7c, 0xfc, 0x07, 0x14, 0x27, 0x38, 0x91, 0xe2, + 0xc4, 0x91, 0x2a, 0xaa, 0x80, 0xf8, 0xce, 0x9d, 0x23, 0xf5, 0xfa, 0x63, 0xd4, 0x92, 0x6c, 0xd8, + 0xc0, 0xad, 0xdf, 0x7b, 0xbf, 0xf7, 0xfa, 0xe3, 0x7d, 0xce, 0xa0, 0x17, 0x23, 0xae, 0xc6, 0xd3, + 0xa3, 0xcd, 0x58, 0x4c, 0xda, 0x27, 0x54, 0xd1, 0xaf, 0xc5, 0x22, 0x55, 0x94, 0xa7, 0x4c, 0xe6, + 0x4b, 0x74, 0x2e, 0xe3, 0x76, 0xc2, 0x8f, 0xf2, 0x76, 0x26, 0x85, 0x12, 0xb1, 0x48, 0xec, 0x2a, + 0x6f, 0x8b, 0x98, 0x6f, 0xea, 0x25, 0xae, 0x8c, 0x64, 0x16, 0xdf, 0x89, 0x46, 0x62, 0x24, 0x8c, + 0xf0, 0x68, 0x7a, 0xdc, 0x06, 0x4a, 0x13, 0x7a, 0x65, 0x90, 0xd1, 0x1f, 0xcb, 0xa8, 0x32, 0xc8, + 0x58, 0x8c, 0x43, 0xb4, 0xfa, 0x11, 0x93, 0x39, 0x17, 0x69, 0x18, 0xac, 0x07, 0x1b, 0x75, 0xe2, + 0x48, 0xfc, 0x25, 0xb4, 0xda, 0x97, 0x22, 0x66, 0x79, 0x1e, 0x96, 0xd6, 0x83, 0x8d, 0xc6, 0x83, + 0xe6, 0x26, 0x98, 0xdf, 0xb4, 0x4c, 0xe2, 0xa4, 0xb8, 0x85, 0x2a, 0x44, 0x08, 0x15, 0x96, 0x35, + 0x0a, 0x19, 0x14, 0x70, 0x88, 0xe6, 0xe3, 0x3b, 0xa8, 0xb6, 0x23, 0x72, 0x95, 0xd2, 0x09, 0x0b, + 0x2b, 0x7a, 0x8f, 0x82, 0xc6, 0x5f, 0x46, 0xd5, 0x5d, 0x31, 0x4d, 0x55, 0x1e, 0xae, 0xac, 0x97, + 0x37, 0x1a, 0x0f, 0x1a, 0x46, 0x5b, 0xf3, 0xb6, 0x2a, 0xaf, 0xfe, 0xf6, 0xde, 0x35, 0x62, 0x01, + 0xf8, 0x7d, 0xb4, 0xb2, 0x23, 0xc4, 0x49, 0x1e, 0x56, 0xf5, 0x3e, 0x16, 0xa9, 0x59, 0xc4, 0x48, + 0xf0, 0xf7, 0x50, 0xa3, 0x93, 0xa6, 0x42, 0x51, 0xc5, 0x45, 0x9a, 0x87, 0xab, 0xda, 0xe4, 0xe7, + 0x0d, 0x10, 0x6e, 0xbb, 0xe9, 0x49, 0x9f, 0xa4, 0x4a, 0x9e, 0x13, 0x1f, 0x0f, 0x3b, 0x3c, 0xe7, + 0xe9, 0xf4, 0x65, 0x58, 0xf3, 0x77, 0xd0, 0x2c, 0x62, 0x24, 0xf0, 0x28, 0x03, 0x91, 0x50, 0xc9, + 0xf3, 0xb0, 0xee, 0x3f, 0x8a, 0x65, 0x12, 0x27, 0x05, 0xe0, 0x21, 0x4f, 0x87, 0xe2, 0x2c, 0x0f, + 0x91, 0x0f, 0xb4, 0x4c, 0xe2, 0xa4, 0x77, 0xbe, 0x8f, 0xde, 0x5a, 0x3c, 0x15, 0x7e, 0x0b, 0x95, + 0x4f, 0xd8, 0xb9, 0x75, 0x08, 0x2c, 0xf1, 0x2d, 0xb4, 0x72, 0x4a, 0x93, 0x29, 0xd3, 0xae, 0xa8, + 0x13, 0x43, 0x3c, 0x2e, 0x3d, 0x0a, 0xa2, 0xdf, 0x95, 0x0b, 0x3f, 0xc1, 0x4b, 0xef, 0x33, 0x39, + 0xe1, 0x29, 0x4d, 0xb4, 0x72, 0x8d, 0x14, 0x34, 0xfe, 0x2a, 0x6a, 0x74, 0x45, 0x9a, 0x8b, 0x84, + 0x0d, 0xf8, 0x8f, 0x99, 0x75, 0x69, 0xdd, 0x1c, 0x6a, 0x4b, 0xbc, 0x24, 0xbe, 0x14, 0xdf, 0x43, + 0x95, 0x83, 0x9c, 0xc9, 0x79, 0x97, 0x02, 0xc7, 0xfa, 0x44, 0x4b, 0x31, 0x46, 0x95, 0x8e, 0x1c, + 0xe5, 0x61, 0x65, 0xbd, 0xbc, 0x51, 0x27, 0x7a, 0x0d, 0x47, 0x7f, 0x92, 0x9e, 0x6a, 0x6f, 0xd6, + 0x09, 0x2c, 0x81, 0xd3, 0x3d, 0x1b, 0x6a, 0xaf, 0xd5, 0x09, 0x2c, 0xf1, 0x77, 0xd0, 0xf5, 0x2e, + 0xcd, 0xe8, 0x11, 0x4f, 0xb8, 0xe2, 0x0c, 0xfc, 0x04, 0xbb, 0xbc, 0xeb, 0x3d, 0xb7, 0x2f, 0x26, + 0x73, 0x60, 0xfc, 0x75, 0xb4, 0x4a, 0x12, 0x3e, 0xe1, 0x2a, 0x0f, 0x6b, 0xda, 0xbf, 0x37, 0x6d, + 0x58, 0xee, 0x0d, 0x7a, 0x3f, 0x34, 0x12, 0x7b, 0x48, 0x87, 0xc3, 0x1b, 0xe8, 0xc6, 0x0b, 0xf1, + 0x82, 0x9d, 0xf5, 0x25, 0x3f, 0xe5, 0x09, 0x1b, 0x31, 0xe3, 0xbc, 0x1a, 0x59, 0x64, 0x03, 0xb2, + 0x93, 0x65, 0x54, 0x4e, 0x84, 0xec, 0x4b, 0x71, 0xcc, 0x13, 0xa6, 0xbd, 0x57, 0x27, 0x8b, 0x6c, + 0xbc, 0x8e, 0x1a, 0x7b, 0x7b, 0xbb, 0x83, 0x58, 0x48, 0xd6, 0x19, 0x7e, 0x1a, 0x36, 0xd6, 0x83, + 0x8d, 0x32, 0xf1, 0x59, 0x38, 0x42, 0xd7, 0x07, 0x2c, 0x81, 0xdb, 0x3c, 0xa7, 0x47, 0x2c, 0x09, + 0xaf, 0x6b, 0x43, 0x73, 0xbc, 0xe8, 0x21, 0x2a, 0x6f, 0x89, 0x97, 0xf8, 0x36, 0xaa, 0xee, 0x30, + 0x3e, 0x1a, 0x2b, 0xed, 0xb5, 0x26, 0xb1, 0x14, 0x78, 0xfd, 0x90, 0x0f, 0xd5, 0x58, 0x7b, 0xab, + 0x49, 0x0c, 0x11, 0xa5, 0xc6, 0x39, 0xf0, 0xb0, 0x07, 0xbd, 0x6d, 0xab, 0x02, 0x4b, 0xe0, 0x3c, + 0xeb, 0x6d, 0x5b, 0x34, 0x2c, 0xf1, 0x17, 0xd1, 0x5a, 0x67, 0x38, 0xe4, 0x10, 0x5b, 0x34, 0x79, + 0xc6, 0x87, 0x79, 0x58, 0x5e, 0x2f, 0x6f, 0x34, 0xc9, 0x02, 0x17, 0x22, 0x07, 0x6c, 0xfa, 0x39, + 0xea, 0xe8, 0xe8, 0x57, 0x01, 0xba, 0xb9, 0xe4, 0x15, 0xd0, 0xd8, 0x12, 0xd3, 0x74, 0xc8, 0xd3, + 0x51, 0x18, 0x68, 0x6f, 0x17, 0x34, 0xbe, 0x8b, 0xea, 0x4f, 0x8e, 0x8f, 0x59, 0xac, 0xf8, 0x29, + 0x44, 0x1a, 0x08, 0x67, 0x0c, 0x78, 0xba, 0x5e, 0x3a, 0x66, 0x92, 0x2b, 0x7a, 0x94, 0x30, 0x7d, + 0xa0, 0x3a, 0xf1, 0x59, 0xa0, 0xdf, 0x87, 0xb8, 0x55, 0x8a, 0x0d, 0x6d, 0x74, 0xcd, 0x18, 0x50, + 0xb2, 0x3a, 0x93, 0x23, 0xce, 0x52, 0x65, 0xc3, 0xcc, 0x91, 0x51, 0x0f, 0x35, 0xbc, 0x30, 0x80, + 0xf8, 0xdc, 0x3f, 0xcf, 0x98, 0xcd, 0x23, 0xbd, 0x06, 0xde, 0x0e, 0x95, 0x43, 0xfd, 0x46, 0x15, + 0xa2, 0xd7, 0xc0, 0x1b, 0x88, 0x63, 0x53, 0xc0, 0x2a, 0x44, 0xaf, 0x23, 0x81, 0x56, 0x74, 0xdd, + 0x81, 0xd3, 0x0e, 0x59, 0xae, 0x78, 0xaa, 0x13, 0xd4, 0xda, 0xf2, 0x59, 0xe0, 0xbd, 0x5c, 0x4c, + 0x65, 0xec, 0x92, 0xd3, 0x52, 0x60, 0x56, 0xc1, 0xf6, 0x65, 0xb3, 0x3d, 0xac, 0xe1, 0xec, 0x22, + 0x33, 0xd5, 0xc9, 0xdc, 0xcb, 0x91, 0xd1, 0x07, 0xa6, 0x8a, 0x82, 0x56, 0x9f, 0xaa, 0xb1, 0x3b, + 0x34, 0xac, 0xe1, 0xad, 0x09, 0xa3, 0x43, 0x91, 0x26, 0xe7, 0x7a, 0x8f, 0x1a, 0x29, 0xe8, 0xe8, + 0x0f, 0x25, 0x5b, 0x17, 0xf1, 0x7d, 0x54, 0xeb, 0x4b, 0x96, 0x2b, 0x2a, 0x95, 0xf6, 0x48, 0x91, + 0xb8, 0x20, 0xb6, 0x39, 0x51, 0x20, 0xf0, 0x26, 0xaa, 0xf7, 0x45, 0xae, 0x0c, 0xbc, 0x74, 0x05, + 0x7c, 0x06, 0xd1, 0xd6, 0x35, 0x21, 0x32, 0xed, 0xb2, 0xcb, 0xad, 0x5b, 0x04, 0xfe, 0x00, 0x35, + 0xbb, 0x92, 0x51, 0xc5, 0xc8, 0x34, 0x55, 0x5c, 0x07, 0xd5, 0xe5, 0x2a, 0xf3, 0x30, 0xfc, 0x18, + 0xdd, 0x30, 0x8c, 0xae, 0xeb, 0x7c, 0xb6, 0x31, 0x2c, 0x6b, 0x2e, 0x02, 0xf1, 0x23, 0xb4, 0x36, + 0x80, 0xa3, 0xce, 0x54, 0xab, 0x57, 0xa8, 0x2e, 0xe0, 0xa2, 0x8f, 0x51, 0x05, 0xa4, 0x97, 0xbe, + 0xbd, 0x2b, 0x72, 0xa5, 0xe5, 0x22, 0x57, 0x9e, 0x15, 0xb9, 0x10, 0xad, 0xee, 0xf3, 0x09, 0x13, + 0x53, 0xa5, 0xd3, 0xa7, 0x4c, 0x1c, 0x19, 0xfd, 0x66, 0xc5, 0x76, 0x15, 0xfc, 0x5d, 0xd4, 0x38, + 0xe8, 0x6d, 0xef, 0xd2, 0x2c, 0xe3, 0xe9, 0x28, 0xb7, 0x2e, 0xba, 0xe5, 0x55, 0xbd, 0x42, 0x68, + 0x8f, 0xe9, 0xc3, 0x41, 0xfb, 0x99, 0xa7, 0x5d, 0xfa, 0xef, 0xda, 0x1e, 0x1c, 0xb7, 0x51, 0x75, + 0x70, 0x9e, 0xc7, 0x2a, 0xb1, 0xbe, 0xf3, 0x8b, 0xed, 0xa6, 0x91, 0x98, 0x86, 0x68, 0x61, 0xf8, + 0x01, 0xaa, 0x13, 0x66, 0x02, 0x39, 0xd7, 0x57, 0x9a, 0xdf, 0xac, 0x90, 0x91, 0x19, 0x0c, 0x52, + 0xa5, 0x3b, 0x92, 0x62, 0x9a, 0xe5, 0xfa, 0x15, 0x57, 0x4c, 0xaa, 0x78, 0x2c, 0xfc, 0x18, 0xa1, + 0x17, 0x74, 0xc2, 0xf2, 0x8c, 0x82, 0xd9, 0xea, 0xd2, 0x1d, 0x0a, 0xa1, 0xbd, 0x83, 0x87, 0x86, + 0xc2, 0xbf, 0xcd, 0x4e, 0x79, 0xcc, 0x5c, 0x63, 0xbf, 0xe9, 0x29, 0x1a, 0x89, 0x2b, 0xfc, 0x16, + 0x87, 0xef, 0xa3, 0xd5, 0x01, 0x8b, 0x63, 0x31, 0xc9, 0x6c, 0x4b, 0xc7, 0x9e, 0x8a, 0x95, 0x10, + 0x07, 0xc1, 0xf7, 0xd1, 0x4d, 0xc8, 0xc0, 0xe3, 0xbc, 0x2f, 0x45, 0x46, 0x47, 0x26, 0xdf, 0xeb, + 0xfa, 0x12, 0xcb, 0x02, 0xb8, 0xec, 0x2e, 0xcd, 0x4f, 0xd8, 0x10, 0x2e, 0x06, 0x4d, 0x5e, 0x57, + 0x31, 0x8f, 0x85, 0xef, 0xa1, 0xa6, 0xcb, 0x52, 0x83, 0x69, 0x68, 0xcc, 0x3c, 0x13, 0xb7, 0x10, + 0xd2, 0x85, 0xc6, 0x6f, 0x12, 0x1e, 0x07, 0xb7, 0x51, 0xad, 0x97, 0x2a, 0x96, 0x90, 0xa1, 0x0a, + 0x9b, 0xfa, 0x12, 0x6f, 0xfb, 0x4e, 0xb7, 0x22, 0x52, 0x80, 0xee, 0x7c, 0x1b, 0x35, 0x3c, 0x87, + 0xbe, 0xd1, 0x2c, 0xf1, 0x5e, 0x31, 0xb4, 0x00, 0x68, 0x38, 0x9d, 0x4c, 0x9c, 0xa2, 0x21, 0x00, + 0xe0, 0x06, 0x9c, 0xcb, 0x01, 0x9f, 0xa0, 0xb5, 0xf9, 0x60, 0xd4, 0xbd, 0x4d, 0xe4, 0xaa, 0x68, + 0x54, 0x96, 0xd2, 0xc1, 0xe2, 0x12, 0xb0, 0xe8, 0x59, 0x3e, 0x4b, 0x97, 0x65, 0x18, 0x55, 0xca, + 0x5a, 0xa4, 0xd7, 0xd1, 0x23, 0x6b, 0xbf, 0x88, 0x8b, 0xab, 0x8a, 0xbc, 0x8e, 0xc0, 0xd2, 0x2c, + 0x8f, 0xa3, 0x5f, 0x04, 0xa8, 0xe1, 0x85, 0xca, 0x55, 0xb9, 0xae, 0x6d, 0x95, 0x3c, 0x5b, 0xb7, + 0xd0, 0xca, 0x2e, 0xfd, 0x54, 0x98, 0x59, 0xa8, 0x4c, 0x0c, 0xa1, 0xb9, 0x3c, 0x15, 0xd2, 0x66, + 0xbb, 0x21, 0xa0, 0x4e, 0x3f, 0xe5, 0x09, 0xdb, 0x15, 0x43, 0xa6, 0xa3, 0xbf, 0x49, 0x0a, 0xda, + 0x75, 0xeb, 0xea, 0x52, 0xb7, 0x5e, 0x2d, 0xba, 0x75, 0xf4, 0xf7, 0x92, 0xbd, 0xde, 0x2c, 0xa7, + 0xbe, 0x35, 0x8b, 0xfa, 0x60, 0x29, 0x73, 0x8d, 0xc4, 0x24, 0xd8, 0x62, 0xec, 0xc3, 0x64, 0xcd, + 0x26, 0x42, 0x9e, 0xdb, 0x51, 0xcf, 0xcf, 0x16, 0x23, 0x20, 0x16, 0x80, 0xd7, 0x51, 0xb9, 0xdb, + 0x3f, 0xb0, 0xc3, 0xde, 0x9a, 0x3f, 0x86, 0xf5, 0x0f, 0x08, 0x88, 0xf0, 0x17, 0x50, 0xa5, 0x0f, + 0xc3, 0x83, 0x29, 0x04, 0x37, 0x3c, 0x08, 0xb0, 0x89, 0x16, 0x42, 0xb6, 0x6d, 0x25, 0x22, 0x3e, + 0xe9, 0xed, 0xe9, 0xcb, 0xcf, 0x67, 0x9b, 0x95, 0x10, 0x07, 0xc1, 0x4f, 0xd1, 0xda, 0xce, 0x74, + 0xc4, 0x32, 0x3a, 0x62, 0xcf, 0xcd, 0x38, 0x67, 0xca, 0x41, 0xe8, 0x29, 0xcd, 0x01, 0x5c, 0xed, + 0x9e, 0xd7, 0x82, 0x5d, 0x5f, 0x30, 0x75, 0x26, 0xe4, 0x89, 0x9d, 0x23, 0xfd, 0x5d, 0xad, 0x84, + 0x38, 0x48, 0xf4, 0x57, 0x17, 0x05, 0xf6, 0xea, 0xb7, 0xa0, 0x38, 0x4f, 0xb8, 0x19, 0xbc, 0xca, + 0xc4, 0x10, 0x10, 0x9b, 0x84, 0xe5, 0x4c, 0x9e, 0x9a, 0x1a, 0x50, 0x32, 0xc3, 0x9d, 0xc7, 0xd2, + 0xb1, 0x79, 0x46, 0x33, 0x1b, 0x14, 0x7a, 0x0d, 0x91, 0xfe, 0x21, 0x93, 0x29, 0x4b, 0x6c, 0x50, + 0x58, 0x0a, 0xa6, 0x19, 0xb3, 0xda, 0xef, 0xf6, 0xf5, 0xcb, 0x94, 0xc9, 0x8c, 0x01, 0xf9, 0x0f, + 0xda, 0x19, 0x4f, 0xe1, 0x4b, 0xab, 0xaa, 0x47, 0x10, 0x8f, 0x83, 0xbf, 0x82, 0xde, 0xda, 0xe6, + 0x39, 0x8c, 0x45, 0x7b, 0x7b, 0xbb, 0x1f, 0xf2, 0x24, 0x61, 0x52, 0x5f, 0xb4, 0x46, 0x96, 0xf8, + 0xd1, 0x9f, 0x02, 0x54, 0x73, 0x8e, 0x83, 0xe3, 0x0c, 0xc6, 0x54, 0xea, 0xc0, 0x01, 0xa3, 0x96, + 0x82, 0x2b, 0xff, 0x60, 0x2a, 0x14, 0xb5, 0xd7, 0x32, 0x04, 0xa0, 0xfb, 0x4c, 0x72, 0x31, 0xb4, + 0x53, 0x90, 0xa5, 0x60, 0x22, 0x26, 0x8c, 0x26, 0xd0, 0x9c, 0x67, 0xad, 0x1c, 0xf4, 0x16, 0xd9, + 0x30, 0x6a, 0x3a, 0x96, 0xb5, 0xb4, 0xa2, 0x2d, 0x2d, 0x70, 0xe1, 0xe9, 0xba, 0xd9, 0x34, 0xb7, + 0x1f, 0x04, 0x7a, 0x0d, 0xbc, 0x5d, 0x36, 0x31, 0x5f, 0x02, 0x75, 0xa2, 0xd7, 0xd1, 0x99, 0x9d, + 0x3a, 0x0f, 0xf5, 0x2c, 0x6c, 0xb3, 0xb6, 0xc8, 0xc6, 0xe0, 0xd2, 0x6c, 0x2c, 0xf9, 0xd9, 0x78, + 0x1b, 0x55, 0x8d, 0xae, 0xad, 0x20, 0x96, 0x82, 0x17, 0x7f, 0xce, 0xe8, 0xb1, 0x95, 0x55, 0xb4, + 0xcc, 0xe3, 0x44, 0x07, 0xe8, 0x6d, 0xbd, 0xf1, 0xfe, 0x58, 0x0a, 0xa5, 0x12, 0xf6, 0x3f, 0x6c, + 0x8d, 0x51, 0x85, 0x50, 0xc5, 0xdc, 0x44, 0x09, 0xeb, 0xe8, 0x9f, 0x65, 0x74, 0xdd, 0x4f, 0x05, + 0xef, 0x7c, 0xc1, 0x7f, 0x38, 0x5f, 0x69, 0xf1, 0x7c, 0xb8, 0x83, 0xae, 0xfb, 0x6f, 0x72, 0x49, + 0x47, 0xf7, 0xc5, 0x36, 0x6d, 0xe6, 0x54, 0xf0, 0x01, 0x7a, 0xc7, 0xdd, 0x0e, 0xba, 0xd1, 0x56, + 0x96, 0x5b, 0x5b, 0x66, 0x4c, 0xfb, 0x9c, 0x67, 0x6b, 0xfe, 0x15, 0xac, 0xb5, 0xcb, 0xb5, 0xf1, + 0x21, 0xba, 0xed, 0x04, 0x87, 0x92, 0x2b, 0x36, 0xb3, 0xbb, 0xf2, 0xd9, 0xec, 0x5e, 0xa1, 0xee, + 0x1b, 0x86, 0x1d, 0x7b, 0x7b, 0xfd, 0x81, 0x35, 0x5c, 0x7d, 0x43, 0xc3, 0xf3, 0xea, 0xf8, 0x47, + 0xe8, 0xdd, 0xb9, 0x2d, 0x3d, 0xcb, 0xab, 0x9f, 0xcd, 0xf2, 0x55, 0xfa, 0xd1, 0xfb, 0xa8, 0x5e, + 0x54, 0xc8, 0xcb, 0xeb, 0x4c, 0xf4, 0x53, 0xf7, 0x65, 0xe5, 0x17, 0x72, 0xc0, 0x76, 0x92, 0x44, + 0x9c, 0xd9, 0x4f, 0x78, 0x43, 0xfc, 0xdf, 0xbd, 0xe9, 0x36, 0xaa, 0x76, 0x62, 0xfd, 0x37, 0xc7, + 0xcc, 0x65, 0x96, 0x8a, 0x12, 0x1b, 0x95, 0xb6, 0x42, 0xc2, 0x24, 0xdb, 0x4d, 0x68, 0x9e, 0x17, + 0x0d, 0xdb, 0x91, 0x78, 0x0b, 0xa1, 0xbe, 0xe4, 0x42, 0x9a, 0x8f, 0x76, 0x33, 0x80, 0xde, 0x5d, + 0x98, 0x45, 0xe4, 0x31, 0x8d, 0x99, 0x45, 0x9d, 0xbb, 0x21, 0x6e, 0xa6, 0x15, 0x3d, 0x45, 0x78, + 0xb9, 0xb2, 0x43, 0xdf, 0xec, 0xd3, 0x11, 0xcb, 0xa1, 0xdb, 0x9b, 0x7e, 0x5c, 0xd0, 0xb3, 0x97, + 0x33, 0x5f, 0x6c, 0xf6, 0xe5, 0x76, 0xd0, 0xed, 0xcb, 0xf7, 0x84, 0x77, 0x82, 0xe1, 0xc0, 0xf5, + 0x75, 0x58, 0x6b, 0xfb, 0x56, 0x6e, 0xf3, 0xa9, 0xa0, 0xa3, 0x5f, 0x06, 0xf6, 0x01, 0xdc, 0x18, + 0x78, 0x0f, 0x35, 0xb7, 0xd9, 0x31, 0x9d, 0x26, 0xaa, 0x13, 0x7b, 0x9f, 0x7c, 0xf3, 0x4c, 0x40, + 0x75, 0x64, 0x3c, 0xe6, 0x8a, 0xc5, 0x6a, 0x2a, 0x99, 0xfb, 0x3e, 0x98, 0x67, 0xc2, 0xe1, 0x9f, + 0x26, 0x74, 0x94, 0xdb, 0x4f, 0x05, 0x43, 0xe0, 0x6f, 0xa0, 0x1a, 0x4c, 0x68, 0x34, 0x49, 0x72, + 0x9b, 0x70, 0x73, 0x73, 0xa9, 0x11, 0xb9, 0x4f, 0x2a, 0x87, 0x8c, 0x38, 0xba, 0xe1, 0x9f, 0xb3, + 0x23, 0x47, 0x60, 0xbe, 0x97, 0x0e, 0xd9, 0x4b, 0x5b, 0xe1, 0x0d, 0x01, 0xdc, 0x8f, 0x8a, 0xf9, + 0xae, 0x42, 0x0c, 0x01, 0x6f, 0xa0, 0x17, 0xfb, 0x67, 0xc2, 0x96, 0xa5, 0x82, 0xc6, 0x6b, 0xa8, + 0xb4, 0x97, 0xd9, 0xef, 0xfe, 0xd2, 0x5e, 0x16, 0xfd, 0xbc, 0x78, 0x13, 0xb3, 0x39, 0x98, 0xd4, + 0x13, 0x97, 0xfd, 0xd2, 0x37, 0x84, 0x09, 0xa9, 0xa2, 0x43, 0xea, 0x90, 0xd2, 0x6f, 0x73, 0x17, + 0xd5, 0x98, 0x94, 0xa9, 0x90, 0xcc, 0x96, 0xde, 0x9d, 0x6b, 0xa4, 0xe0, 0xe0, 0xb6, 0xf7, 0xd7, + 0xa8, 0xf1, 0xe0, 0x9d, 0xe5, 0x89, 0xbc, 0x23, 0xdd, 0x27, 0x8c, 0x06, 0x6e, 0x21, 0x54, 0x7b, + 0x02, 0xca, 0x84, 0xa9, 0xe8, 0x9b, 0xa8, 0x39, 0x37, 0xf7, 0x82, 0x1f, 0x9e, 0x3f, 0xec, 0xd2, + 0x78, 0xcc, 0x06, 0xf1, 0x98, 0x4d, 0xa8, 0xf3, 0xd6, 0x1c, 0x73, 0xeb, 0x67, 0xc1, 0xab, 0xd7, + 0xad, 0x6b, 0x7f, 0x79, 0xdd, 0xba, 0xf6, 0xaf, 0xd7, 0xad, 0xe0, 0x27, 0x17, 0xad, 0xe0, 0xd7, + 0x17, 0xad, 0xe0, 0xb7, 0x17, 0xad, 0xe0, 0xf7, 0x17, 0xad, 0xe0, 0xd5, 0x45, 0x2b, 0xf8, 0xf3, + 0x45, 0x2b, 0xf8, 0xc7, 0x45, 0x2b, 0xf8, 0xf8, 0x93, 0x37, 0xfc, 0x27, 0x2b, 0x4d, 0xfb, 0x6b, + 0x9f, 0x72, 0xa9, 0x3c, 0x51, 0x76, 0x32, 0x6a, 0xd3, 0x11, 0x4b, 0x95, 0xf7, 0xbf, 0x16, 0x6e, + 0x7a, 0x54, 0xd5, 0xf4, 0xc3, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xce, 0xda, 0xe2, 0x21, 0xfc, + 0x15, 0x00, 0x00, } func (this *Spec) Equal(that interface{}) bool { @@ -2103,6 +2113,30 @@ func (this *Hooks) Equal(that interface{}) bool { return false } } + if len(this.CreateRuntime) != len(that1.CreateRuntime) { + return false + } + for i := range this.CreateRuntime { + if !this.CreateRuntime[i].Equal(&that1.CreateRuntime[i]) { + return false + } + } + if len(this.CreateContainer) != len(that1.CreateContainer) { + return false + } + for i := range this.CreateContainer { + if !this.CreateContainer[i].Equal(&that1.CreateContainer[i]) { + return false + } + } + if len(this.StartContainer) != len(that1.StartContainer) { + return false + } + for i := range this.StartContainer { + if !this.StartContainer[i].Equal(&that1.StartContainer[i]) { + return false + } + } if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { return false } @@ -3674,6 +3708,48 @@ func (m *Hooks) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.StartContainer) > 0 { + for iNdEx := len(m.StartContainer) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.StartContainer[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOci(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } + if len(m.CreateContainer) > 0 { + for iNdEx := len(m.CreateContainer) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.CreateContainer[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOci(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + } + if len(m.CreateRuntime) > 0 { + for iNdEx := len(m.CreateRuntime) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.CreateRuntime[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOci(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } if len(m.Poststop) > 0 { for iNdEx := len(m.Poststop) - 1; iNdEx >= 0; iNdEx-- { { @@ -5277,8 +5353,32 @@ func NewPopulatedHooks(r randyOci, easy bool) *Hooks { this.Poststop[i] = *v21 } } + if r.Intn(5) != 0 { + v22 := r.Intn(5) + this.CreateRuntime = make([]Hook, v22) + for i := 0; i < v22; i++ { + v23 := NewPopulatedHook(r, easy) + this.CreateRuntime[i] = *v23 + } + } + if r.Intn(5) != 0 { + v24 := r.Intn(5) + this.CreateContainer = make([]Hook, v24) + for i := 0; i < v24; i++ { + v25 := NewPopulatedHook(r, easy) + this.CreateContainer[i] = *v25 + } + } + if r.Intn(5) != 0 { + v26 := r.Intn(5) + this.StartContainer = make([]Hook, v26) + for i := 0; i < v26; i++ { + v27 := NewPopulatedHook(r, easy) + this.StartContainer[i] = *v27 + } + } if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedOci(r, 4) + this.XXX_unrecognized = randUnrecognizedOci(r, 7) } return this } @@ -5286,14 +5386,14 @@ func NewPopulatedHooks(r randyOci, easy bool) *Hooks { func NewPopulatedHook(r randyOci, easy bool) *Hook { this := &Hook{} this.Path = string(randStringOci(r)) - v22 := r.Intn(10) - this.Args = make([]string, v22) - for i := 0; i < v22; i++ { + v28 := r.Intn(10) + this.Args = make([]string, v28) + for i := 0; i < v28; i++ { this.Args[i] = string(randStringOci(r)) } - v23 := r.Intn(10) - this.Env = make([]string, v23) - for i := 0; i < v23; i++ { + v29 := r.Intn(10) + this.Env = make([]string, v29) + for i := 0; i < v29; i++ { this.Env[i] = string(randStringOci(r)) } this.Timeout = int64(r.Int63()) @@ -5309,25 +5409,25 @@ func NewPopulatedHook(r randyOci, easy bool) *Hook { func NewPopulatedLinux(r randyOci, easy bool) *Linux { this := &Linux{} if r.Intn(5) != 0 { - v24 := r.Intn(5) - this.UIDMappings = make([]LinuxIDMapping, v24) - for i := 0; i < v24; i++ { - v25 := NewPopulatedLinuxIDMapping(r, easy) - this.UIDMappings[i] = *v25 + v30 := r.Intn(5) + this.UIDMappings = make([]LinuxIDMapping, v30) + for i := 0; i < v30; i++ { + v31 := NewPopulatedLinuxIDMapping(r, easy) + this.UIDMappings[i] = *v31 } } if r.Intn(5) != 0 { - v26 := r.Intn(5) - this.GIDMappings = make([]LinuxIDMapping, v26) - for i := 0; i < v26; i++ { - v27 := NewPopulatedLinuxIDMapping(r, easy) - this.GIDMappings[i] = *v27 + v32 := r.Intn(5) + this.GIDMappings = make([]LinuxIDMapping, v32) + for i := 0; i < v32; i++ { + v33 := NewPopulatedLinuxIDMapping(r, easy) + this.GIDMappings[i] = *v33 } } if r.Intn(5) != 0 { - v28 := r.Intn(10) + v34 := r.Intn(10) this.Sysctl = make(map[string]string) - for i := 0; i < v28; i++ { + for i := 0; i < v34; i++ { this.Sysctl[randStringOci(r)] = randStringOci(r) } } @@ -5336,33 +5436,33 @@ func NewPopulatedLinux(r randyOci, easy bool) *Linux { } this.CgroupsPath = string(randStringOci(r)) if r.Intn(5) != 0 { - v29 := r.Intn(5) - this.Namespaces = make([]LinuxNamespace, v29) - for i := 0; i < v29; i++ { - v30 := NewPopulatedLinuxNamespace(r, easy) - this.Namespaces[i] = *v30 + v35 := r.Intn(5) + this.Namespaces = make([]LinuxNamespace, v35) + for i := 0; i < v35; i++ { + v36 := NewPopulatedLinuxNamespace(r, easy) + this.Namespaces[i] = *v36 } } if r.Intn(5) != 0 { - v31 := r.Intn(5) - this.Devices = make([]LinuxDevice, v31) - for i := 0; i < v31; i++ { - v32 := NewPopulatedLinuxDevice(r, easy) - this.Devices[i] = *v32 + v37 := r.Intn(5) + this.Devices = make([]LinuxDevice, v37) + for i := 0; i < v37; i++ { + v38 := NewPopulatedLinuxDevice(r, easy) + this.Devices[i] = *v38 } } if r.Intn(5) != 0 { this.Seccomp = NewPopulatedLinuxSeccomp(r, easy) } this.RootfsPropagation = string(randStringOci(r)) - v33 := r.Intn(10) - this.MaskedPaths = make([]string, v33) - for i := 0; i < v33; i++ { + v39 := r.Intn(10) + this.MaskedPaths = make([]string, v39) + for i := 0; i < v39; i++ { this.MaskedPaths[i] = string(randStringOci(r)) } - v34 := r.Intn(10) - this.ReadonlyPaths = make([]string, v34) - for i := 0; i < v34; i++ { + v40 := r.Intn(10) + this.ReadonlyPaths = make([]string, v40) + for i := 0; i < v40; i++ { this.ReadonlyPaths[i] = string(randStringOci(r)) } this.MountLabel = string(randStringOci(r)) @@ -5438,11 +5538,11 @@ func NewPopulatedLinuxDevice(r randyOci, easy bool) *LinuxDevice { func NewPopulatedLinuxResources(r randyOci, easy bool) *LinuxResources { this := &LinuxResources{} if r.Intn(5) != 0 { - v35 := r.Intn(5) - this.Devices = make([]LinuxDeviceCgroup, v35) - for i := 0; i < v35; i++ { - v36 := NewPopulatedLinuxDeviceCgroup(r, easy) - this.Devices[i] = *v36 + v41 := r.Intn(5) + this.Devices = make([]LinuxDeviceCgroup, v41) + for i := 0; i < v41; i++ { + v42 := NewPopulatedLinuxDeviceCgroup(r, easy) + this.Devices[i] = *v42 } } if r.Intn(5) != 0 { @@ -5458,11 +5558,11 @@ func NewPopulatedLinuxResources(r randyOci, easy bool) *LinuxResources { this.BlockIO = NewPopulatedLinuxBlockIO(r, easy) } if r.Intn(5) != 0 { - v37 := r.Intn(5) - this.HugepageLimits = make([]LinuxHugepageLimit, v37) - for i := 0; i < v37; i++ { - v38 := NewPopulatedLinuxHugepageLimit(r, easy) - this.HugepageLimits[i] = *v38 + v43 := r.Intn(5) + this.HugepageLimits = make([]LinuxHugepageLimit, v43) + for i := 0; i < v43; i++ { + v44 := NewPopulatedLinuxHugepageLimit(r, easy) + this.HugepageLimits[i] = *v44 } } if r.Intn(5) != 0 { @@ -5564,44 +5664,44 @@ func NewPopulatedLinuxBlockIO(r randyOci, easy bool) *LinuxBlockIO { this := &LinuxBlockIO{} this.Weight = uint32(r.Uint32()) this.LeafWeight = uint32(r.Uint32()) - if r.Intn(5) != 0 { - v39 := r.Intn(5) - this.WeightDevice = make([]LinuxWeightDevice, v39) - for i := 0; i < v39; i++ { - v40 := NewPopulatedLinuxWeightDevice(r, easy) - this.WeightDevice[i] = *v40 - } - } - if r.Intn(5) != 0 { - v41 := r.Intn(5) - this.ThrottleReadBpsDevice = make([]LinuxThrottleDevice, v41) - for i := 0; i < v41; i++ { - v42 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleReadBpsDevice[i] = *v42 - } - } - if r.Intn(5) != 0 { - v43 := r.Intn(5) - this.ThrottleWriteBpsDevice = make([]LinuxThrottleDevice, v43) - for i := 0; i < v43; i++ { - v44 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleWriteBpsDevice[i] = *v44 - } - } if r.Intn(5) != 0 { v45 := r.Intn(5) - this.ThrottleReadIOPSDevice = make([]LinuxThrottleDevice, v45) + this.WeightDevice = make([]LinuxWeightDevice, v45) for i := 0; i < v45; i++ { - v46 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleReadIOPSDevice[i] = *v46 + v46 := NewPopulatedLinuxWeightDevice(r, easy) + this.WeightDevice[i] = *v46 } } if r.Intn(5) != 0 { v47 := r.Intn(5) - this.ThrottleWriteIOPSDevice = make([]LinuxThrottleDevice, v47) + this.ThrottleReadBpsDevice = make([]LinuxThrottleDevice, v47) for i := 0; i < v47; i++ { v48 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleWriteIOPSDevice[i] = *v48 + this.ThrottleReadBpsDevice[i] = *v48 + } + } + if r.Intn(5) != 0 { + v49 := r.Intn(5) + this.ThrottleWriteBpsDevice = make([]LinuxThrottleDevice, v49) + for i := 0; i < v49; i++ { + v50 := NewPopulatedLinuxThrottleDevice(r, easy) + this.ThrottleWriteBpsDevice[i] = *v50 + } + } + if r.Intn(5) != 0 { + v51 := r.Intn(5) + this.ThrottleReadIOPSDevice = make([]LinuxThrottleDevice, v51) + for i := 0; i < v51; i++ { + v52 := NewPopulatedLinuxThrottleDevice(r, easy) + this.ThrottleReadIOPSDevice[i] = *v52 + } + } + if r.Intn(5) != 0 { + v53 := r.Intn(5) + this.ThrottleWriteIOPSDevice = make([]LinuxThrottleDevice, v53) + for i := 0; i < v53; i++ { + v54 := NewPopulatedLinuxThrottleDevice(r, easy) + this.ThrottleWriteIOPSDevice[i] = *v54 } } if !easy && r.Intn(10) != 0 { @@ -5645,11 +5745,11 @@ func NewPopulatedLinuxNetwork(r randyOci, easy bool) *LinuxNetwork { this := &LinuxNetwork{} this.ClassID = uint32(r.Uint32()) if r.Intn(5) != 0 { - v49 := r.Intn(5) - this.Priorities = make([]LinuxInterfacePriority, v49) - for i := 0; i < v49; i++ { - v50 := NewPopulatedLinuxInterfacePriority(r, easy) - this.Priorities[i] = *v50 + v55 := r.Intn(5) + this.Priorities = make([]LinuxInterfacePriority, v55) + for i := 0; i < v55; i++ { + v56 := NewPopulatedLinuxInterfacePriority(r, easy) + this.Priorities[i] = *v56 } } if !easy && r.Intn(10) != 0 { @@ -5681,22 +5781,22 @@ func NewPopulatedLinuxInterfacePriority(r randyOci, easy bool) *LinuxInterfacePr func NewPopulatedLinuxSeccomp(r randyOci, easy bool) *LinuxSeccomp { this := &LinuxSeccomp{} this.DefaultAction = string(randStringOci(r)) - v51 := r.Intn(10) - this.Architectures = make([]string, v51) - for i := 0; i < v51; i++ { + v57 := r.Intn(10) + this.Architectures = make([]string, v57) + for i := 0; i < v57; i++ { this.Architectures[i] = string(randStringOci(r)) } - v52 := r.Intn(10) - this.Flags = make([]string, v52) - for i := 0; i < v52; i++ { + v58 := r.Intn(10) + this.Flags = make([]string, v58) + for i := 0; i < v58; i++ { this.Flags[i] = string(randStringOci(r)) } if r.Intn(5) != 0 { - v53 := r.Intn(5) - this.Syscalls = make([]LinuxSyscall, v53) - for i := 0; i < v53; i++ { - v54 := NewPopulatedLinuxSyscall(r, easy) - this.Syscalls[i] = *v54 + v59 := r.Intn(5) + this.Syscalls = make([]LinuxSyscall, v59) + for i := 0; i < v59; i++ { + v60 := NewPopulatedLinuxSyscall(r, easy) + this.Syscalls[i] = *v60 } } if !easy && r.Intn(10) != 0 { @@ -5719,9 +5819,9 @@ func NewPopulatedLinuxSeccompArg(r randyOci, easy bool) *LinuxSeccompArg { func NewPopulatedLinuxSyscall(r randyOci, easy bool) *LinuxSyscall { this := &LinuxSyscall{} - v55 := r.Intn(10) - this.Names = make([]string, v55) - for i := 0; i < v55; i++ { + v61 := r.Intn(10) + this.Names = make([]string, v61) + for i := 0; i < v61; i++ { this.Names[i] = string(randStringOci(r)) } this.Action = string(randStringOci(r)) @@ -5731,11 +5831,11 @@ func NewPopulatedLinuxSyscall(r randyOci, easy bool) *LinuxSyscall { this.ErrnoRet = NewPopulatedLinuxSyscall_Errnoret(r, easy) } if r.Intn(5) != 0 { - v56 := r.Intn(5) - this.Args = make([]LinuxSeccompArg, v56) - for i := 0; i < v56; i++ { - v57 := NewPopulatedLinuxSeccompArg(r, easy) - this.Args[i] = *v57 + v62 := r.Intn(5) + this.Args = make([]LinuxSeccompArg, v62) + for i := 0; i < v62; i++ { + v63 := NewPopulatedLinuxSeccompArg(r, easy) + this.Args[i] = *v63 } } if !easy && r.Intn(10) != 0 { @@ -5777,9 +5877,9 @@ func randUTF8RuneOci(r randyOci) rune { return rune(ru + 61) } func randStringOci(r randyOci) string { - v58 := r.Intn(100) - tmps := make([]rune, v58) - for i := 0; i < v58; i++ { + v64 := r.Intn(100) + tmps := make([]rune, v64) + for i := 0; i < v64; i++ { tmps[i] = randUTF8RuneOci(r) } return string(tmps) @@ -5801,11 +5901,11 @@ func randFieldOci(dAtA []byte, r randyOci, fieldNumber int, wire int) []byte { switch wire { case 0: dAtA = encodeVarintPopulateOci(dAtA, uint64(key)) - v59 := r.Int63() + v65 := r.Int63() if r.Intn(2) == 0 { - v59 *= -1 + v65 *= -1 } - dAtA = encodeVarintPopulateOci(dAtA, uint64(v59)) + dAtA = encodeVarintPopulateOci(dAtA, uint64(v65)) case 1: dAtA = encodeVarintPopulateOci(dAtA, uint64(key)) dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) @@ -6133,6 +6233,24 @@ func (m *Hooks) Size() (n int) { n += 1 + l + sovOci(uint64(l)) } } + if len(m.CreateRuntime) > 0 { + for _, e := range m.CreateRuntime { + l = e.Size() + n += 1 + l + sovOci(uint64(l)) + } + } + if len(m.CreateContainer) > 0 { + for _, e := range m.CreateContainer { + l = e.Size() + n += 1 + l + sovOci(uint64(l)) + } + } + if len(m.StartContainer) > 0 { + for _, e := range m.StartContainer { + l = e.Size() + n += 1 + l + sovOci(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6797,7 +6915,7 @@ func (this *Spec) String() string { } repeatedStringForMounts += "}" keysForAnnotations := make([]string, 0, len(this.Annotations)) - for k := range this.Annotations { + for k, _ := range this.Annotations { keysForAnnotations = append(keysForAnnotations, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForAnnotations) @@ -6948,10 +7066,28 @@ func (this *Hooks) String() string { repeatedStringForPoststop += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," } repeatedStringForPoststop += "}" + repeatedStringForCreateRuntime := "[]Hook{" + for _, f := range this.CreateRuntime { + repeatedStringForCreateRuntime += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," + } + repeatedStringForCreateRuntime += "}" + repeatedStringForCreateContainer := "[]Hook{" + for _, f := range this.CreateContainer { + repeatedStringForCreateContainer += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," + } + repeatedStringForCreateContainer += "}" + repeatedStringForStartContainer := "[]Hook{" + for _, f := range this.StartContainer { + repeatedStringForStartContainer += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," + } + repeatedStringForStartContainer += "}" s := strings.Join([]string{`&Hooks{`, `Prestart:` + repeatedStringForPrestart + `,`, `Poststart:` + repeatedStringForPoststart + `,`, `Poststop:` + repeatedStringForPoststop + `,`, + `CreateRuntime:` + repeatedStringForCreateRuntime + `,`, + `CreateContainer:` + repeatedStringForCreateContainer + `,`, + `StartContainer:` + repeatedStringForStartContainer + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -6996,7 +7132,7 @@ func (this *Linux) String() string { } repeatedStringForDevices += "}" keysForSysctl := make([]string, 0, len(this.Sysctl)) - for k := range this.Sysctl { + for k, _ := range this.Sysctl { keysForSysctl = append(keysForSysctl, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForSysctl) @@ -9298,6 +9434,108 @@ func (m *Hooks) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateRuntime", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOci + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOci + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOci + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CreateRuntime = append(m.CreateRuntime, Hook{}) + if err := m.CreateRuntime[len(m.CreateRuntime)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateContainer", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOci + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOci + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOci + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CreateContainer = append(m.CreateContainer, Hook{}) + if err := m.CreateContainer[len(m.CreateContainer)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StartContainer", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOci + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOci + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOci + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StartContainer = append(m.StartContainer, Hook{}) + if err := m.StartContainer[len(m.StartContainer)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipOci(dAtA[iNdEx:]) diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go index 3b65e1152..b56bfabb6 100644 --- a/src/runtime/virtcontainers/pkg/mock/mock.go +++ b/src/runtime/virtcontainers/pkg/mock/mock.go @@ -125,6 +125,10 @@ func (p *HybridVSockTTRPCMockImp) UpdateContainer(ctx context.Context, req *pb.U return emptyResp, nil } +func (p *HybridVSockTTRPCMockImp) UpdateEphemeralMounts(ctx context.Context, req *pb.UpdateEphemeralMountsRequest) (*gpb.Empty, error) { + return emptyResp, nil +} + func (p *HybridVSockTTRPCMockImp) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*gpb.Empty, error) { return emptyResp, nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index bc4e2085c..aa87247d9 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1797,7 +1797,7 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op return err } - devID := device.ID + devID := *(*device).GetID() machineType := q.HypervisorConfig().HypervisorMachineType if op == AddDevice { @@ -1814,29 +1814,31 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. if q.state.HotplugVFIOOnRootBus { - - // In case MachineType is q35, a PCIe device is hotplugged on a PCIe Root Port. - switch machineType { - case QemuQ35: - if device.IsPCIe && q.state.PCIeRootPort <= 0 { - q.Logger().WithField("dev-id", device.ID).Warn("VFIO device is a PCIe device. It's recommended to add the PCIe Root Port by setting the pcie_root_port parameter in the configuration for q35") - device.Bus = "" + switch (*device).GetType() { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + // In case MachineType is q35, a PCIe device is hotplugged on a PCIe Root Port. + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device) } - default: - device.Bus = "" - } + switch machineType { + case QemuQ35: + if pciDevice.IsPCIe && q.state.PCIeRootPort <= 0 { + q.Logger().WithField("dev-id", (*device).GetID()).Warn("VFIO device is a PCIe device. It's recommended to add the PCIe Root Port by setting the pcie_root_port parameter in the configuration for q35") + pciDevice.Bus = "" + } + default: + pciDevice.Bus = "" + } + *device = pciDevice - switch device.Type { - case config.VFIODeviceNormalType: - err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, device.Bus, romFile) - case config.VFIODeviceMediatedType: - if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { - err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) + if pciDevice.Type == config.VFIOPCIDeviceNormalType { + err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, pciDevice.Bus, romFile) } else { - err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", device.Bus, romFile) + err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), "", pciDevice.Bus, romFile) } - default: - return fmt.Errorf("Incorrect VFIO device type found") + case config.VFIOAPDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev()) } } else { addr, bridge, err := q.arch.addDeviceToBridge(ctx, devID, types.PCI) @@ -1850,15 +1852,17 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op } }() - switch device.Type { - case config.VFIODeviceNormalType: - err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile) - case config.VFIODeviceMediatedType: - if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { - err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) - } else { - err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile) + switch (*device).GetType() { + case config.VFIOPCIDeviceNormalType: + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device) } + err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, addr, bridge.ID, romFile) + case config.VFIOPCIDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), addr, bridge.ID, romFile) + case config.VFIOAPDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev()) default: return fmt.Errorf("Incorrect VFIO device type found") } @@ -1866,13 +1870,24 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op if err != nil { return err } - // XXX: Depending on whether we're doing root port or - // bridge hotplug, and how the bridge is set up in - // other parts of the code, we may or may not already - // have information about the slot number of the - // bridge and or the device. For simplicity, just - // query both of them back from qemu - device.GuestPciPath, err = q.qomGetPciPath(devID) + + switch (*device).GetType() { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device) + } + // XXX: Depending on whether we're doing root port or + // bridge hotplug, and how the bridge is set up in + // other parts of the code, we may or may not already + // have information about the slot number of the + // bridge and or the device. For simplicity, just + // query both of them back from qemu + guestPciPath, err := q.qomGetPciPath(devID) + pciDevice.GuestPciPath = guestPciPath + *device = pciDevice + return err + } return err } else { q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device") diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 7b61bc052..3de1aabc9 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -708,16 +708,17 @@ func (q *qemuArchBase) appendVhostUserDevice(ctx context.Context, devices []govm } func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDev config.VFIODev) []govmmQemu.Device { - if vfioDev.BDF == "" { + pciDevice := vfioDev.(config.VFIOPCIDev) + if pciDevice.BDF == "" { return devices } devices = append(devices, govmmQemu.VFIODevice{ - BDF: vfioDev.BDF, - VendorID: vfioDev.VendorID, - DeviceID: vfioDev.DeviceID, - Bus: vfioDev.Bus, + BDF: pciDevice.BDF, + VendorID: pciDevice.VendorID, + DeviceID: pciDevice.DeviceID, + Bus: pciDevice.Bus, }, ) diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 37611bb5b..51c11bd91 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -463,7 +463,7 @@ func TestQemuArchBaseAppendVFIODevice(t *testing.T) { }, } - vfDevice := config.VFIODev{ + vfDevice := config.VFIOPCIDev{ BDF: bdf, } @@ -483,7 +483,7 @@ func TestQemuArchBaseAppendVFIODeviceWithVendorDeviceID(t *testing.T) { }, } - vfDevice := config.VFIODev{ + vfDevice := config.VFIOPCIDev{ BDF: bdf, VendorID: vendorID, DeviceID: deviceID, diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 5177aee9a..32b20ab93 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -17,7 +17,9 @@ import ( "os" "os/exec" "path/filepath" + //"strconv" + "strings" "sync" "syscall" @@ -48,6 +50,9 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" ) // sandboxTracingTags defines tags for the trace span @@ -1864,11 +1869,15 @@ func (s *Sandbox) HotplugAddDevice(ctx context.Context, device api.Device, devTy // adding a group of VFIO devices for _, dev := range vfioDevices { if _, err := s.hypervisor.HotplugAddDevice(ctx, dev, VfioDev); err != nil { + bdf := "" + if pciDevice, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDevice.BDF + } s.Logger(). WithFields(logrus.Fields{ "sandbox": s.id, - "vfio-device-ID": dev.ID, - "vfio-device-BDF": dev.BDF, + "vfio-device-ID": (*dev).GetID(), + "vfio-device-BDF": bdf, }).WithError(err).Error("failed to hotplug VFIO device") return err } @@ -1917,11 +1926,15 @@ func (s *Sandbox) HotplugRemoveDevice(ctx context.Context, device api.Device, de // remove a group of VFIO devices for _, dev := range vfioDevices { if _, err := s.hypervisor.HotplugRemoveDevice(ctx, dev, VfioDev); err != nil { + bdf := "" + if pciDevice, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDevice.BDF + } s.Logger().WithError(err). WithFields(logrus.Fields{ "sandbox": s.id, - "vfio-device-ID": dev.ID, - "vfio-device-BDF": dev.BDF, + "vfio-device-ID": (*dev).GetID(), + "vfio-device-BDF": bdf, }).Error("failed to hot unplug VFIO device") return err } @@ -2121,11 +2134,69 @@ func (s *Sandbox) updateResources(ctx context.Context) error { if newMemoryMB == finalMemoryMB { break } + } + tmpfsMounts, err := s.prepareEphemeralMounts(finalMemoryMB) + if err != nil { + return err + } + if err := s.agent.updateEphemeralMounts(ctx, tmpfsMounts); err != nil { + // upgrade path: if runtime is newer version, but agent is old + // then ignore errUnimplemented + if grpcStatus.Convert(err).Code() == codes.Unimplemented { + s.Logger().Warnf("agent does not support updateMounts") + return nil + } + return err } return nil +} +func (s *Sandbox) prepareEphemeralMounts(memoryMB uint32) ([]*grpc.Storage, error) { + tmpfsMounts := []*grpc.Storage{} + for _, c := range s.containers { + for _, mount := range c.mounts { + // if a tmpfs ephemeral mount is present + // update its size to occupy the entire sandbox's memory + if mount.Type == KataEphemeralDevType { + sizeLimited := false + for _, opt := range mount.Options { + if strings.HasPrefix(opt, "size") { + sizeLimited = true + } + } + if sizeLimited { // do not resize sizeLimited emptyDirs + continue + } + + mountOptions := []string{"remount", fmt.Sprintf("size=%dM", memoryMB)} + + origin_src := mount.Source + stat := syscall.Stat_t{} + err := syscall.Stat(origin_src, &stat) + if err != nil { + return nil, err + } + + // if volume's gid isn't root group(default group), this means there's + // an specific fsGroup is set on this local volume, then it should pass + // to guest. + if stat.Gid != 0 { + mountOptions = append(mountOptions, fmt.Sprintf("%s=%d", fsGid, stat.Gid)) + } + + tmpfsMounts = append(tmpfsMounts, &grpc.Storage{ + Driver: KataEphemeralDevType, + MountPoint: filepath.Join(ephemeralPath(), filepath.Base(mount.Source)), + Source: "tmpfs", + Fstype: "tmpfs", + Options: mountOptions, + }) + } + } + } + return tmpfsMounts, nil } func (s *Sandbox) updateMemory(ctx context.Context, newMemoryMB uint32) error { diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 8b86fa995..de3b1885c 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -179,6 +179,65 @@ func TestCalculateSandboxMem(t *testing.T) { } } +func TestPrepareEphemeralMounts(t *testing.T) { + sandbox := &Sandbox{} + sandbox.containers = map[string]*Container{ + "container1": { + mounts: []Mount{ + { + // happy path + Type: KataEphemeralDevType, + Options: []string{ + "rw", + "relatime", + }, + Source: "/tmp", + }, + { + // should be ignored because it is not KataEphemeralDevType + Type: KataLocalDevType, + Options: []string{ + "rw", + "relatime", + }, + Source: "/tmp", + }, + { + // should be ignored because it is sizeLimited + Type: KataLocalDevType, + Options: []string{ + "rw", + "relatime", + "size=1M", + }, + Source: "/tmp", + }, + }, + }, + } + + storages, err := sandbox.prepareEphemeralMounts(1024) + assert.NoError(t, err) + assert.Equal(t, len(storages), 1) + for _, s := range storages { + assert.Equal(t, s.Driver, KataEphemeralDevType) + assert.Equal(t, s.Source, "tmpfs") + assert.Equal(t, s.Fstype, "tmpfs") + assert.Equal(t, s.MountPoint, filepath.Join(ephemeralPath(), "tmp")) + assert.Equal(t, len(s.Options), 2) // remount, size=1024M + + validSet := map[string]struct{}{ + "remount": {}, + "size=1024M": {}, + } + for _, opt := range s.Options { + if _, ok := validSet[opt]; !ok { + t.Error("invalid mount opt: " + opt) + } + } + } +} + func TestCalculateSandboxMemHandlesNegativeLimits(t *testing.T) { sandbox := &Sandbox{} sandbox.config = &SandboxConfig{} diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go index 40c5c360e..a31b8d351 100644 --- a/src/runtime/virtcontainers/utils/utils_linux.go +++ b/src/runtime/virtcontainers/utils/utils_linux.go @@ -89,8 +89,7 @@ func FindContextID() (*os.File, uint64, error) { const ( procMountsFile = "/proc/mounts" - fieldsPerLine = 6 - vfioAPSysfsDir = "vfio_ap" + fieldsPerLine = 6 ) const ( @@ -142,18 +141,6 @@ func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string } } -// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device -// by checking for the existence of "vfio_ap" in the path -func IsAPVFIOMediatedDevice(sysfsdev string) bool { - split := strings.Split(sysfsdev, string(os.PathSeparator)) - for _, el := range split { - if el == vfioAPSysfsDir { - return true - } - } - return false -} - func waitProcessUsingPidfd(pid int, timeoutSecs uint, logger *logrus.Entry) (bool, error) { pidfd, err := unix.PidfdOpen(pid, 0) diff --git a/src/runtime/virtcontainers/utils/utils_linux_test.go b/src/runtime/virtcontainers/utils/utils_linux_test.go index dbf9fde38..c8e213c2c 100644 --- a/src/runtime/virtcontainers/utils/utils_linux_test.go +++ b/src/runtime/virtcontainers/utils/utils_linux_test.go @@ -63,19 +63,3 @@ func TestGetDevicePathAndFsTypeOptionsSuccessful(t *testing.T) { assert.Equal(fstype, fstypeOut) assert.Equal(fsOptions, optsOut) } - -func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) { - assert := assert.New(t) - - // Should be false for a PCI device - isAPMdev := IsAPVFIOMediatedDevice("/sys/bus/pci/devices/0000:00:02.0/a297db4a-f4c2-11e6-90f6-d3b88d6c9525") - assert.False(isAPMdev) -} - -func TestIsAPVFIOMediatedDeviceTrue(t *testing.T) { - assert := assert.New(t) - - // Typical AP sysfsdev - isAPMdev := IsAPVFIOMediatedDevice("/sys/devices/vfio_ap/matrix/a297db4a-f4c2-11e6-90f6-d3b88d6c9525") - assert.True(isAPMdev) -} diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index deb3385f6..234830c3a 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -250,9 +250,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock index 9a8e92748..f9ebce67b 100644 --- a/src/tools/runk/Cargo.lock +++ b/src/tools/runk/Cargo.lock @@ -233,9 +233,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/tools/runk/libcontainer/Cargo.toml b/src/tools/runk/libcontainer/Cargo.toml index 82e56b06e..8df62d3b2 100644 --- a/src/tools/runk/libcontainer/Cargo.toml +++ b/src/tools/runk/libcontainer/Cargo.toml @@ -20,7 +20,7 @@ chrono = { version = "0.4.19", features = ["serde"] } serde = { version = "1.0.133", features = ["derive"] } serde_json = "1.0.74" scopeguard = "1.1.0" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } procfs = "0.14.0" [dev-dependencies] diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 9b75217fb..18ac9aed9 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -41,6 +41,9 @@ readonly qemu_cc_builder="${static_build_dir}/qemu/build-static-qemu-cc.sh" source "${script_dir}/../../scripts/lib.sh" +readonly jenkins_url="http://jenkins.katacontainers.io" +readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" + ARCH=$(uname -m) workdir="${WORKDIR:-$PWD}" @@ -542,12 +545,52 @@ install_cc_sev_ovmf(){ #Install guest image install_image() { + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-image-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-image" + + local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + + install_cached_component \ + "${component}" \ + "${jenkins}" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "Create image" "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" } #Install guest initrd install_initrd() { + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-initrd-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-initrd" + + local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + + install_cached_component \ + "${component}" \ + "${jenkins}" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-initrd" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "Create initrd" "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" } @@ -560,13 +603,35 @@ install_cc_initrd() { #Install kernel asset install_kernel() { export kernel_version="$(yq r $versions_yaml assets.kernel.version)" + local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + + install_cached_component \ + "kernel" \ + "${jenkins_url}/job/kata-containers-main-kernel-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -f -v "${kernel_version}" } #Install dragonball experimental kernel asset install_dragonball_experimental_kernel() { info "build dragonball experimental kernel" - export kernel_version="$(yq r $versions_yaml assets.dragonball-kernel-experimental.version)" + export kernel_version="$(yq r $versions_yaml assets.kernel-dragonball-experimental.version)" + local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + + install_cached_component \ + "kernel-dragonball-experimental" \ + "${jenkins_url}/job/kata-containers-main-kernel-dragonball-experimental-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -e -t dragonball -v ${kernel_version} } @@ -575,21 +640,51 @@ install_dragonball_experimental_kernel() { install_experimental_kernel() { info "build experimental kernel" export kernel_version="$(yq r $versions_yaml assets.kernel-experimental.tag)" + local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + + install_cached_component \ + "kernel-experimental" \ + "${jenkins_url}/job/kata-containers-main-kernel-experimental-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "Kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -f -b experimental -v ${kernel_version} } # Install static qemu asset install_qemu() { - info "build static qemu" export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" + + install_cached_component \ + "QEMU" \ + "${jenkins_url}/job/kata-containers-main-qemu-$(uname -m)/${cached_artifacts_path}" \ + "${qemu_version}-$(calc_qemu_files_sha256sum)" \ + "$(get_qemu_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + + info "build static qemu" "${qemu_builder}" tar xvf "${builddir}/kata-static-qemu.tar.gz" -C "${destdir}" } # Install static firecracker asset install_firecracker() { + install_cached_component \ + "firecracker" \ + "${jenkins_url}/job/kata-containers-main-firecracker-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.hypervisor.firecracker.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static firecracker" "${firecracker_builder}" info "Install static firecracker" @@ -600,6 +695,15 @@ install_firecracker() { # Install static cloud-hypervisor asset install_clh() { + install_cached_component \ + "cloud-hypervisor" \ + "${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + if [[ "${ARCH}" == "x86_64" ]]; then export features="tdx" fi @@ -613,6 +717,15 @@ install_clh() { # Install static virtiofsd asset install_virtiofsd() { + install_cached_component \ + "virtiofsd" \ + "${jenkins_url}/job/kata-containers-main-virtiofsd-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" \ + "$(get_virtiofsd_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static virtiofsd" "${virtiofsd_builder}" info "Install static virtiofsd" @@ -622,6 +735,15 @@ install_virtiofsd() { # Install static nydus asset install_nydus() { + install_cached_component \ + "nydus" \ + "${jenkins_url}/job/kata-containers-main-nydus-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "externals.nydus.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static nydus" "${nydus_builder}" info "Install static nydus" @@ -633,8 +755,22 @@ install_nydus() { #Install all components that are not assets install_shimv2() { - GO_VERSION="$(yq r ${versions_yaml} languages.golang.meta.newest-version)" - RUST_VERSION="$(yq r ${versions_yaml} languages.rust.meta.newest-version)" + local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" + local runtime_rs_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime-rs")" + local protocols_last_commit="$(get_last_modification "${repo_root_dir}/src/libs/protocols")" + local GO_VERSION="$(get_from_kata_deps "languages.golang.meta.newest-version")" + local RUST_VERSION="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local shim_v2_version="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${GO_VERSION}-${RUST_VERSION}" + + install_cached_component \ + "shim-v2" \ + "${jenkins_url}/job/kata-containers-main-shim-v2-$(uname -m)/${cached_artifacts_path}" \ + "${shim_v2_version}" \ + "$(get_shim_v2_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + export GO_VERSION export RUST_VERSION DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 68aead413..203114312 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -557,7 +557,7 @@ main() { case "${arch_target}" in "aarch64") build_type="arm-experimental" - kernel_version=$(get_from_kata_deps "assets.arm-kernel-experimental.version") + kernel_version=$(get_from_kata_deps "assets.kernel-arm-experimental.version") ;; *) info "No arch-specific experimental kernel supported, using experimental one instead" @@ -565,7 +565,7 @@ main() { ;; esac elif [[ ${build_type} == "dragonball-experimental" ]]; then - kernel_version=$(get_from_kata_deps "assets.dragonball-kernel-experimental.version") + kernel_version=$(get_from_kata_deps "assets.kernel-dragonball-experimental.version") elif [[ "${conf_guest}" != "" ]]; then #If specifying a tag for kernel_version, must be formatted version-like to avoid unintended parsing issues kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index 0798fad1c..c6e1b79de 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -101cc +102cc diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index b884cb0a3..4a1198842 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -185,9 +185,9 @@ sha256sum_from_files() { } calc_qemu_files_sha256sum() { - local files="${this_script_dir}/../qemu \ - ${this_script_dir}/../static-build/qemu.blacklist \ - ${this_script_dir}/../static-build/scripts" + local files="${this_script_dir}/tools/packaging/qemu \ + ${this_script_dir}/../tools/packaging/static-build/qemu.blacklist \ + ${this_script_dir}/../tools/packaging/static-build/scripts" sha256sum_from_files "$files" } @@ -198,7 +198,7 @@ get_initramfs_image_name() { } get_kernel_image_name() { - kernel_script_dir="${this_script_dir}/../static-build/kernel" + kernel_script_dir="${repo_root_dir}/tools/packaging/static-build/kernel" echo "${CC_BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)" } @@ -208,12 +208,12 @@ get_ovmf_image_name() { } get_qemu_image_name() { - qemu_script_dir="${this_script_dir}/../static-build/qemu" + qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu" echo "${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)" } get_shim_v2_image_name() { - shim_v2_script_dir="${this_script_dir}/../static-build/shim-v2" + shim_v2_script_dir="${repo_root_dir}/tools/packaging/static-build/shim-v2" echo "${CC_BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" } @@ -239,6 +239,6 @@ get_virtiofsd_image_name() { ;; esac - virtiofsd_script_dir="${this_script_dir}/../static-build/virtiofsd" + virtiofsd_script_dir="${repo_root_dir}/tools/packaging/static-build/virtiofsd" echo "${CC_BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)" } diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh new file mode 100755 index 000000000..e447ab4bf --- /dev/null +++ b/tools/packaging/static-build/cache_components_main.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${script_dir}/../scripts/lib.sh" + +KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-experimental | kernel-arm-experimetnal | kernel-dragonball-experimental +ROOTFS_IMAGE_TYPE="${ROOTFS_IMAGE_TYPE:-image}" # image | initrd + +cache_clh_artifacts() { + local clh_tarball_name="kata-static-cloud-hypervisor.tar.xz" + local current_clh_version="$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" + create_cache_asset "${clh_tarball_name}" "${current_clh_version}" "" +} + +cache_firecracker_artifacts() { + local fc_tarball_name="kata-static-firecracker.tar.xz" + local current_fc_version="$(get_from_kata_deps "assets.hypervisor.firecracker.version")" + create_cache_asset "${fc_tarball_name}" "${current_fc_version}" "" +} + +cache_kernel_artifacts() { + local kernel_tarball_name="kata-static-${KERNEL_FLAVOUR}.tar.xz" + local current_kernel_image="$(get_kernel_image_name)" + local current_kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")-${current_kernel_kata_config_version}" + create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}" +} + +cache_nydus_artifacts() { + local nydus_tarball_name="kata-static-nydus.tar.xz" + local current_nydus_version="$(get_from_kata_deps "externals.nydus.version")" + create_cache_asset "${nydus_tarball_name}" "${current_nydus_version}" "" +} + +cache_qemu_artifacts() { + local qemu_tarball_name="kata-static-qemu.tar.xz" + local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") + local qemu_sha=$(calc_qemu_files_sha256sum) + local current_qemu_image="$(get_qemu_image_name)" + create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}" +} + +cache_rootfs_artifacts() { + local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local rootfs_tarball_name="kata-static-rootfs-${ROOTFS_IMAGE_TYPE}.tar.xz" + local current_rootfs_version="${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${ROOTFS_IMAGE_TYPE}" + create_cache_asset "${rootfs_tarball_name}" "${current_rootfs_version}" "" +} + +cache_shim_v2_artifacts() { + local shim_v2_tarball_name="kata-static-shim-v2.tar.xz" + local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" + local protocols_last_commit="$(get_last_modification "${repo_root_dir}/src/libs/protocols")" + local runtime_rs_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime-rs")" + local golang_version="$(get_from_kata_deps "languages.golang.meta.newest-version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local current_shim_v2_version="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${golang_version}-${rust_version}" + local current_shim_v2_image="$(get_shim_v2_image_name)" + create_cache_asset "${shim_v2_tarball_name}" "${current_shim_v2_version}" "${current_shim_v2_image}" +} + +cache_virtiofsd_artifacts() { + local virtiofsd_tarball_name="kata-static-virtiofsd.tar.xz" + local current_virtiofsd_version="$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" + local current_virtiofsd_image="$(get_virtiofsd_image_name)" + create_cache_asset "${virtiofsd_tarball_name}" "${current_virtiofsd_version}" "${current_virtiofsd_image}" +} + +create_cache_asset() { + local component_name="${1}" + local component_version="${2}" + local component_image="${3}" + + sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" . + sudo chown -R "${USER}:${USER}" . + sha256sum "${component_name}" > "sha256sum-${component_name}" + cat "sha256sum-${component_name}" + echo "${component_version}" > "latest" + cat "latest" + echo "${component_image}" > "latest_image" + cat "latest_image" +} + +help() { +echo "$(cat << EOF +Usage: $0 "[options]" + Description: + Builds the cache of several kata components. + Options: + -c Cloud hypervisor cache + -F Firecracker cache + -k Kernel cache + * Export KERNEL_FLAVOUR="kernel|kernek-experimental|kernel-arm-experimental|kernel-dragonball-experimental" for a specific build + The default KERNEL_FLAVOUR value is "kernel" + -n Nydus cache + -q QEMU cache + -r RootFS cache + * Export ROOTFS_IMAGE_TYPE="image|initrd" for one of those two types + The default ROOTFS_IMAGE_TYPE value is "image" + -s Shim v2 cache + -v VirtioFS cache + -h Shows help +EOF +)" +} + +main() { + local cloud_hypervisor_component="${cloud_hypervisor_component:-}" + local firecracker_component="${firecracker_component:-}" + local kernel_component="${kernel_component:-}" + local nydus_component="${nydus_component:-}" + local qemu_component="${qemu_component:-}" + local rootfs_component="${rootfs_component:-}" + local shim_v2_component="${shim_v2_component:-}" + local virtiofsd_component="${virtiofsd_component:-}" + local OPTIND + while getopts ":cFknqrsvh:" opt + do + case "$opt" in + c) + cloud_hypervisor_component="1" + ;; + F) + firecracker_component="1" + ;; + k) + kernel_component="1" + ;; + n) + nydus_component="1" + ;; + q) + qemu_component="1" + ;; + r) + rootfs_component="1" + ;; + s) + shim_v2_component="1" + ;; + v) + virtiofsd_component="1" + ;; + h) + help + exit 0; + ;; + :) + echo "Missing argument for -$OPTARG"; + help + exit 1; + ;; + esac + done + shift $((OPTIND-1)) + + [[ -z "${cloud_hypervisor_component}" ]] && \ + [[ -z "${firecracker_component}" ]] && \ + [[ -z "${kernel_component}" ]] && \ + [[ -z "${nydus_component}" ]] && \ + [[ -z "${qemu_component}" ]] && \ + [[ -z "${rootfs_component}" ]] && \ + [[ -z "${shim_v2_component}" ]] && \ + [[ -z "${virtiofsd_component}" ]] && \ + help && die "Must choose at least one option" + + mkdir -p "${WORKSPACE}/artifacts" + pushd "${WORKSPACE}/artifacts" + echo "Artifacts:" + + [ "${cloud_hypervisor_component}" == "1" ] && cache_clh_artifacts + [ "${firecracker_component}" == "1" ] && cache_firecracker_artifacts + [ "${kernel_component}" == "1" ] && cache_kernel_artifacts + [ "${nydus_component}" == "1" ] && cache_nydus_artifacts + [ "${qemu_component}" == "1" ] && cache_qemu_artifacts + [ "${rootfs_component}" == "1" ] && cache_rootfs_artifacts + [ "${shim_v2_component}" == "1" ] && cache_shim_v2_artifacts + [ "${virtiofsd_component}" == "1" ] && cache_virtiofsd_artifacts + + ls -la "${WORKSPACE}/artifacts/" + popd + sync +} + +main "$@" diff --git a/versions.yaml b/versions.yaml index 500ac289c..ab0112d27 100644 --- a/versions.yaml +++ b/versions.yaml @@ -83,7 +83,7 @@ assets: uscan-url: >- https://github.com/firecracker-microvm/firecracker/tags .*/v?(\d\S+)\.tar\.gz - version: "v1.1.0" + version: "v1.3.1" qemu: description: "VMM that uses KVM" @@ -175,12 +175,12 @@ assets: url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" tag: "v5.13.10" - arm-kernel-experimental: + kernel-arm-experimental: description: "Linux kernel with cpu/mem hotplug support on arm64" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" version: "v5.15.7" - dragonball-kernel-experimental: + kernel-dragonball-experimental: description: "Linux kernel with Dragonball VMM optimizations like upcall" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" version: "v5.10.25"