diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 173c0f7b7..06f2899b6 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -264,12 +264,12 @@ parts: # download source qemu_dir=${SNAPCRAFT_STAGE}/qemu rm -rf "${qemu_dir}" - git clone --branch ${branch} --single-branch ${url} "${qemu_dir}" + git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}" cd ${qemu_dir} [ -z "${commit}" ] || git checkout ${commit} - [ -n "$(ls -A ui/keycodemapdb)" ] || git clone https://github.com/qemu/keycodemapdb ui/keycodemapdb/ - [ -n "$(ls -A capstone)" ] || git clone https://github.com/qemu/capstone capstone + [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/ + [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone # Apply branch patches [ -d "${patches_version_dir}" ] || mkdir "${patches_version_dir}" @@ -320,20 +320,23 @@ parts: plugin: nil after: [godeps] override-build: | - sudo apt-get -y update - sudo apt-get -y install ca-certificates curl gnupg lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get -y update - sudo apt-get -y install docker-ce docker-ce-cli containerd.io - sudo systemctl start docker.socket + arch=$(uname -m) + if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then + sudo apt-get -y update + sudo apt-get -y install ca-certificates curl gnupg lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get -y update + sudo apt-get -y install docker-ce docker-ce-cli containerd.io + sudo systemctl start docker.socket - export GOPATH=${SNAPCRAFT_STAGE}/gopath - kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME} - cd ${kata_dir} - sudo -E NO_TTY=true make cloud-hypervisor-tarball - tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/ - install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor + export GOPATH=${SNAPCRAFT_STAGE}/gopath + kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME} + cd ${kata_dir} + sudo -E NO_TTY=true make cloud-hypervisor-tarball + tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/ + install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor + fi apps: runtime: diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 0f9d3a3ec..916ec6117 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -367,6 +367,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.5" @@ -993,6 +1017,7 @@ dependencies = [ "slog", "slog-scope", "slog-stdlog", + "sysinfo", "tempfile", "thiserror", "tokio", @@ -1879,6 +1904,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -2281,6 +2331,21 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sysinfo" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e757000a4bed2b1be9be65a3f418b9696adf30bb419214c73997422de73a591" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "take_mut" version = "0.2.2" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index d513859d9..33cad6714 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -20,6 +20,7 @@ scopeguard = "1.0.0" thiserror = "1.0.26" regex = "1.5.4" serial_test = "0.5.1" +sysinfo = "0.23.0" # Async helpers async-trait = "0.1.42" diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 942a625d5..c4cc8a095 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -215,7 +215,6 @@ pub trait BaseContainer { async fn start(&mut self, p: Process) -> Result<()>; async fn run(&mut self, p: Process) -> Result<()>; async fn destroy(&mut self) -> Result<()>; - fn signal(&self, sig: Signal, all: bool) -> Result<()>; fn exec(&mut self) -> Result<()>; } @@ -1057,18 +1056,6 @@ impl BaseContainer for LinuxContainer { Ok(()) } - fn signal(&self, sig: Signal, all: bool) -> Result<()> { - if all { - for pid in self.processes.keys() { - signal::kill(Pid::from_raw(*pid), Some(sig))?; - } - } - - signal::kill(Pid::from_raw(self.init_process_pid), Some(sig))?; - - Ok(()) - } - fn exec(&mut self) -> Result<()> { let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME); let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?; @@ -2049,14 +2036,6 @@ mod tests { assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret); } - #[test] - fn test_linuxcontainer_signal() { - let ret = new_linux_container_and_then(|c: LinuxContainer| { - c.signal(nix::sys::signal::SIGCONT, true) - }); - assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret); - } - #[test] fn test_linuxcontainer_exec() { let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec()); diff --git a/src/agent/samples/configuration-all-endpoints.toml b/src/agent/samples/configuration-all-endpoints.toml index 1c8b01367..8e6dd9c3c 100644 --- a/src/agent/samples/configuration-all-endpoints.toml +++ b/src/agent/samples/configuration-all-endpoints.toml @@ -25,6 +25,7 @@ allowed = [ "ReadStreamRequest", "RemoveContainerRequest", "ReseedRandomDevRequest", + "ResizeVolumeRequest", "ResumeContainerRequest", "SetGuestDateTimeRequest", "SignalProcessRequest", @@ -34,6 +35,7 @@ allowed = [ "UpdateContainerRequest", "UpdateInterfaceRequest", "UpdateRoutesRequest", + "VolumeStatsRequest", "WaitProcessRequest", "WriteStreamRequest" ] diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index d7dbc08ef..fbead0953 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -193,13 +193,6 @@ async fn ephemeral_storage_handler( storage: &Storage, sandbox: Arc>, ) -> Result { - let mut sb = sandbox.lock().await; - let new_storage = sb.set_sandbox_storage(&storage.mount_point); - - if !new_storage { - return Ok("".to_string()); - } - // hugetlbfs if storage.fstype == FS_TYPE_HUGETLB { return handle_hugetlbfs_storage(logger, storage).await; @@ -255,13 +248,6 @@ async fn local_storage_handler( storage: &Storage, sandbox: Arc>, ) -> Result { - let mut sb = sandbox.lock().await; - let new_storage = sb.set_sandbox_storage(&storage.mount_point); - - if !new_storage { - return Ok("".to_string()); - } - fs::create_dir_all(&storage.mount_point).context(format!( "failed to create dir all {:?}", &storage.mount_point @@ -401,7 +387,7 @@ fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> { async fn virtiommio_blk_storage_handler( logger: &Logger, storage: &Storage, - _sandbox: Arc>, + sandbox: Arc>, ) -> Result { //The source path is VmPath common_storage_handler(logger, storage) @@ -641,6 +627,14 @@ pub async fn add_storages( "subsystem" => "storage", "storage-type" => handler_name.to_owned())); + { + let mut sb = sandbox.lock().await; + let new_storage = sb.set_sandbox_storage(&storage.mount_point); + if !new_storage { + continue; + } + } + let res = match handler_name.as_str() { DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await, DRIVER_BLK_CCW_TYPE => { diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index db5e04740..448f13502 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -23,9 +23,10 @@ use oci::{LinuxNamespace, Root, Spec}; use protobuf::{Message, RepeatedField, SingularPtrField}; use protocols::agent::{ AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics, - OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse, - WriteStreamResponse, + OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, VolumeStatsRequest, + WaitProcessResponse, WriteStreamResponse, }; +use protocols::csi::{VolumeCondition, VolumeStatsResponse, VolumeUsage, VolumeUsage_Unit}; use protocols::empty::Empty; use protocols::health::{ HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse, @@ -43,13 +44,15 @@ use nix::sys::stat; use nix::unistd::{self, Pid}; use rustjail::process::ProcessOperations; +use sysinfo::{DiskExt, System, SystemExt}; + use crate::device::{ add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci, }; use crate::image_rpc; use crate::linux_abi::*; use crate::metrics::get_metrics; -use crate::mount::{add_storages, baremount, remove_mounts, STORAGE_HANDLER_LIST}; +use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST}; use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS}; use crate::network::setup_guest_dns; use crate::pci; @@ -69,6 +72,7 @@ use tracing::instrument; use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ}; use std::convert::TryFrom; use std::fs; +use std::os::unix::fs::MetadataExt; use std::os::unix::prelude::PermissionsExt; use std::process::{Command, Stdio}; use std::time::Duration; @@ -302,8 +306,6 @@ impl AgentService { // Find the sandbox storage used by this container let mounts = sandbox.container_mounts.get(&cid); if let Some(mounts) = mounts { - remove_mounts(mounts)?; - for m in mounts.iter() { if sandbox.storages.get(m).is_some() { cmounts.push(m.to_string()); @@ -1321,6 +1323,47 @@ impl protocols::agent_ttrpc::AgentService for AgentService { Err(ttrpc_error!(ttrpc::Code::INTERNAL, "")) } + async fn get_volume_stats( + &self, + ctx: &TtrpcContext, + req: VolumeStatsRequest, + ) -> ttrpc::Result { + trace_rpc_call!(ctx, "get_volume_stats", req); + is_allowed!(req); + + info!(sl!(), "get volume stats!"); + let mut resp = VolumeStatsResponse::new(); + + let mut condition = VolumeCondition::new(); + + match File::open(&req.volume_guest_path) { + Ok(_) => { + condition.abnormal = false; + condition.message = String::from("OK"); + } + Err(e) => { + info!(sl!(), "failed to open the volume"); + return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e)); + } + }; + + let mut usage_vec = Vec::new(); + + // to get volume capacity stats + get_volume_capacity_stats(&req.volume_guest_path) + .map(|u| usage_vec.push(u)) + .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?; + + // to get volume inode stats + get_volume_inode_stats(&req.volume_guest_path) + .map(|u| usage_vec.push(u)) + .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?; + + resp.usage = RepeatedField::from_vec(usage_vec); + resp.volume_condition = SingularPtrField::some(condition); + Ok(resp) + } + async fn add_swap( &self, ctx: &TtrpcContext, @@ -1408,6 +1451,48 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { Ok((size, plug)) } +fn get_volume_capacity_stats(path: &str) -> Result { + let mut usage = VolumeUsage::new(); + + let s = System::new(); + for disk in s.disks() { + if let Some(v) = disk.name().to_str() { + if v.to_string().eq(path) { + usage.available = disk.available_space(); + usage.total = disk.total_space(); + usage.used = usage.total - usage.available; + usage.unit = VolumeUsage_Unit::BYTES; // bytes + break; + } + } else { + return Err(anyhow!(nix::Error::EINVAL)); + } + } + + Ok(usage) +} + +fn get_volume_inode_stats(path: &str) -> Result { + let mut usage = VolumeUsage::new(); + + let s = System::new(); + for disk in s.disks() { + if let Some(v) = disk.name().to_str() { + if v.to_string().eq(path) { + let meta = fs::metadata(disk.mount_point())?; + let inode = meta.ino(); + usage.used = inode; + usage.unit = VolumeUsage_Unit::INODES; + break; + } + } else { + return Err(anyhow!(nix::Error::EINVAL)); + } + } + + Ok(usage) +} + pub fn have_seccomp() -> bool { if cfg!(feature = "seccomp") { return true; diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 0dee97af7..0422a21a1 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -553,7 +553,7 @@ mod tests { assert!( s.remove_sandbox_storage(srcdir_path).is_err(), - "Expect Err as the directory i not a mountpoint" + "Expect Err as the directory is not a mountpoint" ); assert!(s.remove_sandbox_storage("").is_err()); @@ -588,7 +588,6 @@ mod tests { let logger = slog::Logger::root(slog::Discard, o!()); let mut s = Sandbox::new(&logger).unwrap(); - // FIX: This test fails, not sure why yet. assert!( s.unset_and_remove_sandbox_storage("/tmp/testEphePath") .is_err(), diff --git a/src/libs/protocols/build.rs b/src/libs/protocols/build.rs index d78f79f80..201a5a7c1 100644 --- a/src/libs/protocols/build.rs +++ b/src/libs/protocols/build.rs @@ -95,6 +95,7 @@ fn real_main() -> Result<(), std::io::Error> { let protos = vec![ "protos/agent.proto", + "protos/csi.proto", "protos/google/protobuf/empty.proto", "protos/health.proto", "protos/oci.proto", diff --git a/src/libs/protocols/hack/update-generated-proto.sh b/src/libs/protocols/hack/update-generated-proto.sh index 4befd0f09..64952b03c 100755 --- a/src/libs/protocols/hack/update-generated-proto.sh +++ b/src/libs/protocols/hack/update-generated-proto.sh @@ -51,6 +51,8 @@ generate_go_sources() { --gogottrpc_out=plugins=ttrpc+fieldpath,\ import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ \ +Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ +\ Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\ \ Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ @@ -69,7 +71,7 @@ if [ "$(basename $(pwd))" != "agent" ]; then fi # Protocol buffer files required to generate golang/rust bindings. -proto_files_list=(agent.proto health.proto oci.proto types.proto) +proto_files_list=(agent.proto csi.proto health.proto oci.proto types.proto) if [ "$1" = "" ]; then show_usage "${proto_files_list[@]}" diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index ede77ec0f..d0de9cf39 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -12,6 +12,7 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt package grpc; import "oci.proto"; +import "csi.proto"; import "types.proto"; import "google/protobuf/empty.proto"; @@ -65,6 +66,8 @@ service AgentService { rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty); rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent); rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty); + rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse); + rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty); } message CreateContainerRequest { @@ -505,3 +508,14 @@ message GetMetricsRequest {} message Metrics { string metrics = 1; } + +message VolumeStatsRequest { + // The volume path on the guest outside the container + string volume_guest_path = 1; +} + +message ResizeVolumeRequest { + // Full VM guest path of the volume (outside the container) + string volume_guest_path = 1; + uint64 size = 2; +} diff --git a/src/libs/protocols/protos/csi.proto b/src/libs/protocols/protos/csi.proto new file mode 100644 index 000000000..e6da50c8b --- /dev/null +++ b/src/libs/protocols/protos/csi.proto @@ -0,0 +1,60 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; +option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"; + +package grpc; +import "gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.populate_all) = true; +option (gogoproto.testgen_all) = true; +option (gogoproto.benchgen_all) = true; + +// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto) +message VolumeStatsResponse { + // This field is OPTIONAL. + repeated VolumeUsage usage = 1; + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the VOLUME_CONDITION node + // capability is supported. + VolumeCondition volume_condition = 2; +} +message VolumeUsage { + enum Unit { + UNKNOWN = 0; + BYTES = 1; + INODES = 2; + } + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + uint64 available = 1; + + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + uint64 total = 2; + + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + uint64 used = 3; + + // Units by which values are measured. This field is REQUIRED. + Unit unit = 4; +} + +// VolumeCondition represents the current condition of a volume. +message VolumeCondition { + + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + // This field is REQUIRED. + bool abnormal = 1; + + // The message describing the condition of the volume. + // This field is REQUIRED. + string message = 2; +} diff --git a/src/libs/protocols/protos/oci.proto b/src/libs/protocols/protos/oci.proto index 5de9b7d19..aa0db0123 100644 --- a/src/libs/protocols/protos/oci.proto +++ b/src/libs/protocols/protos/oci.proto @@ -6,11 +6,9 @@ // syntax = "proto3"; - option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"; package grpc; - import "gogo/protobuf/gogoproto/gogo.proto"; option (gogoproto.equal_all) = true; diff --git a/src/libs/protocols/src/lib.rs b/src/libs/protocols/src/lib.rs index 0fbb67123..f1b1b04c2 100644 --- a/src/libs/protocols/src/lib.rs +++ b/src/libs/protocols/src/lib.rs @@ -7,6 +7,7 @@ pub mod agent; pub mod agent_ttrpc; +pub mod csi; pub mod empty; pub mod health; pub mod health_ttrpc; diff --git a/src/runtime/Makefile b/src/runtime/Makefile index ea0f4eb35..9f34c3da7 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -1,5 +1,6 @@ # # Copyright (c) 2018-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -28,6 +29,9 @@ ARCH_FILE = $(ARCH_DIR)/$(ARCH)$(ARCH_FILE_SUFFIX) ARCH_FILES = $(wildcard arch/*$(ARCH_FILE_SUFFIX)) ALL_ARCHES = $(patsubst $(ARCH_DIR)/%$(ARCH_FILE_SUFFIX),%,$(ARCH_FILES)) +# Build as safely as possible +export CGO_CPPFLAGS = -D_FORTIFY_SOURCE=2 -fstack-protector + ifeq (,$(realpath $(ARCH_FILE))) $(error "ERROR: invalid architecture: '$(ARCH)'") else @@ -158,6 +162,7 @@ DEFMEMSLOTS := 10 DEFBRIDGES := 1 DEFENABLEANNOTATIONS := [] DEFDISABLEGUESTSECCOMP := true +DEFDISABLEGUESTEMPTYDIR := false #Default experimental features enabled DEFAULTEXPFEATURES := [] @@ -434,6 +439,7 @@ USER_VARS += DEFNETWORKMODEL_ACRN USER_VARS += DEFNETWORKMODEL_CLH USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFNETWORKMODEL_QEMU +USER_VARS += DEFDISABLEGUESTEMPTYDIR USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFDISABLESELINUX USER_VARS += DEFAULTEXPFEATURES diff --git a/src/runtime/cmd/kata-runtime/kata-exec.go b/src/runtime/cmd/kata-runtime/kata-exec.go index 456134a5f..0a7b80d70 100644 --- a/src/runtime/cmd/kata-runtime/kata-exec.go +++ b/src/runtime/cmd/kata-runtime/kata-exec.go @@ -19,8 +19,8 @@ import ( "time" "github.com/containerd/console" - kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" "github.com/pkg/errors" "github.com/urfave/cli" @@ -154,7 +154,7 @@ func (s *iostream) Read(data []byte) (n int, err error) { } func getConn(sandboxID string, port uint64) (net.Conn, error) { - client, err := kataMonitor.BuildShimClient(sandboxID, defaultTimeout) + client, err := shimclient.BuildShimClient(sandboxID, defaultTimeout) if err != nil { return nil, err } diff --git a/src/runtime/cmd/kata-runtime/kata-volume.go b/src/runtime/cmd/kata-runtime/kata-volume.go new file mode 100644 index 000000000..147709129 --- /dev/null +++ b/src/runtime/cmd/kata-runtime/kata-volume.go @@ -0,0 +1,145 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "encoding/json" + "net/url" + + containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" + "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" + + "github.com/urfave/cli" +) + +var volumeSubCmds = []cli.Command{ + addCommand, + removeCommand, + statsCommand, + resizeCommand, +} + +var ( + mountInfo string + volumePath string + size uint64 +) + +var kataVolumeCommand = cli.Command{ + Name: "direct-volume", + Usage: "directly assign a volume to Kata Containers to manage", + Subcommands: volumeSubCmds, + Action: func(context *cli.Context) { + cli.ShowSubcommandHelp(context) + }, +} + +var addCommand = cli.Command{ + Name: "add", + Usage: "add a direct assigned block volume device to the Kata Containers runtime", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + cli.StringFlag{ + Name: "mount-info", + Usage: "the mount info for the Kata Containers runtime to manage the volume", + Destination: &mountInfo, + }, + }, + Action: func(c *cli.Context) error { + return volume.Add(volumePath, mountInfo) + }, +} + +var removeCommand = cli.Command{ + Name: "remove", + Usage: "remove a direct assigned block volume device from the Kata Containers runtime", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + }, + Action: func(c *cli.Context) error { + return volume.Remove(volumePath) + }, +} + +var statsCommand = cli.Command{ + Name: "stats", + Usage: "get the filesystem stat of a direct assigned volume", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + }, + Action: func(c *cli.Context) (string, error) { + stats, err := Stats(volumePath) + if err != nil { + return "", err + } + + return string(stats), nil + }, +} + +var resizeCommand = cli.Command{ + Name: "resize", + Usage: "resize a direct assigned block volume", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + cli.Uint64Flag{ + Name: "size", + Usage: "the new size of the volume", + Destination: &size, + }, + }, + Action: func(c *cli.Context) error { + return Resize(volumePath, size) + }, +} + +// Stats retrieves the filesystem stats of the direct volume inside the guest. +func Stats(volumePath string) ([]byte, error) { + sandboxId, err := volume.GetSandboxIdForVolume(volumePath) + if err != nil { + return nil, err + } + urlSafeDevicePath := url.PathEscape(volumePath) + body, err := shimclient.DoGet(sandboxId, defaultTimeout, containerdshim.DirectVolumeStatUrl+"/"+urlSafeDevicePath) + if err != nil { + return nil, err + } + return body, nil +} + +// Resize resizes a direct volume inside the guest. +func Resize(volumePath string, size uint64) error { + sandboxId, err := volume.GetSandboxIdForVolume(volumePath) + if err != nil { + return err + } + resizeReq := containerdshim.ResizeRequest{ + VolumePath: volumePath, + Size: size, + } + encoded, err := json.Marshal(resizeReq) + if err != nil { + return err + } + return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, encoded) +} diff --git a/src/runtime/cmd/kata-runtime/main.go b/src/runtime/cmd/kata-runtime/main.go index 17309fe83..def7431f0 100644 --- a/src/runtime/cmd/kata-runtime/main.go +++ b/src/runtime/cmd/kata-runtime/main.go @@ -124,6 +124,7 @@ var runtimeCommands = []cli.Command{ kataExecCLICommand, kataMetricsCLICommand, factoryCLICommand, + kataVolumeCommand, } // runtimeBeforeSubcommands is the function to run before command-line diff --git a/src/runtime/config/configuration-acrn.toml.in b/src/runtime/config/configuration-acrn.toml.in index 2a9736e9e..f0be92ad0 100644 --- a/src/runtime/config/configuration-acrn.toml.in +++ b/src/runtime/config/configuration-acrn.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -219,6 +220,10 @@ disable_selinux=@DEFDISABLESELINUX@ # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 4afafaf08..fcf95b2e2 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -25,9 +26,12 @@ image = "@IMAGEPATH@" # Known limitations: # * Does not work by design: # - CPU Hotplug -# - Device Hotplug # - Memory Hotplug # - NVDIMM devices +# - SharedFS, such as virtio-fs and virtio-fs-nydus +# +# Requirements: +# * virtio-block used as rootfs, thus the usage of devmapper snapshotter. # # Supported TEEs: # * Intel TDX @@ -306,6 +310,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@ # vfio_mode="@DEFVFIOMODE@" +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/config/configuration-fc.toml.in b/src/runtime/config/configuration-fc.toml.in index d9eb093c2..b6892bd17 100644 --- a/src/runtime/config/configuration-fc.toml.in +++ b/src/runtime/config/configuration-fc.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -103,14 +104,6 @@ default_memory = @DEFMEMSZ@ # Default 0 #memory_offset = 0 -# Disable block device from being used for a container's rootfs. -# In case of a storage driver like devicemapper where a container's -# root file system is backed by a block device, the block device is passed -# directly to the hypervisor for performance reasons. -# This flag prevents the block device from being passed to the hypervisor, -# 9pfs is used instead to pass the rootfs. -disable_block_device_use = @DEFDISABLEBLOCK@ - # Block storage driver to be used for the hypervisor in case the container # rootfs is backed by a block device. This is virtio-scsi, virtio-blk # or nvdimm. @@ -352,6 +345,10 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # - When running single containers using a tool like ctr, container sizing information will be available. static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_FC@ +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index af5c58364..422056f7f 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -26,7 +27,6 @@ machine_type = "@MACHINETYPE@" # Known limitations: # * Does not work by design: # - CPU Hotplug -# - Device Hotplug # - Memory Hotplug # - NVDIMM devices # @@ -144,7 +144,7 @@ default_memory = @DEFMEMSZ@ # root file system is backed by a block device, the block device is passed # directly to the hypervisor for performance reasons. # This flag prevents the block device from being passed to the hypervisor, -# 9pfs is used instead to pass the rootfs. +# virtio-fs is used instead to pass the rootfs. disable_block_device_use = @DEFDISABLEBLOCK@ # Shared file system type: @@ -574,6 +574,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@ # vfio_mode="@DEFVFIOMODE@" +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/pkg/containerd-shim-v2/create.go b/src/runtime/pkg/containerd-shim-v2/create.go index bf6970b5d..1edf00591 100644 --- a/src/runtime/pkg/containerd-shim-v2/create.go +++ b/src/runtime/pkg/containerd-shim-v2/create.go @@ -1,6 +1,7 @@ // Copyright (c) 2014,2015,2016 Docker, Inc. // Copyright (c) 2017 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -57,10 +58,10 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con detach := !r.Terminal ociSpec, bundlePath, err := loadSpec(r) + if err != nil { return nil, err } - containerType, err := oci.ContainerType(*ociSpec) if err != nil { return nil, err @@ -69,16 +70,18 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) rootfs := filepath.Join(r.Bundle, "rootfs") + runtimeConfig, err := loadRuntimeConfig(s, r, ociSpec.Annotations) + if err != nil { + return nil, err + } + switch containerType { case vc.PodSandbox, vc.SingleContainer: if s.sandbox != nil { return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID()) } - s.config, err = loadRuntimeConfig(s, r, ociSpec.Annotations) - if err != nil { - return nil, err - } + s.config = runtimeConfig // create tracer // This is the earliest location we can create the tracer because we must wait @@ -176,7 +179,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con } }() - _, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput) + _, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, runtimeConfig.DisableGuestEmptyDir) if err != nil { return nil, err } diff --git a/src/runtime/pkg/containerd-shim-v2/shim_management.go b/src/runtime/pkg/containerd-shim-v2/shim_management.go index 5ccf03304..b5ad03eed 100644 --- a/src/runtime/pkg/containerd-shim-v2/shim_management.go +++ b/src/runtime/pkg/containerd-shim-v2/shim_management.go @@ -7,26 +7,33 @@ package containerdshim import ( "context" + "encoding/json" "expvar" "fmt" "io" + "io/ioutil" "net/http" "net/http/pprof" + "net/url" "path/filepath" "strconv" "strings" + "google.golang.org/grpc/codes" + cdshim "github.com/containerd/containerd/runtime/v2/shim" + mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" "github.com/opencontainers/runtime-spec/specs-go" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" +) - "google.golang.org/grpc/codes" - - mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" +const ( + DirectVolumeStatUrl = "/direct-volume/stats" + DirectVolumeResizeUrl = "/direct-volume/resize" ) var ( @@ -34,6 +41,11 @@ var ( shimMgtLog = shimLog.WithField("subsystem", "shim-management") ) +type ResizeRequest struct { + VolumePath string + Size uint64 +} + // agentURL returns URL for agent func (s *service) agentURL(w http.ResponseWriter, r *http.Request) { url, err := s.sandbox.GetAgentURL() @@ -126,6 +138,52 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily { return list } +func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) { + volumePath, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, DirectVolumeStatUrl)) + if err != nil { + shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + buf, err := s.sandbox.GuestVolumeStats(context.Background(), volumePath) + if err != nil { + shimMgtLog.WithError(err).WithField("volume-path", volumePath).Error("failed to get volume stats") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Write(buf) +} + +func (s *service) serveVolumeResize(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + shimMgtLog.WithError(err).Error("failed to read request body") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + var resizeReq ResizeRequest + err = json.Unmarshal(body, &resizeReq) + if err != nil { + shimMgtLog.WithError(err).Error("failed to unmarshal the http request body") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + err = s.sandbox.ResizeGuestVolume(context.Background(), resizeReq.VolumePath, resizeReq.Size) + if err != nil { + shimMgtLog.WithError(err).WithField("volume-path", resizeReq.VolumePath).Error("failed to resize the volume") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Write([]byte("")) +} + func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) { // metrics socket will under sandbox's bundle path metricsAddress := SocketAddress(s.id) @@ -148,6 +206,8 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec m := http.NewServeMux() m.Handle("/metrics", http.HandlerFunc(s.serveMetrics)) m.Handle("/agent-url", http.HandlerFunc(s.agentURL)) + m.Handle(DirectVolumeStatUrl, http.HandlerFunc(s.serveVolumeStats)) + m.Handle(DirectVolumeResizeUrl, http.HandlerFunc(s.serveVolumeResize)) s.mountPprofHandle(m, ociSpec) // register shim metrics diff --git a/src/runtime/pkg/direct-volume/utils.go b/src/runtime/pkg/direct-volume/utils.go new file mode 100644 index 000000000..275b0508f --- /dev/null +++ b/src/runtime/pkg/direct-volume/utils.go @@ -0,0 +1,108 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package volume + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + mountInfoFileName = "mountInfo.json" +) + +var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes" + +// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM. +type MountInfo struct { + // The type of the volume (ie. block) + VolumeType string `json:"volume-type"` + // The device backing the volume. + Device string `json:"device"` + // The filesystem type to be mounted on the volume. + FsType string `json:"fstype"` + // Additional metadata to pass to the agent regarding this volume. + Metadata map[string]string `json:"metadata,omitempty"` + // Additional mount options. + Options []string `json:"options,omitempty"` +} + +// Add writes the mount info of a direct volume into a filesystem path known to Kata Container. +func Add(volumePath string, mountInfo string) error { + volumeDir := filepath.Join(kataDirectVolumeRootPath, volumePath) + stat, err := os.Stat(volumeDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if stat != nil && !stat.IsDir() { + return fmt.Errorf("%s should be a directory", volumeDir) + } + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(volumeDir, 0700); err != nil { + return err + } + } + var deserialized MountInfo + if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600) +} + +// Remove deletes the direct volume path including all the files inside it. +func Remove(volumePath string) error { + // Find the base of the volume path to delete the whole volume path + base := strings.SplitN(volumePath, string(os.PathSeparator), 2)[0] + return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, base)) +} + +// VolumeMountInfo retrieves the mount info of a direct volume. +func VolumeMountInfo(volumePath string) (*MountInfo, error) { + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName) + if _, err := os.Stat(mountInfoFilePath); err != nil { + return nil, err + } + buf, err := ioutil.ReadFile(mountInfoFilePath) + if err != nil { + return nil, err + } + var mountInfo MountInfo + if err := json.Unmarshal(buf, &mountInfo); err != nil { + return nil, err + } + return &mountInfo, nil +} + +// RecordSandboxId associates a sandbox id with a direct volume. +func RecordSandboxId(sandboxId string, volumePath string) error { + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName) + if _, err := os.Stat(mountInfoFilePath); err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, volumePath, sandboxId), []byte(""), 0600) +} + +func GetSandboxIdForVolume(volumePath string) (string, error) { + files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, volumePath)) + if err != nil { + return "", err + } + // Find the id of the first sandbox. + // We expect a direct-assigned volume is associated with only a sandbox at a time. + for _, file := range files { + if file.Name() != mountInfoFileName { + return file.Name(), nil + } + } + return "", fmt.Errorf("no sandbox found for %s", volumePath) +} diff --git a/src/runtime/pkg/direct-volume/utils_test.go b/src/runtime/pkg/direct-volume/utils_test.go new file mode 100644 index 000000000..6ca80dab8 --- /dev/null +++ b/src/runtime/pkg/direct-volume/utils_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package volume + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" + "github.com/stretchr/testify/assert" +) + +func TestAdd(t *testing.T) { + var err error + kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "add-test") + assert.Nil(t, err) + defer os.RemoveAll(kataDirectVolumeRootPath) + var volumePath = "/a/b/c" + var basePath = "a" + actual := MountInfo{ + VolumeType: "block", + Device: "/dev/sda", + FsType: "ext4", + Options: []string{"journal_dev", "noload"}, + } + buf, err := json.Marshal(actual) + assert.Nil(t, err) + + // Add the mount info + assert.Nil(t, Add(volumePath, string(buf))) + + // Validate the mount info + expected, err := VolumeMountInfo(volumePath) + assert.Nil(t, err) + assert.Equal(t, expected.Device, actual.Device) + assert.Equal(t, expected.FsType, actual.FsType) + assert.Equal(t, expected.Options, actual.Options) + + // Remove the file + err = Remove(volumePath) + assert.Nil(t, err) + _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, basePath)) + assert.True(t, errors.Is(err, os.ErrNotExist)) + + // Test invalid mount info json + assert.Error(t, Add(volumePath, "{invalid json}")) +} + +func TestRecordSandboxId(t *testing.T) { + var err error + kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test") + assert.Nil(t, err) + defer os.RemoveAll(kataDirectVolumeRootPath) + + var volumePath = "/a/b/c" + mntInfo := MountInfo{ + VolumeType: "block", + Device: "/dev/sda", + FsType: "ext4", + Options: []string{"journal_dev", "noload"}, + } + buf, err := json.Marshal(mntInfo) + assert.Nil(t, err) + + // Add the mount info + assert.Nil(t, Add(volumePath, string(buf))) + + sandboxId := uuid.Generate().String() + err = RecordSandboxId(sandboxId, volumePath) + assert.Nil(t, err) + + id, err := GetSandboxIdForVolume(volumePath) + assert.Nil(t, err) + assert.Equal(t, sandboxId, id) +} + +func TestRecordSandboxIdNoMountInfoFile(t *testing.T) { + var err error + kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test") + assert.Nil(t, err) + defer os.RemoveAll(kataDirectVolumeRootPath) + + var volumePath = "/a/b/c" + sandboxId := uuid.Generate().String() + err = RecordSandboxId(sandboxId, volumePath) + assert.Error(t, err) + assert.True(t, errors.Is(err, os.ErrNotExist)) +} diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index 84098c88f..7249906ce 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -16,7 +16,7 @@ import ( "time" mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" - + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/expfmt" @@ -224,7 +224,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { } func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*dto.MetricFamily, error) { - body, err := doGet(sandboxID, defaultTimeout, "metrics") + body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics") if err != nil { return nil, err } @@ -234,7 +234,7 @@ func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]* // GetSandboxMetrics will get sandbox's metrics from shim func GetSandboxMetrics(sandboxID string) (string, error) { - body, err := doGet(sandboxID, defaultTimeout, "metrics") + body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics") if err != nil { return "", err } diff --git a/src/runtime/pkg/kata-monitor/monitor.go b/src/runtime/pkg/kata-monitor/monitor.go index 9004cf103..ed3ea5c08 100644 --- a/src/runtime/pkg/kata-monitor/monitor.go +++ b/src/runtime/pkg/kata-monitor/monitor.go @@ -14,6 +14,8 @@ import ( "sync" "time" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" + "github.com/fsnotify/fsnotify" "github.com/sirupsen/logrus" ) @@ -180,7 +182,7 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) { return } - data, err := doGet(sandboxID, defaultTimeout, "agent-url") + data, err := shimclient.DoGet(sandboxID, defaultTimeout, "agent-url") if err != nil { commonServeError(w, http.StatusBadRequest, err) return diff --git a/src/runtime/pkg/kata-monitor/shim_client.go b/src/runtime/pkg/kata-monitor/shim_client.go index bdb62d401..388ac6fff 100644 --- a/src/runtime/pkg/kata-monitor/shim_client.go +++ b/src/runtime/pkg/kata-monitor/shim_client.go @@ -7,13 +7,9 @@ package katamonitor import ( "fmt" - "io" - "net" "net/http" "time" - cdshim "github.com/containerd/containerd/runtime/v2/shim" - shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" ) @@ -40,51 +36,3 @@ func getSandboxIDFromReq(r *http.Request) (string, error) { func getSandboxFS() string { return shim.GetSandboxesStoragePath() } - -// BuildShimClient builds and returns an http client for communicating with the provided sandbox -func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) { - return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout) -} - -// buildUnixSocketClient build http client for Unix socket -func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) { - transport := &http.Transport{ - DisableKeepAlives: true, - Dial: func(proto, addr string) (conn net.Conn, err error) { - return cdshim.AnonDialer(socketAddr, timeout) - }, - } - - client := &http.Client{ - Transport: transport, - } - - if timeout > 0 { - client.Timeout = timeout - } - - return client, nil -} - -func doGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) { - client, err := BuildShimClient(sandboxID, timeoutInSeconds) - if err != nil { - return nil, err - } - - resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath)) - if err != nil { - return nil, err - } - - defer func() { - resp.Body.Close() - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - return body, nil -} diff --git a/src/runtime/pkg/katatestutils/README.md b/src/runtime/pkg/katatestutils/README.md index 4e8221cb8..3a84afd49 100644 --- a/src/runtime/pkg/katatestutils/README.md +++ b/src/runtime/pkg/katatestutils/README.md @@ -102,36 +102,6 @@ func TestOnlyRunWhenNotRoot(t *testing.T) { } ``` -#### Skip tests based on distro - -Use the `NeedDistro()` constraint to skip a test unless running on a -particular Linux distribution: - -```go -func TestOnlyRunOnUbuntu(t *testing.T) { - - if tc.NotValid(ktu.NeedDistro("ubuntu")) { - t.Skip("skipping test as not running on ubuntu") - } - - // Test code to run on Ubuntu only ... -} -``` - -Use the `NeedDistroNotEquals()` constraint to skip a test unless running -on a Linux distribution other than the one specified: - -```go -func TestDontRunOnFedora(t *testing.T) { - - if tc.NotValid(ktu.NeedDistroNotEquals("fedora")) { - t.Skip("skipping test as running on fedora") - } - - // Test code to run on any distro apart from Fedora ... -} -``` - #### Skip tests based on kernel version Use the `NeedKernelVersionGE()` constraint to skip a test unless running on a diff --git a/src/runtime/pkg/katatestutils/constraints.go b/src/runtime/pkg/katatestutils/constraints.go index fd73d0781..48b3e5d3b 100644 --- a/src/runtime/pkg/katatestutils/constraints.go +++ b/src/runtime/pkg/katatestutils/constraints.go @@ -18,17 +18,9 @@ import ( const ( TestDisabledNeedRoot = "Test disabled as requires root user" TestDisabledNeedNonRoot = "Test disabled as requires non-root user" - - // See https://www.freedesktop.org/software/systemd/man/os-release.html - osRelease = "/etc/os-release" - osReleaseAlternative = "/usr/lib/os-release" ) -var ( - errUnknownDistroName = errors.New("unknown distro name") - errUnknownDistroVersion = errors.New("unknown distro version") - errInvalidOpForConstraint = errors.New("invalid operator for constraint type") -) +var errInvalidOpForConstraint = errors.New("invalid operator for constraint type") // String converts the operator to a human-readable value. func (o Operator) String() (s string) { @@ -87,54 +79,6 @@ func getKernelVersion() (string, error) { return fixKernelVersion(fields[2]), nil } -// getDistroDetails returns the distributions name and version string. -// If it is not possible to determine both values an error is -// returned. -func getDistroDetails() (name, version string, err error) { - files := []string{osRelease, osReleaseAlternative} - name = "" - version = "" - - for _, file := range files { - contents, err := getFileContents(file) - if err != nil { - if os.IsNotExist(err) { - continue - } - - return "", "", err - } - - lines := strings.Split(contents, "\n") - - for _, line := range lines { - if strings.HasPrefix(line, "ID=") && name == "" { - fields := strings.Split(line, "=") - name = strings.Trim(fields[1], `"`) - name = strings.ToLower(name) - } else if strings.HasPrefix(line, "VERSION_ID=") && version == "" { - fields := strings.Split(line, "=") - version = strings.Trim(fields[1], `"`) - version = strings.ToLower(version) - } - } - - if name != "" && version != "" { - return name, version, nil - } - } - - if name == "" { - return "", "", errUnknownDistroName - } - - if version == "" { - return "", "", errUnknownDistroVersion - } - - return name, version, nil -} - // fixKernelVersion replaces underscores with dashes in a version string. // This change is primarily for Fedora, RHEL and CentOS version numbers which // can contain underscores. By replacing them with dashes, a valid semantic @@ -157,42 +101,6 @@ func fixKernelVersion(version string) string { return strings.Replace(version, "+", "", -1) } -// handleDistroName checks that the current distro is compatible with -// the constraint specified by the arguments. -func (tc *TestConstraint) handleDistroName(name string, op Operator) (result Result, err error) { - if name == "" { - return Result{}, fmt.Errorf("distro name cannot be blank") - } - - name = strings.ToLower(name) - - var success bool - - switch op { - case eqOperator: - success = name == tc.DistroName - case neOperator: - success = name != tc.DistroName - default: - return Result{}, errInvalidOpForConstraint - } - - descr := fmt.Sprintf("need distro %s %q, got distro %q", op, name, tc.DistroName) - - result = Result{ - Description: descr, - Success: success, - } - - return result, nil -} - -// handleDistroVersion checks that the current distro version is compatible with -// the constraint specified by the arguments. -func (tc *TestConstraint) handleDistroVersion(version string, op Operator) (result Result, err error) { - return handleVersionType("distro", tc.DistroVersion, op, version) -} - // handleKernelVersion checks that the current kernel version is compatible with // the constraint specified by the arguments. func (tc *TestConstraint) handleKernelVersion(version string, op Operator) (result Result, err error) { @@ -459,23 +367,6 @@ func (tc *TestConstraint) constraintValid(fn Constraint) bool { } } - if c.DistroName != "" { - result, err := tc.handleDistroName(c.DistroName, c.Operator) - tc.handleResults(result, err) - if !result.Success { - return false - } - } - - if c.DistroVersion != "" { - result, err := tc.handleDistroVersion(c.DistroVersion, c.Operator) - tc.handleResults(result, err) - if !result.Success { - return false - } - - } - if c.KernelVersion != "" { result, err := tc.handleKernelVersion(c.KernelVersion, c.Operator) tc.handleResults(result, err) diff --git a/src/runtime/pkg/katatestutils/constraints_api.go b/src/runtime/pkg/katatestutils/constraints_api.go index 1bae25541..ce27ee424 100644 --- a/src/runtime/pkg/katatestutils/constraints_api.go +++ b/src/runtime/pkg/katatestutils/constraints_api.go @@ -9,7 +9,6 @@ package katatestutils import ( "os" - "strings" ) // Operator represents an operator to apply to a test constraint value. @@ -28,13 +27,6 @@ const ( type Constraints struct { Issue string - // DistroName is the name of a distro in all lower-case letters. - DistroName string - - // DistroVersion is the version of the particular distro in string - // format. It may contain periods and dashes. - DistroVersion string - // KernelVersion is the version of a particular kernel. KernelVersion string @@ -54,8 +46,6 @@ type Constraint func(c *Constraints) // TestConstraint records details about test constraints. type TestConstraint struct { - DistroName string - DistroVersion string KernelVersion string // Optionally used to record an issue number that relates to the @@ -76,11 +66,6 @@ type TestConstraint struct { // NewKataTest creates a new TestConstraint object and is the main interface // to the test constraints feature. func NewTestConstraint(debug bool) TestConstraint { - distroName, distroVersion, err := getDistroDetails() - if err != nil { - panic(err) - } - kernelVersion, err := getKernelVersion() if err != nil { panic(err) @@ -90,8 +75,6 @@ func NewTestConstraint(debug bool) TestConstraint { Debug: debug, ActualEUID: os.Geteuid(), - DistroName: distroName, - DistroVersion: distroVersion, KernelVersion: kernelVersion, } } @@ -102,7 +85,7 @@ func NewTestConstraint(debug bool) TestConstraint { // Notes: // // - Constraints are applied in the order specified. -// - A constraint type (user, distro) can only be specified once. +// - A constraint type (user, kernel) can only be specified once. // - If the function fails to determine whether it can check the constraints, // it will panic. Since this is facility is used for testing, this seems like // the best approach as it unburdens the caller from checking for an error @@ -146,99 +129,7 @@ func NeedNonRoot() Constraint { return NeedUID(0, neOperator) } -// NeedDistroWithOp skips the test unless the distro constraint specified by -// the arguments is true. -func NeedDistroWithOp(distro string, op Operator) Constraint { - return func(c *Constraints) { - c.DistroName = strings.ToLower(distro) - c.Operator = op - } -} - -// NeedDistroEquals will skip the test unless running on the specified distro. -func NeedDistroEquals(distro string) Constraint { - return NeedDistroWithOp(distro, eqOperator) -} - -// NeedDistroNotEquals will skip the test unless run a distro that does not -// match the specified name. -func NeedDistroNotEquals(distro string) Constraint { - return NeedDistroWithOp(distro, neOperator) -} - -// NeedDistro will skip the test unless running on the specified distro. -func NeedDistro(distro string) Constraint { - return NeedDistroEquals(distro) -} - -// NeedDistroVersionWithOp skips the test unless the distro version constraint -// specified by the arguments is true. -// -// Note: distro versions vary in format. -func NeedDistroVersionWithOp(version string, op Operator) Constraint { - return func(c *Constraints) { - c.DistroVersion = version - c.Operator = op - } -} - -// NeedDistroVersionEquals will skip the test unless the distro version is the -// same as the specified version. -// -// Note: distro versions vary in format. -func NeedDistroVersionEquals(version string) Constraint { - return NeedDistroVersionWithOp(version, eqOperator) -} - -// NeedDistroVersionNotEquals will skip the test unless the distro version is -// different to the specified version. -// -// Note: distro versions vary in format. -func NeedDistroVersionNotEquals(version string) Constraint { - return NeedDistroVersionWithOp(version, neOperator) -} - -// NeedDistroVersionLE will skip the test unless the distro version is older -// than or the same as the specified version. -// -// Note: distro versions vary in format. -func NeedDistroVersionLE(version string) Constraint { - return NeedDistroVersionWithOp(version, leOperator) -} - -// NeedDistroVersionLT will skip the test unless the distro version is older -// than the specified version. -// -// Note: distro versions vary in format. -func NeedDistroVersionLT(version string) Constraint { - return NeedDistroVersionWithOp(version, ltOperator) -} - -// NeedDistroVersionGE will skip the test unless the distro version is newer -// than or the same as the specified version. -// -// Note: distro versions vary in format. -func NeedDistroVersionGE(version string) Constraint { - return NeedDistroVersionWithOp(version, geOperator) -} - -// NeedDistroVersionGT will skip the test unless the distro version is newer -// than the specified version. -// -// Note: distro versions vary in format. -func NeedDistroVersionGT(version string) Constraint { - return NeedDistroVersionWithOp(version, gtOperator) -} - -// NeedDistroVersion will skip the test unless running on the specified -// (exact) version of some distro. -// -// Note: distro versions vary in format. -func NeedDistroVersion(version string) Constraint { - return NeedDistroVersionEquals(version) -} - -// NeedKernelVersionWithOp skips the test unless the distro version constraint +// NeedKernelVersionWithOp skips the test unless the kernel version constraint // specified by the arguments is true. func NeedKernelVersionWithOp(version string, op Operator) Constraint { return func(c *Constraints) { @@ -247,54 +138,44 @@ func NeedKernelVersionWithOp(version string, op Operator) Constraint { } } -// NeedKernelVersionLT will skip the test unless the distro version is older -// than the specified version. +// NeedKernelVersionEquals will skip the test unless the kernel version is same as +// the specified version. func NeedKernelVersionEquals(version string) Constraint { return NeedKernelVersionWithOp(version, eqOperator) } -// NeedKernelVersionNotEquals will skip the test unless the distro version is +// NeedKernelVersionNotEquals will skip the test unless the kernel version is // different to the specified version. func NeedKernelVersionNotEquals(version string) Constraint { return NeedKernelVersionWithOp(version, neOperator) } -// NeedKernelVersionLT will skip the test unless the distro version is older +// NeedKernelVersionLT will skip the test unless the kernel version is older // than the specified version. -// -// Note: distro versions vary in format. func NeedKernelVersionLT(version string) Constraint { return NeedKernelVersionWithOp(version, ltOperator) } -// NeedKernelVersionLE will skip the test unless the distro version is older +// NeedKernelVersionLE will skip the test unless the kernel version is older // than or the same as the specified version. -// -// Note: distro versions vary in format. func NeedKernelVersionLE(version string) Constraint { return NeedKernelVersionWithOp(version, leOperator) } -// NeedKernelVersionGT will skip the test unless the distro version is newer +// NeedKernelVersionGT will skip the test unless the kernel version is newer // than the specified version. -// -// Note: distro versions vary in format. func NeedKernelVersionGT(version string) Constraint { return NeedKernelVersionWithOp(version, gtOperator) } -// NeedKernelVersionGE will skip the test unless the distro version is newer +// NeedKernelVersionGE will skip the test unless the kernel version is newer // than or the same as the specified version. -// -// Note: distro versions vary in format. func NeedKernelVersionGE(version string) Constraint { return NeedKernelVersionWithOp(version, geOperator) } -// NeedKernelVersion will skip the test unless running on the specified -// (exact) version of some distro. -// -// Note: distro versions vary in format. +// NeedKernelVersion will skip the test unless the kernel version is same as +// the specified version. func NeedKernelVersion(version string) Constraint { return NeedKernelVersionEquals(version) } diff --git a/src/runtime/pkg/katatestutils/constraints_test.go b/src/runtime/pkg/katatestutils/constraints_test.go index 314d017c1..ec7d43ecd 100644 --- a/src/runtime/pkg/katatestutils/constraints_test.go +++ b/src/runtime/pkg/katatestutils/constraints_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "os" - "os/exec" "path/filepath" "strconv" "strings" @@ -21,8 +20,6 @@ import ( const ( invalidOperator = 1234 - - skipUnknownDistroName = "skipping test as cannot determine distro name" ) // nolint: govet @@ -32,36 +29,16 @@ type testDataUID struct { c Constraints } -// nolint: govet -type testDataDistro struct { - distro string - op Operator - c Constraints -} +var ( + thisUID = os.Getuid() + rootUID = 0 +) -var distros = []string{ - "centos", - "clear-linux-os", - "debian", - "fedora", - "opensuse", - "rhel", - "sles", - "ubuntu", -} - -var thisUID = os.Getuid() -var rootUID = 0 - -// name and version of current distro and kernel version of system tests are +// name and version of current kernel version of system tests are // running on -var distroName string -var distroVersion string var kernelVersion string -// error saved when attempting to determine distro name+version and kernel -// version. -var getDistroErr error +// error saved when attempting to determine kernel version. var getKernelErr error // true if running as root @@ -87,49 +64,8 @@ var uidNotEqualsRootData = testDataUID{ }, } -var distroEqualsCurrentData testDataDistro -var distroNotEqualsCurrentData testDataDistro - func init() { - distroName, distroVersion, getDistroErr = testGetDistro() kernelVersion, getKernelErr = testGetKernelVersion() - - distroEqualsCurrentData = testDataDistro{ - distro: distroName, - op: eqOperator, - c: Constraints{ - DistroName: distroName, - Operator: eqOperator, - }, - } - - distroNotEqualsCurrentData = testDataDistro{ - distro: distroName, - op: neOperator, - c: Constraints{ - DistroName: distroName, - Operator: neOperator, - }, - } -} - -func fileExists(path string) bool { - if _, err := os.Stat(path); os.IsNotExist(err) { - return false - } - - return true -} - -// getAnotherDistro returns a distro name not equal to the one specified. -func getAnotherDistro(distro string) string { - for _, d := range distros { - if d != distro { - return d - } - } - - panic(fmt.Sprintf("failed to find a distro different to %s", distro)) } func checkUIDConstraints(assert *assert.Assertions, a, b Constraints, desc string) { @@ -140,13 +76,6 @@ func checkUIDConstraints(assert *assert.Assertions, a, b Constraints, desc strin assert.Equal(a.UIDSet, b.UIDSet, msg) } -func checkDistroConstraints(assert *assert.Assertions, a, b Constraints, desc string) { - msg := fmt.Sprintf("%s: a: %+v, b: %+v", desc, a, b) - - assert.Equal(a.DistroName, b.DistroName, msg) - assert.Equal(a.Operator, b.Operator, msg) -} - func checkKernelConstraint(assert *assert.Assertions, f Constraint, version string, op Operator, msg string) { c := Constraints{} @@ -156,19 +85,6 @@ func checkKernelConstraint(assert *assert.Assertions, f Constraint, version stri assert.Equal(c.Operator, op, msg) } -// runCommand runs a command and returns its output -func runCommand(args ...string) ([]string, error) { - cmd := exec.Command(args[0], args[1:]...) - bytes, err := cmd.Output() - if err != nil { - return []string{}, err - } - - output := strings.Split(string(bytes), "\n") - - return output, nil -} - // semverBumpVersion takes an existing semantic version and increments one or // more parts of it, returning the new version number as a string. func semverBumpVersion(ver semver.Version, bumpMajor, bumpMinor, bumpPatch bool) (string, error) { @@ -263,56 +179,6 @@ func decrementVersion(version string) (string, error) { return changeVersion(version, true) } -// testGetDistro is an alternative implementation of getDistroDetails() used -// for testing. -func testGetDistro() (name, version string, err error) { - files := []string{"/etc/os-release", "/usr/lib/os-release"} - - for _, file := range files { - if !fileExists(file) { - continue - } - - output, err := runCommand("grep", "^ID=", file) - if err != nil { - return "", "", err - } - - line := output[0] - fields := strings.Split(line, "=") - if name == "" { - name = strings.Trim(fields[1], `"`) - name = strings.ToLower(name) - } - - output, err = runCommand("grep", "^VERSION_ID=", file) - if err != nil { - return "", "", err - } - - line = output[0] - fields = strings.Split(line, "=") - if version == "" { - version = strings.Trim(fields[1], `"`) - version = strings.ToLower(version) - } - } - - if name != "" && version != "" { - return name, version, nil - } - - if name == "" { - return "", "", errUnknownDistroName - } - - if version == "" { - return "", "", errUnknownDistroVersion - } - - return "", "", errors.New("BUG: something bad happened") -} - func testGetKernelVersion() (version string, err error) { const file = "/proc/version" @@ -360,11 +226,6 @@ func TestOperatorString(t *testing.T) { } func TestNewTestConstraint(t *testing.T) { - if getDistroErr != nil { - t.Skipf("skipping as unable to determine distro name/version: %v", - getDistroErr) - } - if getKernelErr != nil { t.Skipf("skipping as unable to determine kernel version: %v", getKernelErr) @@ -379,17 +240,11 @@ func TestNewTestConstraint(t *testing.T) { assert.Equal(debug, c.Debug, msg) - assert.Equal(distroName, c.DistroName, msg) - assert.Equal(distroVersion, c.DistroVersion, msg) assert.Equal(kernelVersion, c.KernelVersion, msg) assert.Equal(thisUID, c.ActualEUID) toCheck := []string{ - distroName, - distroVersion, kernelVersion, - c.DistroName, - c.DistroVersion, c.KernelVersion, } @@ -438,26 +293,6 @@ func TestGetFileContents(t *testing.T) { } } -func TestGetDistroDetails(t *testing.T) { - assert := assert.New(t) - - if getDistroErr == errUnknownDistroName { - t.Skip(skipUnknownDistroName) - } - - assert.NoError(getDistroErr) - assert.NotNil(distroName) - assert.NotNil(distroVersion) - - name, version, err := getDistroDetails() - assert.NoError(err) - assert.NotNil(name) - assert.NotNil(version) - - assert.Equal(name, distroName) - assert.Equal(version, distroVersion) -} - func TestGetKernelVersion(t *testing.T) { assert := assert.New(t) @@ -471,158 +306,6 @@ func TestGetKernelVersion(t *testing.T) { assert.Equal(version, kernelVersion) } -func TestConstraintHandleDistroName(t *testing.T) { - assert := assert.New(t) - - // nolint: govet - type testData struct { - distro string - op Operator - result Result - expectError bool - } - - distroName, _, err := testGetDistro() - if err != nil && err == errUnknownDistroName { - t.Skip(skipUnknownDistroName) - } - - // Look for the first distro that is not the same as the distro this - // test is currently running on. - differentDistro := getAnotherDistro(distroName) - - data := []testData{ - {"", eqOperator, Result{}, true}, - {"", neOperator, Result{}, true}, - {"", invalidOperator, Result{}, true}, - {distroName, invalidOperator, Result{}, true}, - {distroName, invalidOperator, Result{}, true}, - - { - distroName, - eqOperator, - Result{ - Description: distroName, - Success: true, - }, - false, - }, - { - distroName, - neOperator, - Result{ - Description: distroName, - Success: false, - }, - false, - }, - { - differentDistro, - eqOperator, - Result{ - Description: differentDistro, - Success: false, - }, - false, - }, - - { - differentDistro, - neOperator, - Result{ - Description: differentDistro, - Success: true, - }, - false, - }, - } - - for _, debug := range []bool{true, false} { - tc := NewTestConstraint(debug) - - for i, d := range data { - result, err := tc.handleDistroName(d.distro, d.op) - - msg := fmt.Sprintf("test[%d]: %+v, result: %+v", i, d, result) - - if d.expectError { - assert.Error(err, msg) - continue - - } - - assert.NoError(err, msg) - assert.Equal(result.Success, d.result.Success, msg) - assert.NotNil(result.Description, msg) - } - } -} - -func TestConstraintHandleDistroVersion(t *testing.T) { - assert := assert.New(t) - - assert.NotNil(distroVersion) - - // Generate a new distro version for testing purposes. Since we don't - // know the format of this particular distros versioning scheme, we - // need to calculate it. - higherVersion, err := incrementVersion(distroVersion) - assert.NoError(err) - assert.NotEqual(distroVersion, higherVersion) - - // nolint: govet - type testData struct { - version string - op Operator - result Result - expectError bool - } - - data := []testData{ - {"", eqOperator, Result{}, true}, - {"", geOperator, Result{}, true}, - {"", gtOperator, Result{}, true}, - {"", leOperator, Result{}, true}, - {"", ltOperator, Result{}, true}, - {"", neOperator, Result{}, true}, - - {distroVersion, eqOperator, Result{Success: true}, false}, - {higherVersion, eqOperator, Result{Success: false}, false}, - - {distroVersion, gtOperator, Result{Success: false}, false}, - {higherVersion, gtOperator, Result{Success: false}, false}, - - {distroVersion, geOperator, Result{Success: true}, false}, - {higherVersion, geOperator, Result{Success: false}, false}, - - {distroVersion, ltOperator, Result{Success: false}, false}, - {higherVersion, ltOperator, Result{Success: true}, false}, - - {distroVersion, leOperator, Result{Success: true}, false}, - {higherVersion, leOperator, Result{Success: true}, false}, - - {distroVersion, neOperator, Result{Success: false}, false}, - {higherVersion, neOperator, Result{Success: true}, false}, - } - - for _, debug := range []bool{true, false} { - tc := NewTestConstraint(debug) - - for i, d := range data { - result, err := tc.handleDistroVersion(d.version, d.op) - - msg := fmt.Sprintf("test[%d]: %+v, result: %+v", i, d, result) - - if d.expectError { - assert.Error(err, msg) - continue - } - - assert.Equal(d.result.Success, result.Success, msg) - } - } -} - func TestConstraintHandleVersionType(t *testing.T) { assert := assert.New(t) @@ -638,7 +321,6 @@ func TestConstraintHandleVersionType(t *testing.T) { data := []testData{ //---------- - {"", "", eqOperator, "", Result{}, true}, {"name", "foo", eqOperator, "", Result{}, true}, @@ -960,10 +642,12 @@ func TestNeedUID(t *testing.T) { data := []testDataUID{ uidEqualsRootData, uidNotEqualsRootData, - {thisUID, eqOperator, Constraints{ - Operator: eqOperator, - UID: thisUID, - UIDSet: true}, + { + thisUID, eqOperator, Constraints{ + Operator: eqOperator, + UID: thisUID, + UIDSet: true, + }, }, } @@ -1000,62 +684,6 @@ func TestNeedNonRoot(t *testing.T) { checkUIDConstraints(assert, c, uidNotEqualsRootData.c, "TestNeedNonRoot") } -func TestNeedDistroWithOp(t *testing.T) { - assert := assert.New(t) - - if getDistroErr == errUnknownDistroName { - t.Skip(skipUnknownDistroName) - } - - data := []testDataDistro{ - distroEqualsCurrentData, - distroNotEqualsCurrentData, - - // check name provided is lower-cased - { - strings.ToUpper(distroName), - eqOperator, - Constraints{ - DistroName: distroName, - Operator: eqOperator, - }, - }, - } - - for i, d := range data { - - c := Constraints{} - - f := NeedDistroWithOp(d.distro, d.op) - f(&c) - - desc := fmt.Sprintf("test[%d]: %+v, constraints: %+v", i, d, c) - checkDistroConstraints(assert, d.c, c, desc) - } -} - -func TestNeedDistroEquals(t *testing.T) { - assert := assert.New(t) - - c := Constraints{} - - f := NeedDistroEquals(distroName) - f(&c) - - checkDistroConstraints(assert, c, distroEqualsCurrentData.c, "TestNeedDistroEquals") -} - -func TestNeedDistroNotEquals(t *testing.T) { - assert := assert.New(t) - - c := Constraints{} - - f := NeedDistroNotEquals(distroName) - f(&c) - - checkDistroConstraints(assert, c, distroNotEqualsCurrentData.c, "TestNeedDistroNotEquals") -} - func TestWithIssue(t *testing.T) { assert := assert.New(t) @@ -1171,22 +799,7 @@ func TestConstraintNotValid(t *testing.T) { assert.False(result) } - // Now test specification of multiple constraints - if root { - result := tc.NotValid(NeedRoot(), NeedDistro(distroName)) - assert.False(result) - - result = tc.NotValid(NeedNonRoot(), NeedDistro(distroName)) - assert.True(result) - } else { - result := tc.NotValid(NeedRoot(), NeedDistro(distroName)) - assert.True(result) - - result = tc.NotValid(NeedNonRoot(), NeedDistro(distroName)) - assert.False(result) - } } - } func TestConstraintNotValidKernelVersion(t *testing.T) { @@ -1278,62 +891,6 @@ func TestConstraintNotValidKernelVersion(t *testing.T) { } } -func TestConstraintNotValidDistroVersion(t *testing.T) { - assert := assert.New(t) - - assert.NotNil(distroVersion) - - // Generate new distro versions for testing purposes based on the - // current kernel version. - higherVersion, err := incrementVersion(distroVersion) - assert.NoError(err) - assert.NotEqual(distroVersion, higherVersion) - - lowerVersion, err := decrementVersion(distroVersion) - assert.NoError(err) - assert.NotEqual(distroVersion, lowerVersion) - - for _, debug := range []bool{true, false} { - tc := NewTestConstraint(debug) - - result := tc.NotValid(NeedDistroVersionEquals(higherVersion)) - assert.True(result) - - result = tc.NotValid(NeedDistroVersionEquals(distroVersion)) - assert.False(result) - - result = tc.NotValid(NeedDistroVersionLE(higherVersion)) - assert.False(result) - - result = tc.NotValid(NeedDistroVersionLE(distroVersion)) - assert.False(result) - - result = tc.NotValid(NeedDistroVersionLT(higherVersion)) - assert.False(result) - - result = tc.NotValid(NeedDistroVersionLT(distroVersion)) - assert.True(result) - - result = tc.NotValid(NeedDistroVersionGE(higherVersion)) - assert.True(result) - - result = tc.NotValid(NeedDistroVersionGE(distroVersion)) - assert.False(result) - - result = tc.NotValid(NeedDistroVersionGT(higherVersion)) - assert.True(result) - - result = tc.NotValid(NeedDistroVersionGT(distroVersion)) - assert.True(result) - - result = tc.NotValid(NeedDistroVersionNotEquals(higherVersion)) - assert.False(result) - - result = tc.NotValid(NeedDistroVersionNotEquals(distroVersion)) - assert.True(result) - } -} - func TestConstraintConstraintValid(t *testing.T) { assert := assert.New(t) @@ -1352,100 +909,6 @@ func TestConstraintConstraintValid(t *testing.T) { true, TestConstraint{Issue: issue}, }, - - { - NeedDistroWithOp(distroName, eqOperator), - true, - TestConstraint{ - Passed: []Result{ - {Success: true}, - }, - }, - }, - { - NeedDistroWithOp(distroName, neOperator), - false, - TestConstraint{ - Failed: []Result{ - {Success: false}, - }, - }, - }, - { - NeedDistroWithOp(getAnotherDistro(distroName), eqOperator), - false, - TestConstraint{ - Failed: []Result{ - {Success: false}, - }, - }, - }, - { - NeedDistroWithOp(getAnotherDistro(distroName), neOperator), - true, - TestConstraint{ - Failed: []Result{ - {Success: true}, - }, - }, - }, - - { - NeedDistroEquals(distroName), - true, - TestConstraint{ - Passed: []Result{ - {Success: true}, - }, - }, - }, - { - NeedDistroEquals(getAnotherDistro(distroName)), - false, - TestConstraint{ - Failed: []Result{ - {Success: false}, - }, - }, - }, - - { - NeedDistroNotEquals(getAnotherDistro(distroName)), - true, - TestConstraint{ - Passed: []Result{ - {Success: true}, - }, - }, - }, - { - NeedDistroNotEquals(distroName), - false, - TestConstraint{ - Failed: []Result{ - {Success: false}, - }, - }, - }, - - { - NeedDistro(distroName), - true, - TestConstraint{ - Passed: []Result{ - {Success: true}, - }, - }, - }, - { - NeedDistro(getAnotherDistro(distroName)), - false, - TestConstraint{ - Failed: []Result{ - {Success: false}, - }, - }, - }, } if root { @@ -1541,7 +1004,6 @@ func TestEvalIntVersion(t *testing.T) { data := []testData{ //---------- - {"", eqOperator, "", false, true}, {"", eqOperator, "1", false, true}, {"1", eqOperator, "", false, true}, @@ -1647,7 +1109,6 @@ func TestEvalFloatVersion(t *testing.T) { data := []testData{ //---------- - {"", eqOperator, "", false, true}, {"foo", eqOperator, "", false, true}, {"", eqOperator, "foo", false, true}, @@ -1763,7 +1224,6 @@ func TestEvalSemverVersion(t *testing.T) { data := []testData{ //---------- - {"", eqOperator, "", false, true}, {"foo", eqOperator, "", false, true}, {"", eqOperator, "foo", false, true}, diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index f97b64459..b9994c3f5 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -1,5 +1,6 @@ // Copyright (c) 2018-2022 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -154,6 +155,7 @@ type runtime struct { SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` EnablePprof bool `toml:"enable_pprof"` + DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` } type agent struct { @@ -568,7 +570,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { EntropySource: h.GetEntropySource(), EntropySourceList: h.EntropySourceList, DefaultBridges: h.defaultBridges(), - DisableBlockDeviceUse: h.DisableBlockDeviceUse, + DisableBlockDeviceUse: false, // shared fs is not supported in Firecracker, HugePages: h.HugePages, Debug: h.Debug, DisableNestingChecks: h.DisableNestingChecks, @@ -1174,6 +1176,8 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat } config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts + config.DisableGuestEmptyDir = tomlConf.Runtime.DisableGuestEmptyDir + if err := checkConfig(config); err != nil { return "", config, err } diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go index d429558f4..e456d3727 100644 --- a/src/runtime/pkg/katautils/create.go +++ b/src/runtime/pkg/katautils/create.go @@ -1,5 +1,6 @@ // Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -96,12 +97,12 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon // For the given pod ephemeral volume is created only once // backed by tmpfs inside the VM. For successive containers // of the same pod the already existing volume is reused. -func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec { +func SetEphemeralStorageType(ociSpec specs.Spec, disableGuestEmptyDir bool) specs.Spec { for idx, mnt := range ociSpec.Mounts { if vc.IsEphemeralStorage(mnt.Source) { ociSpec.Mounts[idx].Type = vc.KataEphemeralDevType } - if vc.Isk8sHostEmptyDir(mnt.Source) { + if vc.Isk8sHostEmptyDir(mnt.Source) && !disableGuestEmptyDir { ociSpec.Mounts[idx].Type = vc.KataLocalDevType } } @@ -218,14 +219,14 @@ func checkForFIPS(sandboxConfig *vc.SandboxConfig) error { } // CreateContainer create a container -func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) { +func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool, disableGuestEmptyDir bool) (vc.Process, error) { var c vc.VCContainer span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags) katatrace.AddTags(span, "container_id", containerID) defer span.End() - ociSpec = SetEphemeralStorageType(ociSpec) + ociSpec = SetEphemeralStorageType(ociSpec, disableGuestEmptyDir) contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput) if err != nil { diff --git a/src/runtime/pkg/katautils/create_test.go b/src/runtime/pkg/katautils/create_test.go index dab665dce..e2488aaa9 100644 --- a/src/runtime/pkg/katautils/create_test.go +++ b/src/runtime/pkg/katautils/create_test.go @@ -1,5 +1,6 @@ // Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -145,7 +146,7 @@ func TestSetEphemeralStorageType(t *testing.T) { ociMounts = append(ociMounts, mount) ociSpec.Mounts = ociMounts - ociSpec = SetEphemeralStorageType(ociSpec) + ociSpec = SetEphemeralStorageType(ociSpec, false) mountType := ociSpec.Mounts[0].Type assert.Equal(mountType, "ephemeral", @@ -367,7 +368,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) { rootFs := vc.RootFs{Mounted: true} for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput) + _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.False(vcmock.IsMockError(err)) assert.True(strings.Contains(err.Error(), containerType)) @@ -395,7 +396,7 @@ func TestCreateContainerFail(t *testing.T) { rootFs := vc.RootFs{Mounted: true} for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput) + _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.True(vcmock.IsMockError(err)) } @@ -430,7 +431,7 @@ func TestCreateContainer(t *testing.T) { rootFs := vc.RootFs{Mounted: true} for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput) + _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.NoError(err) } } diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index d136ed5da..539219e97 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -1,4 +1,5 @@ // Copyright (c) 2017 Intel Corporation +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -146,6 +147,9 @@ type RuntimeConfig struct { // Determines if enable pprof EnablePprof bool + // Determines if Kata creates emptyDir on the guest + DisableGuestEmptyDir bool + // Offload the CRI image management service to the Kata agent. ServiceOffload bool } diff --git a/src/runtime/pkg/resourcecontrol/controller.go b/src/runtime/pkg/resourcecontrol/controller.go index 1ffba476b..11dafd0e2 100644 --- a/src/runtime/pkg/resourcecontrol/controller.go +++ b/src/runtime/pkg/resourcecontrol/controller.go @@ -22,14 +22,14 @@ func SetLogger(logger *logrus.Entry) { controllerLogger = logger.WithFields(fields) } -// HypervisorType describes an hypervisor type. +// ResourceControllerType describes a resource controller type. type ResourceControllerType string const ( LinuxCgroups ResourceControllerType = "cgroups" ) -// String converts an hypervisor type to a string. +// String converts a resource type to a string. func (rType *ResourceControllerType) String() string { switch *rType { case LinuxCgroups: diff --git a/src/runtime/pkg/resourcecontrol/utils_linux.go b/src/runtime/pkg/resourcecontrol/utils_linux.go index 73d77dc23..f72890889 100644 --- a/src/runtime/pkg/resourcecontrol/utils_linux.go +++ b/src/runtime/pkg/resourcecontrol/utils_linux.go @@ -20,7 +20,7 @@ import ( // DefaultResourceControllerID runtime-determined location in the cgroups hierarchy. const DefaultResourceControllerID = "/vc" -// validCgroupPath returns a valid cgroup path. +// ValidCgroupPath returns a valid cgroup path. // see https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#cgroups-path func ValidCgroupPath(path string, systemdCgroup bool) (string, error) { if IsSystemdCgroup(path) { diff --git a/src/runtime/pkg/utils/shimclient/shim_management_client.go b/src/runtime/pkg/utils/shimclient/shim_management_client.go new file mode 100644 index 000000000..c9ed3ad69 --- /dev/null +++ b/src/runtime/pkg/utils/shimclient/shim_management_client.go @@ -0,0 +1,79 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package shimclient + +import ( + "bytes" + "fmt" + "io" + "net" + "net/http" + "time" + + cdshim "github.com/containerd/containerd/runtime/v2/shim" + shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" +) + +// BuildShimClient builds and returns an http client for communicating with the provided sandbox +func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) { + return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout) +} + +// buildUnixSocketClient build http client for Unix socket +func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) { + transport := &http.Transport{ + DisableKeepAlives: true, + Dial: func(proto, addr string) (conn net.Conn, err error) { + return cdshim.AnonDialer(socketAddr, timeout) + }, + } + + client := &http.Client{ + Transport: transport, + } + + if timeout > 0 { + client.Timeout = timeout + } + + return client, nil +} + +func DoGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) { + client, err := BuildShimClient(sandboxID, timeoutInSeconds) + if err != nil { + return nil, err + } + + resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath)) + if err != nil { + return nil, err + } + + defer func() { + resp.Body.Close() + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func DoPost(sandboxID string, timeoutInSeconds time.Duration, urlPath string, payload []byte) error { + client, err := BuildShimClient(sandboxID, timeoutInSeconds) + if err != nil { + return err + } + + resp, err := client.Post(fmt.Sprintf("http://shim/%s", urlPath), "application/json", bytes.NewBuffer(payload)) + defer func() { + resp.Body.Close() + }() + return err +} diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index f597d4e47..f91ab8043 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -192,6 +192,12 @@ type agent interface { // getAgentMetrics get metrics of agent and guest through agent getAgentMetrics(context.Context, *grpc.GetMetricsRequest) (*grpc.Metrics, error) + //getGuestVolumeStats get the filesystem stats of a volume specified by the volume mount path on the guest. + getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) + + // resizeGuestVolume resizes a volume specified by the volume mount path on the guest. + resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error + // pullImage will tell the agent to pull an image inside the Pod Sandbox image.ImageService } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 805fda716..95642dd39 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -198,10 +198,125 @@ func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error { return nil } +func (clh *cloudHypervisor) createVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil, nil + } + + virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id) + if err != nil { + return nil, err + } + + if clh.config.SharedFS == config.VirtioFSNydus { + apiSockPath, err := clh.nydusdAPISocketPath(clh.id) + if err != nil { + clh.Logger().WithError(err).Error("Invalid api socket path for nydusd") + return nil, err + } + nd := &nydusd{ + path: clh.config.VirtioFSDaemon, + sockPath: virtiofsdSocketPath, + apiSockPath: apiSockPath, + sourcePath: sharedPath, + debug: clh.config.Debug, + extraArgs: clh.config.VirtioFSExtraArgs, + startFn: startInShimNS, + } + nd.setupShareDirFn = nd.setupPassthroughFS + return nd, nil + } + + // default: use virtiofsd + return &virtiofsd{ + path: clh.config.VirtioFSDaemon, + sourcePath: sharedPath, + socketPath: virtiofsdSocketPath, + extraArgs: clh.config.VirtioFSExtraArgs, + debug: clh.config.Debug, + cache: clh.config.VirtioFSCache, + }, nil +} + +func (clh *cloudHypervisor) setupVirtiofsDaemon(ctx context.Context) error { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil + } + + if clh.config.SharedFS == config.Virtio9P { + return errors.New("cloud-hypervisor only supports virtio based file sharing") + } + + // virtioFS or virtioFsNydus + clh.Logger().WithField("function", "setupVirtiofsDaemon").Info("Starting virtiofsDaemon") + + if clh.virtiofsDaemon == nil { + return errors.New("Missing virtiofsDaemon configuration") + } + + pid, err := clh.virtiofsDaemon.Start(ctx, func() { + clh.StopVM(ctx, false) + }) + if err != nil { + return err + } + clh.state.VirtiofsDaemonPid = pid + + return nil +} + +func (clh *cloudHypervisor) stopVirtiofsDaemon(ctx context.Context) (err error) { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil + } + + if clh.state.VirtiofsDaemonPid == 0 { + clh.Logger().Warn("The virtiofsd had stopped") + return nil + } + + err = clh.virtiofsDaemon.Stop(ctx) + if err != nil { + return err + } + + clh.state.VirtiofsDaemonPid = 0 + + return nil +} + +func (clh *cloudHypervisor) loadVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil, nil + } + + virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id) + if err != nil { + return nil, err + } + + return &virtiofsd{ + PID: clh.state.VirtiofsDaemonPid, + sourcePath: sharedPath, + debug: clh.config.Debug, + socketPath: virtiofsdSocketPath, + }, nil +} + func (clh *cloudHypervisor) nydusdAPISocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.config.VMStorePath, id, nydusdAPISock) } +func (clh *cloudHypervisor) supportsSharedFS() bool { + caps := clh.Capabilities(clh.ctx) + + return caps.IsFsSharingSupported() +} + func (clh *cloudHypervisor) enableProtection() error { protection, err := availableGuestProtection() if err != nil { @@ -248,19 +363,15 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net clh.Logger().WithField("function", "CreateVM").Info("creating Sandbox") - virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id) - if err != nil { - return nil - } - if clh.state.PID > 0 { clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state") - clh.virtiofsDaemon = &virtiofsd{ - PID: clh.state.VirtiofsDaemonPid, - sourcePath: hypervisorConfig.SharedPath, - debug: clh.config.Debug, - socketPath: virtiofsdSocketPath, + + virtiofsDaemon, err := clh.loadVirtiofsDaemon(hypervisorConfig.SharedFS) + if err != nil { + return err } + clh.virtiofsDaemon = virtiofsDaemon + return nil } @@ -402,32 +513,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net ApiInternal: chclient.NewAPIClient(cfg).DefaultApi, } - clh.virtiofsDaemon = &virtiofsd{ - path: clh.config.VirtioFSDaemon, - sourcePath: filepath.Join(GetSharePath(clh.id)), - socketPath: virtiofsdSocketPath, - extraArgs: clh.config.VirtioFSExtraArgs, - debug: clh.config.Debug, - cache: clh.config.VirtioFSCache, - } - - if clh.config.SharedFS == config.VirtioFSNydus { - apiSockPath, err := clh.nydusdAPISocketPath(clh.id) - if err != nil { - clh.Logger().WithError(err).Error("Invalid api socket path for nydusd") - return err - } - nd := &nydusd{ - path: clh.config.VirtioFSDaemon, - sockPath: virtiofsdSocketPath, - apiSockPath: apiSockPath, - sourcePath: filepath.Join(GetSharePath(clh.id)), - debug: clh.config.Debug, - extraArgs: clh.config.VirtioFSExtraArgs, - startFn: startInShimNS, - } - nd.setupShareDirFn = nd.setupPassthroughFS - clh.virtiofsDaemon = nd + clh.virtiofsDaemon, err = clh.createVirtiofsDaemon(filepath.Join(GetSharePath(clh.id))) + if err != nil { + return err } if clh.config.SGXEPCSize > 0 { @@ -461,10 +549,6 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { return err } - if clh.virtiofsDaemon == nil { - return errors.New("Missing virtiofsDaemon configuration") - } - // This needs to be done as late as possible, just before launching // virtiofsd are executed by kata-runtime after this call, run with // the SELinux label. If these processes require privileged, we do @@ -477,24 +561,20 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { defer label.SetProcessLabel("") } - if clh.config.SharedFS == config.VirtioFS || clh.config.SharedFS == config.VirtioFSNydus { - clh.Logger().WithField("function", "StartVM").Info("Starting virtiofsDaemon") - pid, err := clh.virtiofsDaemon.Start(ctx, func() { - clh.StopVM(ctx, false) - }) - if err != nil { - return err - } - clh.state.VirtiofsDaemonPid = pid - } else { - return errors.New("cloud-hypervisor only supports virtio based file sharing") + err = clh.setupVirtiofsDaemon(ctx) + if err != nil { + return err } + defer func() { + if err != nil { + if shutdownErr := clh.stopVirtiofsDaemon(ctx); shutdownErr != nil { + clh.Logger().WithError(shutdownErr).Warn("error shutting down VirtiofsDaemon") + } + } + }() pid, err := clh.launchClh() if err != nil { - if shutdownErr := clh.virtiofsDaemon.Stop(ctx); shutdownErr != nil { - clh.Logger().WithError(shutdownErr).Warn("error shutting down VirtiofsDaemon") - } return fmt.Errorf("failed to launch cloud-hypervisor: %q", err) } clh.state.PID = pid @@ -638,10 +718,6 @@ func (clh *cloudHypervisor) HotplugAddDevice(ctx context.Context, devInfo interf span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugAddDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() - if clh.config.ConfidentialGuest { - return nil, errors.New("Device hotplug addition is not supported in confidential mode") - } - switch devType { case BlockDev: drive := devInfo.(*config.BlockDrive) @@ -659,10 +735,6 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugRemoveDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() - if clh.config.ConfidentialGuest { - return nil, errors.New("Device hotplug removal is not supported in confidential mode") - } - var deviceID string switch devType { @@ -890,6 +962,10 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{}, case types.HybridVSock: clh.addVSock(defaultGuestVSockCID, v.UdsPath) case types.Volume: + if !clh.supportsSharedFS() { + return fmt.Errorf("SharedFS is not supported") + } + err = clh.addVolume(v) default: clh.Logger().WithField("function", "AddDevice").Warnf("Add device of type %v is not supported.", v) @@ -916,10 +992,10 @@ func (clh *cloudHypervisor) Capabilities(ctx context.Context) types.Capabilities clh.Logger().WithField("function", "Capabilities").Info("get Capabilities") var caps types.Capabilities - caps.SetFsSharingSupport() if !clh.config.ConfidentialGuest { - caps.SetBlockDeviceHotplugSupport() + caps.SetFsSharingSupport() } + caps.SetBlockDeviceHotplugSupport() return caps } @@ -957,12 +1033,9 @@ func (clh *cloudHypervisor) terminate(ctx context.Context, waitOnly bool) (err e return err } - if clh.virtiofsDaemon == nil { - return errors.New("virtiofsDaemon config is nil, failed to stop it") - } - clh.Logger().Debug("stop virtiofsDaemon") - if err = clh.virtiofsDaemon.Stop(ctx); err != nil { + + if err = clh.stopVirtiofsDaemon(ctx); err != nil { clh.Logger().WithError(err).Error("failed to stop virtiofsDaemon") } diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 69cdbe1a9..eaf9a1231 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" @@ -598,22 +599,50 @@ func (c *Container) createBlockDevices(ctx context.Context) error { } // iterate all mounts and create block device if it's block based. - for i, m := range c.mounts { - if len(m.BlockDeviceID) > 0 { + for i := range c.mounts { + if len(c.mounts[i].BlockDeviceID) > 0 { // Non-empty m.BlockDeviceID indicates there's already one device // associated with the mount,so no need to create a new device for it // and we only create block device for bind mount continue } - if m.Type != "bind" { + if c.mounts[i].Type != "bind" { // We only handle for bind-mounts continue } + // Handle directly assigned volume. Update the mount info based on the mount info json. + mntInfo, e := volume.VolumeMountInfo(c.mounts[i].Source) + if e != nil && !os.IsNotExist(e) { + c.Logger().WithError(e).WithField("mount-source", c.mounts[i].Source). + Error("failed to parse the mount info file for a direct assigned volume") + continue + } + + if mntInfo != nil { + // Write out sandbox info file on the mount source to allow CSI to communicate with the runtime + if err := volume.RecordSandboxId(c.sandboxID, c.mounts[i].Source); err != nil { + c.Logger().WithError(err).Error("error writing sandbox info") + } + + readonly := false + for _, flag := range mntInfo.Options { + if flag == "ro" { + readonly = true + break + } + } + + c.mounts[i].Source = mntInfo.Device + c.mounts[i].Type = mntInfo.FsType + c.mounts[i].Options = mntInfo.Options + c.mounts[i].ReadOnly = readonly + } + var stat unix.Stat_t - if err := unix.Stat(m.Source, &stat); err != nil { - return fmt.Errorf("stat %q failed: %v", m.Source, err) + if err := unix.Stat(c.mounts[i].Source, &stat); err != nil { + return fmt.Errorf("stat %q failed: %v", c.mounts[i].Source, err) } var di *config.DeviceInfo @@ -623,17 +652,17 @@ func (c *Container) createBlockDevices(ctx context.Context) error { // instead of passing this as a shared mount. if stat.Mode&unix.S_IFBLK == unix.S_IFBLK { di = &config.DeviceInfo{ - HostPath: m.Source, - ContainerPath: m.Destination, + HostPath: c.mounts[i].Source, + ContainerPath: c.mounts[i].Destination, DevType: "b", Major: int64(unix.Major(uint64(stat.Rdev))), Minor: int64(unix.Minor(uint64(stat.Rdev))), - ReadOnly: m.ReadOnly, + ReadOnly: c.mounts[i].ReadOnly, } // Check whether source can be used as a pmem device - } else if di, err = config.PmemDeviceInfo(m.Source, m.Destination); err != nil { + } else if di, err = config.PmemDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination); err != nil { c.Logger().WithError(err). - WithField("mount-source", m.Source). + WithField("mount-source", c.mounts[i].Source). Debug("no loop device") } @@ -642,7 +671,7 @@ func (c *Container) createBlockDevices(ctx context.Context) error { if err != nil { // Do not return an error, try to create // devices for other mounts - c.Logger().WithError(err).WithField("mount-source", m.Source). + c.Logger().WithError(err).WithField("mount-source", c.mounts[i].Source). Error("device manager failed to create new device") continue diff --git a/src/runtime/virtcontainers/interfaces.go b/src/runtime/virtcontainers/interfaces.go index 4c95b44e8..ec3fe87bf 100644 --- a/src/runtime/virtcontainers/interfaces.go +++ b/src/runtime/virtcontainers/interfaces.go @@ -78,6 +78,9 @@ type VCSandbox interface { GetAgentMetrics(ctx context.Context) (string, error) GetAgentURL() (string, error) + GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error) + ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error + // Image management inside Sandbox image.ImageService } diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 8e089ac13..f3edfa286 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -6,6 +6,7 @@ package virtcontainers import ( + b64 "encoding/base64" "encoding/json" "errors" "fmt" @@ -140,6 +141,8 @@ const ( grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" grpcGetMetricsRequest = "grpc.GetMetricsRequest" grpcAddSwapRequest = "grpc.AddSwapRequest" + grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" + grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" ) // newKataAgent returns an agent from an agent type. @@ -884,35 +887,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st return nil } -func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error { - ociMounts := spec.Mounts - var index int - var m specs.Mount - - for i, v := range volumeStorages { - for index, m = range ociMounts { - if m.Destination != v.MountPoint { - continue - } - - // Create a temporary location to mount the Storage. Mounting to the correct location - // will be handled by the OCI mount structure. - filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) - path := filepath.Join(kataGuestSandboxStorageDir(), filename) - - k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) - ociMounts[index].Source = path - volumeStorages[i].MountPoint = path - - break - } - if index == len(ociMounts) { - return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint) - } - } - return nil -} - func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) { // Disable Hooks since they have been handled on the host and there is // no reason to send them to the agent. It would make no sense to try @@ -1249,19 +1223,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co // Append container devices for block devices passed with --device. ctrDevices = k.appendDevices(ctrDevices, c) - // Handle all the volumes that are block device files. - // Note this call modifies the list of container devices to make sure - // all hotplugged devices are unplugged, so this needs be done - // after devices passed with --device are handled. - volumeStorages, err := k.handleBlockVolumes(c) + // Block based volumes will require some adjustments in the OCI spec, and creation of + // storage objects to pass to the agent. + volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec) if err != nil { return nil, err } - if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { - return nil, err - } - ctrStorages = append(ctrStorages, volumeStorages...) grpcSpec, err := grpc.OCItoGRPC(ociSpec) @@ -1524,16 +1492,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D vol.Options = []string{"bind"} vol.MountPoint = m.Destination + // Assign the type from the mount, if it's specified (e.g. direct assigned volume) + if m.Type != "" { + vol.Fstype = m.Type + vol.Options = m.Options + } + return vol, nil } -// handleBlockVolumes handles volumes that are block devices files -// by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { +func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) { + var vol *grpc.Storage + + id := m.BlockDeviceID + device := c.sandbox.devManager.GetDeviceByID(id) + if device == nil { + k.Logger().WithField("device", id).Error("failed to find device by id") + return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) + } + + var err error + switch device.DeviceType() { + case config.DeviceBlock: + vol, err = k.handleDeviceBlockVolume(c, m, device) + case config.VhostUserBlk: + vol, err = k.handleVhostUserBlkVolume(c, m, device) + default: + return nil, fmt.Errorf("Unknown device type") + } + + return vol, err +} + +// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the +// given container and will update the OCI spec to utilize this mount point as the new source for the +// container volume. The container mount structure is updated to store the guest destination mountpoint. +func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage - for _, m := range c.mounts { + for i, m := range c.mounts { id := m.BlockDeviceID if len(id) == 0 { @@ -1544,29 +1542,39 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { // device is detached with detachDevices() for a container. c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination}) - var vol *grpc.Storage - - device := c.sandbox.devManager.GetDeviceByID(id) - if device == nil { - k.Logger().WithField("device", id).Error("failed to find device by id") - return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) - } - - var err error - switch device.DeviceType() { - case config.DeviceBlock: - vol, err = k.handleDeviceBlockVolume(c, m, device) - case config.VhostUserBlk: - vol, err = k.handleVhostUserBlkVolume(c, m, device) - default: - k.Logger().Error("Unknown device type") - continue - } - + // Create Storage structure + vol, err := k.createBlkStorageObject(c, m) if vol == nil || err != nil { return nil, err } + // Each device will be mounted at a unique location within the VM only once. Mounting + // to the container specific location is handled within the OCI spec. Let's ensure that + // the storage mount point is unique for each device. This is then utilized as the source + // in the OCI spec. If multiple containers mount the same block device, it's refcounted inside + // the guest by Kata agent. + filename := b64.StdEncoding.EncodeToString([]byte(vol.Source)) + // Make the base64 encoding path safe. + filename = strings.ReplaceAll(filename, "/", "_") + path := filepath.Join(kataGuestSandboxStorageDir(), filename) + + // Update applicable OCI mount source + for idx, ociMount := range spec.Mounts { + if ociMount.Destination != vol.MountPoint { + continue + } + k.Logger().WithFields(logrus.Fields{ + "original-source": ociMount.Source, + "new-source": path, + }).Debug("Replacing OCI mount source") + spec.Mounts[idx].Source = path + break + } + + // Update storage mountpoint, and save guest device mount path to container mount struct: + vol.MountPoint = path + c.mounts[i].GuestDeviceMount = path + volumeStorages = append(volumeStorages, vol) } @@ -1955,6 +1963,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest)) } + k.reqHandlers[grpcVolumeStatsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.GetVolumeStats(ctx, req.(*grpc.VolumeStatsRequest)) + } + k.reqHandlers[grpcResizeVolumeRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.ResizeVolume(ctx, req.(*grpc.ResizeVolumeRequest)) + } } func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) { @@ -2173,6 +2187,25 @@ func (k *kataAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsReq return resp.(*grpc.Metrics), nil } +func (k *kataAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) { + result, err := k.sendReq(ctx, &grpc.VolumeStatsRequest{VolumeGuestPath: volumeGuestPath}) + if err != nil { + return nil, err + } + + buf, err := json.Marshal(result.(*grpc.VolumeStatsResponse)) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (k *kataAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error { + _, err := k.sendReq(ctx, &grpc.ResizeVolumeRequest{VolumeGuestPath: volumeGuestPath, Size_: size}) + return err +} + func (k *kataAgent) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) { r := &grpc.PullImageRequest{ Image: req.Image, diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index bca567c3f..f494626c6 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -405,24 +405,28 @@ func TestHandleBlockVolume(t *testing.T) { containers[c.id].sandbox = &sandbox containers[c.id].mounts = mounts - volumeStorages, err := k.handleBlockVolumes(c) + vStorage, err := k.createBlkStorageObject(c, vMount) + assert.Nil(t, err, "Error while handling block volumes") + bStorage, err := k.createBlkStorageObject(c, bMount) + assert.Nil(t, err, "Error while handling block volumes") + dStorage, err := k.createBlkStorageObject(c, dMount) assert.Nil(t, err, "Error while handling block volumes") - vStorage := &pb.Storage{ + vStorageExpected := &pb.Storage{ MountPoint: vDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: vPCIPath.String(), } - bStorage := &pb.Storage{ + bStorageExpected := &pb.Storage{ MountPoint: bDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: bPCIPath.String(), } - dStorage := &pb.Storage{ + dStorageExpected := &pb.Storage{ MountPoint: dDestination, Fstype: "ext4", Options: []string{"ro"}, @@ -430,9 +434,9 @@ func TestHandleBlockVolume(t *testing.T) { Source: dPCIPath.String(), } - assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume") - assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume") - assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume") + assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume") + assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume") + assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume") } func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index 0025bb268..aa7c229c3 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -244,6 +244,14 @@ func (n *mockAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsReq return nil, nil } +func (n *mockAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) { + return nil, nil +} + +func (n *mockAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error { + return nil +} + func (k *mockAgent) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) { return nil, nil } diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index c2879f213..0e83bc689 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) { }, nil } - // We get the mount point by recursively peforming stat on the path + // We get the mount point by recursively performing stat on the path // The point where the device changes indicates the mountpoint for { if mountPoint == "/" { @@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string // Mount describes a container mount. type Mount struct { - Source string + // Source is the source of the mount. + Source string + // Destination is the destination of the mount (within the container). Destination string // Type specifies the type of filesystem to mount. @@ -335,6 +337,11 @@ type Mount struct { // HostPath used to store host side bind mount path HostPath string + // GuestDeviceMount represents the path within the VM that the device + // is mounted. Only relevant for block devices. This is tracked in the event + // runtime wants to query the agent for mount stats. + GuestDeviceMount string + // BlockDeviceID represents block device that is attached to the // VM in case this mount is a block device file or a directory // backed by a block device. diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index e4e79dee1..437c9c817 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -2415,6 +2415,87 @@ func (m *Metrics) XXX_DiscardUnknown() { var xxx_messageInfo_Metrics proto.InternalMessageInfo +type VolumeStatsRequest struct { + // The volume path on the guest outside the container + VolumeGuestPath string `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } +func (*VolumeStatsRequest) ProtoMessage() {} +func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{56} +} +func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeStatsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeStatsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeStatsRequest.Merge(m, src) +} +func (m *VolumeStatsRequest) XXX_Size() int { + return m.Size() +} +func (m *VolumeStatsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeStatsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeStatsRequest proto.InternalMessageInfo + +type ResizeVolumeRequest struct { + // Full VM guest path of the volume (outside the container) + VolumeGuestPath string `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"` + Size_ uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } +func (*ResizeVolumeRequest) ProtoMessage() {} +func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{57} +} +func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ResizeVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ResizeVolumeRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ResizeVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeVolumeRequest.Merge(m, src) +} +func (m *ResizeVolumeRequest) XXX_Size() int { + return m.Size() +} +func (m *ResizeVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ResizeVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ResizeVolumeRequest proto.InternalMessageInfo + func init() { proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") @@ -2474,6 +2555,8 @@ func init() { proto.RegisterType((*AddSwapRequest)(nil), "grpc.AddSwapRequest") proto.RegisterType((*GetMetricsRequest)(nil), "grpc.GetMetricsRequest") proto.RegisterType((*Metrics)(nil), "grpc.Metrics") + proto.RegisterType((*VolumeStatsRequest)(nil), "grpc.VolumeStatsRequest") + proto.RegisterType((*ResizeVolumeRequest)(nil), "grpc.ResizeVolumeRequest") } func init() { @@ -2481,194 +2564,198 @@ func init() { } var fileDescriptor_712ce9a559fda969 = []byte{ - // 2978 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x23, 0xc7, - 0xd1, 0xe6, 0x43, 0x22, 0x59, 0x7c, 0x89, 0x23, 0xad, 0x96, 0x4b, 0xdb, 0xfa, 0xd6, 0xb3, 0xf6, - 0x7a, 0x6d, 0x7f, 0xa6, 0xec, 0xb5, 0xf1, 0xad, 0x1f, 0xf0, 0xb7, 0x58, 0x69, 0x65, 0x49, 0xb6, - 0xe5, 0x65, 0x46, 0x16, 0x1c, 0x24, 0x48, 0x06, 0xc3, 0x99, 0x16, 0xd9, 0x16, 0x67, 0x7a, 0xdc, - 0xd3, 0xa3, 0x15, 0x1d, 0x20, 0xc8, 0x29, 0xb9, 0xe5, 0x98, 0x5b, 0xfe, 0x40, 0x90, 0x5b, 0x80, - 0x5c, 0x72, 0xcd, 0xc1, 0xc8, 0x29, 0xc7, 0x9c, 0x82, 0x78, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, - 0x79, 0xf0, 0x21, 0x27, 0x82, 0x80, 0x5c, 0x88, 0xae, 0xea, 0xea, 0x7a, 0x75, 0x57, 0x75, 0x55, - 0x0f, 0x61, 0x30, 0xc2, 0x6c, 0x1c, 0x0f, 0xfb, 0x2e, 0xf1, 0xb7, 0xcf, 0x1c, 0xe6, 0xbc, 0xe9, - 0x92, 0x80, 0x39, 0x38, 0x40, 0x34, 0x9a, 0x83, 0x23, 0xea, 0x6e, 0x4f, 0xf0, 0x30, 0xda, 0x0e, - 0x29, 0x61, 0xc4, 0x25, 0x13, 0x35, 0x8a, 0xb6, 0x9d, 0x11, 0x0a, 0x58, 0x5f, 0x00, 0x46, 0x79, - 0x44, 0x43, 0xb7, 0x57, 0x23, 0x2e, 0x96, 0x88, 0x5e, 0x9d, 0x4d, 0x43, 0x14, 0x29, 0xe0, 0xf9, - 0x11, 0x21, 0xa3, 0x09, 0x92, 0x0b, 0x87, 0xf1, 0xe9, 0x36, 0xf2, 0x43, 0x36, 0x95, 0x93, 0xe6, - 0x6f, 0x8b, 0xb0, 0xb9, 0x4b, 0x91, 0xc3, 0xd0, 0xae, 0x96, 0x6a, 0xa1, 0xaf, 0x63, 0x14, 0x31, - 0xe3, 0x25, 0x68, 0x24, 0x9a, 0xd8, 0xd8, 0xeb, 0x16, 0x6e, 0x17, 0xee, 0xd5, 0xac, 0x7a, 0x82, - 0x3b, 0xf4, 0x8c, 0x9b, 0x50, 0x41, 0x17, 0xc8, 0xe5, 0xb3, 0x45, 0x31, 0xbb, 0xca, 0xc1, 0x43, - 0xcf, 0x78, 0x1b, 0xea, 0x11, 0xa3, 0x38, 0x18, 0xd9, 0x71, 0x84, 0x68, 0xb7, 0x74, 0xbb, 0x70, - 0xaf, 0x7e, 0x7f, 0xad, 0xcf, 0xf5, 0xec, 0x1f, 0x8b, 0x89, 0x93, 0x08, 0x51, 0x0b, 0xa2, 0x64, - 0x6c, 0xdc, 0x85, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x6e, 0xf9, 0x76, 0xe9, 0x5e, 0xfd, 0x7e, - 0x43, 0x92, 0x3f, 0x16, 0x48, 0x4b, 0x4f, 0x1a, 0xaf, 0x41, 0x35, 0x62, 0x84, 0x3a, 0x23, 0x14, - 0x75, 0x57, 0x04, 0x61, 0x53, 0xf3, 0x15, 0x58, 0x2b, 0x99, 0x36, 0x5e, 0x80, 0xd2, 0x93, 0xdd, - 0xc3, 0xee, 0xaa, 0x90, 0x0e, 0x8a, 0x2a, 0x44, 0xae, 0xc5, 0xd1, 0xc6, 0x1d, 0x68, 0x46, 0x4e, - 0xe0, 0x0d, 0xc9, 0x85, 0x1d, 0x62, 0x2f, 0x88, 0xba, 0x95, 0xdb, 0x85, 0x7b, 0x55, 0xab, 0xa1, - 0x90, 0x03, 0x8e, 0x33, 0x3f, 0x80, 0x1b, 0xc7, 0xcc, 0xa1, 0xec, 0x0a, 0xde, 0x31, 0x4f, 0x60, - 0xd3, 0x42, 0x3e, 0x39, 0xbf, 0x92, 0x6b, 0xbb, 0x50, 0x61, 0xd8, 0x47, 0x24, 0x66, 0xc2, 0xb5, - 0x4d, 0x4b, 0x83, 0xe6, 0xef, 0x0b, 0x60, 0xec, 0x5d, 0x20, 0x77, 0x40, 0x89, 0x8b, 0xa2, 0xe8, - 0xbf, 0xb4, 0x5d, 0xaf, 0x42, 0x25, 0x94, 0x0a, 0x74, 0xcb, 0x82, 0x5c, 0xed, 0x82, 0xd6, 0x4a, - 0xcf, 0x9a, 0x5f, 0xc1, 0xc6, 0x31, 0x1e, 0x05, 0xce, 0xe4, 0x1a, 0xf5, 0xdd, 0x84, 0xd5, 0x48, - 0xf0, 0x14, 0xaa, 0x36, 0x2d, 0x05, 0x99, 0x03, 0x30, 0xbe, 0x74, 0x30, 0xbb, 0x3e, 0x49, 0xe6, - 0x9b, 0xb0, 0x9e, 0xe3, 0x18, 0x85, 0x24, 0x88, 0x90, 0x50, 0x80, 0x39, 0x2c, 0x8e, 0x04, 0xb3, - 0x15, 0x4b, 0x41, 0x26, 0x81, 0xcd, 0x93, 0xd0, 0xbb, 0x62, 0x34, 0xdd, 0x87, 0x1a, 0x45, 0x11, - 0x89, 0x29, 0x8f, 0x81, 0xa2, 0x70, 0xea, 0x86, 0x74, 0xea, 0x67, 0x38, 0x88, 0x2f, 0x2c, 0x3d, - 0x67, 0xa5, 0x64, 0xea, 0x7c, 0xb2, 0xe8, 0x2a, 0xe7, 0xf3, 0x03, 0xb8, 0x31, 0x70, 0xe2, 0xe8, - 0x2a, 0xba, 0x9a, 0x1f, 0xf2, 0xb3, 0x1d, 0xc5, 0xfe, 0x95, 0x16, 0xff, 0xae, 0x00, 0xd5, 0xdd, - 0x30, 0x3e, 0x89, 0x9c, 0x11, 0x32, 0xfe, 0x07, 0xea, 0x8c, 0x30, 0x67, 0x62, 0xc7, 0x1c, 0x14, - 0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x04, 0x8d, 0x10, 0x51, 0x37, 0x8c, 0x15, 0x45, 0xf1, - 0x76, 0xe9, 0x5e, 0xd9, 0xaa, 0x4b, 0x9c, 0x24, 0xe9, 0xc3, 0xba, 0x98, 0xb3, 0x71, 0x60, 0x9f, - 0x21, 0x1a, 0xa0, 0x89, 0x4f, 0x3c, 0x24, 0x0e, 0x47, 0xd9, 0xea, 0x88, 0xa9, 0xc3, 0xe0, 0xd3, - 0x64, 0xc2, 0x78, 0x1d, 0x3a, 0x09, 0x3d, 0x3f, 0xf1, 0x82, 0xba, 0x2c, 0xa8, 0xdb, 0x8a, 0xfa, - 0x44, 0xa1, 0xcd, 0x9f, 0x43, 0xeb, 0x8b, 0x31, 0x25, 0x8c, 0x4d, 0x70, 0x30, 0x7a, 0xec, 0x30, - 0x87, 0x87, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, 0x06, 0x74, 0x98, 0xa4, - 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x2d, 0x99, 0x18, 0x28, 0xe2, 0x57, 0xa0, 0x95, 0x12, - 0xf3, 0xe0, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x17, 0xd8, 0x47, 0xe6, 0xb9, 0xf0, 0x95, 0xd8, 0x64, - 0xe3, 0x0d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x13, 0xd2, 0x92, 0x27, 0x44, 0xbb, 0xd3, 0xaa, 0x26, - 0x4e, 0xf9, 0x08, 0xda, 0x2c, 0x51, 0xdc, 0xf6, 0x1c, 0xe6, 0xe4, 0x0f, 0x55, 0xde, 0x2a, 0xab, - 0xc5, 0x72, 0xb0, 0xf9, 0x21, 0xd4, 0x06, 0xd8, 0x8b, 0xa4, 0xe0, 0x2e, 0x54, 0xdc, 0x98, 0x52, - 0x14, 0x30, 0x6d, 0xb2, 0x02, 0x8d, 0x0d, 0x58, 0x99, 0x60, 0x1f, 0x33, 0x65, 0xa6, 0x04, 0x4c, - 0x02, 0x70, 0x84, 0x7c, 0x42, 0xa7, 0xc2, 0x61, 0x1b, 0xb0, 0x92, 0xdd, 0x5c, 0x09, 0x18, 0xcf, - 0x43, 0xcd, 0x77, 0x2e, 0x92, 0x4d, 0xe5, 0x33, 0x55, 0xdf, 0xb9, 0x90, 0xca, 0x77, 0xa1, 0x72, - 0xea, 0xe0, 0x89, 0x1b, 0x30, 0xe5, 0x15, 0x0d, 0xa6, 0x02, 0xcb, 0x59, 0x81, 0x7f, 0x2e, 0x42, - 0x5d, 0x4a, 0x94, 0x0a, 0x6f, 0xc0, 0x8a, 0xeb, 0xb8, 0xe3, 0x44, 0xa4, 0x00, 0x8c, 0xbb, 0x5a, - 0x91, 0x62, 0x36, 0xc3, 0xa5, 0x9a, 0x6a, 0xd5, 0xb6, 0x01, 0xa2, 0xa7, 0x4e, 0xa8, 0x74, 0x2b, - 0x2d, 0x21, 0xae, 0x71, 0x1a, 0xa9, 0xee, 0x3b, 0xd0, 0x90, 0xe7, 0x4e, 0x2d, 0x29, 0x2f, 0x59, - 0x52, 0x97, 0x54, 0x72, 0xd1, 0x1d, 0x68, 0xc6, 0x11, 0xb2, 0xc7, 0x18, 0x51, 0x87, 0xba, 0xe3, - 0x69, 0x77, 0x45, 0x5e, 0x40, 0x71, 0x84, 0x0e, 0x34, 0xce, 0xb8, 0x0f, 0x2b, 0x3c, 0xb7, 0x44, - 0xdd, 0x55, 0x71, 0xd7, 0xbd, 0x90, 0x65, 0x29, 0x4c, 0xed, 0x8b, 0xdf, 0xbd, 0x80, 0xd1, 0xa9, - 0x25, 0x49, 0x7b, 0xef, 0x01, 0xa4, 0x48, 0x63, 0x0d, 0x4a, 0x67, 0x68, 0xaa, 0xe2, 0x90, 0x0f, - 0xb9, 0x73, 0xce, 0x9d, 0x49, 0xac, 0xbd, 0x2e, 0x81, 0x0f, 0x8a, 0xef, 0x15, 0x4c, 0x17, 0xda, - 0x3b, 0x93, 0x33, 0x4c, 0x32, 0xcb, 0x37, 0x60, 0xc5, 0x77, 0xbe, 0x22, 0x54, 0x7b, 0x52, 0x00, - 0x02, 0x8b, 0x03, 0x42, 0x35, 0x0b, 0x01, 0x18, 0x2d, 0x28, 0x92, 0x50, 0xf8, 0xab, 0x66, 0x15, - 0x49, 0x98, 0x0a, 0x2a, 0x67, 0x04, 0x99, 0x7f, 0x2f, 0x03, 0xa4, 0x52, 0x0c, 0x0b, 0x7a, 0x98, - 0xd8, 0x11, 0xa2, 0xfc, 0x7e, 0xb7, 0x87, 0x53, 0x86, 0x22, 0x9b, 0x22, 0x37, 0xa6, 0x11, 0x3e, - 0xe7, 0xfb, 0xc7, 0xcd, 0xbe, 0x21, 0xcd, 0x9e, 0xd1, 0xcd, 0xba, 0x89, 0xc9, 0xb1, 0x5c, 0xb7, - 0xc3, 0x97, 0x59, 0x7a, 0x95, 0x71, 0x08, 0x37, 0x52, 0x9e, 0x5e, 0x86, 0x5d, 0xf1, 0x32, 0x76, - 0xeb, 0x09, 0x3b, 0x2f, 0x65, 0xb5, 0x07, 0xeb, 0x98, 0xd8, 0x5f, 0xc7, 0x28, 0xce, 0x31, 0x2a, - 0x5d, 0xc6, 0xa8, 0x83, 0xc9, 0x0f, 0xc4, 0x82, 0x94, 0xcd, 0x00, 0x6e, 0x65, 0xac, 0xe4, 0xe1, - 0x9e, 0x61, 0x56, 0xbe, 0x8c, 0xd9, 0x66, 0xa2, 0x15, 0xcf, 0x07, 0x29, 0xc7, 0x4f, 0x60, 0x13, - 0x13, 0xfb, 0xa9, 0x83, 0xd9, 0x2c, 0xbb, 0x95, 0xef, 0x31, 0x92, 0xdf, 0x68, 0x79, 0x5e, 0xd2, - 0x48, 0x1f, 0xd1, 0x51, 0xce, 0xc8, 0xd5, 0xef, 0x31, 0xf2, 0x48, 0x2c, 0x48, 0xd9, 0x3c, 0x82, - 0x0e, 0x26, 0xb3, 0xda, 0x54, 0x2e, 0x63, 0xd2, 0xc6, 0x24, 0xaf, 0xc9, 0x0e, 0x74, 0x22, 0xe4, - 0x32, 0x42, 0xb3, 0x87, 0xa0, 0x7a, 0x19, 0x8b, 0x35, 0x45, 0x9f, 0xf0, 0x30, 0x7f, 0x0c, 0x8d, - 0x83, 0x78, 0x84, 0xd8, 0x64, 0x98, 0x24, 0x83, 0x6b, 0xcb, 0x3f, 0xe6, 0x3f, 0x8b, 0x50, 0xdf, - 0x1d, 0x51, 0x12, 0x87, 0xb9, 0x9c, 0x2c, 0x83, 0x74, 0x36, 0x27, 0x0b, 0x12, 0x91, 0x93, 0x25, - 0xf1, 0xbb, 0xd0, 0xf0, 0x45, 0xe8, 0x2a, 0x7a, 0x99, 0x87, 0x3a, 0x73, 0x41, 0x6d, 0xd5, 0xfd, - 0x4c, 0x32, 0xeb, 0x03, 0x84, 0xd8, 0x8b, 0xd4, 0x1a, 0x99, 0x8e, 0xda, 0xaa, 0xdc, 0xd2, 0x29, - 0xda, 0xaa, 0x85, 0x49, 0xb6, 0x7e, 0x1b, 0xea, 0x43, 0xee, 0x24, 0xb5, 0x20, 0x97, 0x8c, 0x52, - 0xef, 0x59, 0x30, 0x4c, 0x83, 0xf0, 0x00, 0x9a, 0x63, 0xe9, 0x32, 0xb5, 0x48, 0x9e, 0xa1, 0x3b, - 0xca, 0x92, 0xd4, 0xde, 0x7e, 0xd6, 0xb3, 0x72, 0x03, 0x1a, 0xe3, 0x0c, 0xaa, 0x77, 0x0c, 0x9d, - 0x39, 0x92, 0x05, 0x39, 0xe8, 0x5e, 0x36, 0x07, 0xd5, 0xef, 0x1b, 0x52, 0x50, 0x76, 0x65, 0x36, - 0x2f, 0xfd, 0xba, 0x08, 0x8d, 0xcf, 0x11, 0x7b, 0x4a, 0xe8, 0x99, 0xd4, 0xd7, 0x80, 0x72, 0xe0, - 0xf8, 0x48, 0x71, 0x14, 0x63, 0xe3, 0x16, 0x54, 0xe9, 0x85, 0x4c, 0x20, 0x6a, 0x3f, 0x2b, 0xf4, - 0x42, 0x24, 0x06, 0xe3, 0x45, 0x00, 0x7a, 0x61, 0x87, 0x8e, 0x7b, 0x86, 0x94, 0x07, 0xcb, 0x56, - 0x8d, 0x5e, 0x0c, 0x24, 0x82, 0x1f, 0x05, 0x7a, 0x61, 0x23, 0x4a, 0x09, 0x8d, 0x54, 0xae, 0xaa, - 0xd2, 0x8b, 0x3d, 0x01, 0xab, 0xb5, 0x1e, 0x25, 0x61, 0x88, 0x3c, 0x91, 0xa3, 0xc5, 0xda, 0xc7, - 0x12, 0xc1, 0xa5, 0x32, 0x2d, 0x75, 0x55, 0x4a, 0x65, 0xa9, 0x54, 0x96, 0x4a, 0xad, 0xc8, 0x95, - 0x2c, 0x2b, 0x95, 0x25, 0x52, 0xab, 0x52, 0x2a, 0xcb, 0x48, 0x65, 0xa9, 0xd4, 0x9a, 0x5e, 0xab, - 0xa4, 0x9a, 0xbf, 0x2a, 0xc0, 0xe6, 0x6c, 0xe1, 0xa7, 0x6a, 0xd3, 0x77, 0xa1, 0xe1, 0x8a, 0xfd, - 0xca, 0x9d, 0xc9, 0xce, 0xdc, 0x4e, 0x5a, 0x75, 0x37, 0x73, 0x8c, 0x1f, 0x40, 0x33, 0x90, 0x0e, - 0x4e, 0x8e, 0x66, 0x29, 0xdd, 0x97, 0xac, 0xef, 0xad, 0x46, 0x90, 0x81, 0x4c, 0x0f, 0x8c, 0x2f, - 0x29, 0x66, 0xe8, 0x98, 0x51, 0xe4, 0xf8, 0xd7, 0x51, 0xdd, 0x1b, 0x50, 0x16, 0xd5, 0x0a, 0xdf, - 0xa6, 0x86, 0x25, 0xc6, 0xe6, 0xab, 0xb0, 0x9e, 0x93, 0xa2, 0x6c, 0x5d, 0x83, 0xd2, 0x04, 0x05, - 0x82, 0x7b, 0xd3, 0xe2, 0x43, 0xd3, 0x81, 0x8e, 0x85, 0x1c, 0xef, 0xfa, 0xb4, 0x51, 0x22, 0x4a, - 0xa9, 0x88, 0x7b, 0x60, 0x64, 0x45, 0x28, 0x55, 0xb4, 0xd6, 0x85, 0x8c, 0xd6, 0x4f, 0xa0, 0xb3, - 0x3b, 0x21, 0x11, 0x3a, 0x66, 0x1e, 0x0e, 0xae, 0xa3, 0x1d, 0xf9, 0x19, 0xac, 0x7f, 0xc1, 0xa6, - 0x5f, 0x72, 0x66, 0x11, 0xfe, 0x06, 0x5d, 0x93, 0x7d, 0x94, 0x3c, 0xd5, 0xf6, 0x51, 0xf2, 0x94, - 0x37, 0x37, 0x2e, 0x99, 0xc4, 0x7e, 0x20, 0x42, 0xa1, 0x69, 0x29, 0xc8, 0xdc, 0x81, 0x86, 0xac, - 0xa1, 0x8f, 0x88, 0x17, 0x4f, 0xd0, 0xc2, 0x18, 0xdc, 0x02, 0x08, 0x1d, 0xea, 0xf8, 0x88, 0x21, - 0x2a, 0xcf, 0x50, 0xcd, 0xca, 0x60, 0xcc, 0xdf, 0x14, 0x61, 0x43, 0xbe, 0x37, 0x1c, 0xcb, 0x36, - 0x5b, 0x9b, 0xd0, 0x83, 0xea, 0x98, 0x44, 0x2c, 0xc3, 0x30, 0x81, 0xb9, 0x8a, 0xbc, 0x3f, 0x97, - 0xdc, 0xf8, 0x30, 0xf7, 0x08, 0x50, 0xba, 0xfc, 0x11, 0x60, 0xae, 0xcd, 0x2f, 0xcf, 0xb7, 0xf9, - 0x3c, 0xda, 0x34, 0x11, 0x96, 0x31, 0x5e, 0xb3, 0x6a, 0x0a, 0x73, 0xe8, 0x19, 0x77, 0xa1, 0x3d, - 0xe2, 0x5a, 0xda, 0x63, 0x42, 0xce, 0xec, 0xd0, 0x61, 0x63, 0x11, 0xea, 0x35, 0xab, 0x29, 0xd0, - 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0xde, 0x87, 0x96, 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, - 0x5d, 0x7e, 0x2a, 0x8a, 0xb2, 0xde, 0xb3, 0x9a, 0x67, 0x19, 0x28, 0x32, 0x6f, 0xc2, 0x8d, 0xc7, - 0x28, 0x62, 0x94, 0x4c, 0xf3, 0x8e, 0x31, 0xff, 0x1f, 0xe0, 0x30, 0x60, 0x88, 0x9e, 0x3a, 0x2e, - 0x8a, 0x8c, 0xb7, 0xb2, 0x90, 0x2a, 0x8e, 0xd6, 0xfa, 0xf2, 0xb9, 0x27, 0x99, 0xb0, 0x32, 0x34, - 0x66, 0x1f, 0x56, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0xb2, 0x1e, 0xa9, 0x75, 0x0d, 0xb5, 0x4e, 0x20, - 0x2d, 0x35, 0x67, 0x1e, 0xe8, 0x16, 0x36, 0x65, 0xa7, 0xb6, 0xa8, 0x0f, 0x35, 0xac, 0x71, 0x2a, - 0xab, 0xcc, 0x8b, 0x4e, 0x49, 0xcc, 0x0f, 0x61, 0x5d, 0x72, 0x92, 0x9c, 0x35, 0x9b, 0x97, 0x61, - 0x95, 0x6a, 0x35, 0x0a, 0xe9, 0x3b, 0x8f, 0x22, 0x52, 0x73, 0xdc, 0x1f, 0x9f, 0xe1, 0x88, 0xa5, - 0x86, 0x68, 0x7f, 0xac, 0x43, 0x87, 0x4f, 0xe4, 0x78, 0x9a, 0x1f, 0x43, 0xe3, 0x91, 0x35, 0xf8, - 0x1c, 0xe1, 0xd1, 0x78, 0xc8, 0xb3, 0xe7, 0xff, 0xe5, 0x61, 0x65, 0xb0, 0xa1, 0xb4, 0xcd, 0x4c, - 0x59, 0x39, 0x3a, 0xf3, 0x13, 0xd8, 0x7c, 0xe4, 0x79, 0x59, 0x94, 0xd6, 0xfa, 0x2d, 0xa8, 0x05, - 0x19, 0x76, 0x99, 0x3b, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x3f, 0x81, 0xf5, 0x27, 0xc1, 0x04, 0x07, - 0x68, 0x77, 0x70, 0x72, 0x84, 0x92, 0x5c, 0x64, 0x40, 0x99, 0xd7, 0x6c, 0x82, 0x47, 0xd5, 0x12, - 0x63, 0x1e, 0x9c, 0xc1, 0xd0, 0x76, 0xc3, 0x38, 0x52, 0x8f, 0x3d, 0xab, 0xc1, 0x70, 0x37, 0x8c, - 0x23, 0x7e, 0xb9, 0xf0, 0xe2, 0x82, 0x04, 0x93, 0xa9, 0x88, 0xd0, 0xaa, 0x55, 0x71, 0xc3, 0xf8, - 0x49, 0x30, 0x99, 0x9a, 0xff, 0x2b, 0x3a, 0x70, 0x84, 0x3c, 0xcb, 0x09, 0x3c, 0xe2, 0x3f, 0x46, - 0xe7, 0x19, 0x09, 0x49, 0xb7, 0xa7, 0x33, 0xd1, 0xb7, 0x05, 0x68, 0x3c, 0x1a, 0xa1, 0x80, 0x3d, - 0x46, 0xcc, 0xc1, 0x13, 0xd1, 0xd1, 0x9d, 0x23, 0x1a, 0x61, 0x12, 0xa8, 0x70, 0xd3, 0x20, 0x6f, - 0xc8, 0x71, 0x80, 0x99, 0xed, 0x39, 0xc8, 0x27, 0x81, 0xe0, 0x52, 0xb5, 0x80, 0xa3, 0x1e, 0x0b, - 0x8c, 0xf1, 0x2a, 0xb4, 0xe5, 0x63, 0x9c, 0x3d, 0x76, 0x02, 0x6f, 0xc2, 0x03, 0xbd, 0x24, 0x42, - 0xb3, 0x25, 0xd1, 0x07, 0x0a, 0x6b, 0xbc, 0x06, 0x6b, 0x2a, 0x0c, 0x53, 0xca, 0xb2, 0xa0, 0x6c, - 0x2b, 0x7c, 0x8e, 0x34, 0x0e, 0x43, 0x42, 0x59, 0x64, 0x47, 0xc8, 0x75, 0x89, 0x1f, 0xaa, 0x76, - 0xa8, 0xad, 0xf1, 0xc7, 0x12, 0x6d, 0x8e, 0x60, 0x7d, 0x9f, 0xdb, 0xa9, 0x2c, 0x49, 0x8f, 0x55, - 0xcb, 0x47, 0xbe, 0x3d, 0x9c, 0x10, 0xf7, 0xcc, 0xe6, 0xc9, 0x51, 0x79, 0x98, 0x17, 0x5c, 0x3b, - 0x1c, 0x79, 0x8c, 0xbf, 0x11, 0x9d, 0x3f, 0xa7, 0x1a, 0x13, 0x16, 0x4e, 0xe2, 0x91, 0x1d, 0x52, - 0x32, 0x44, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, 0xa7, 0x02, 0x6c, - 0xe4, 0x25, 0xa9, 0x54, 0xbf, 0x0d, 0x1b, 0x79, 0x51, 0xea, 0xfa, 0x97, 0xe5, 0x65, 0x27, 0x2b, - 0x50, 0x16, 0x02, 0x0f, 0xa0, 0x29, 0xde, 0x6b, 0x6d, 0x4f, 0x72, 0xca, 0x17, 0x3d, 0xd9, 0x7d, - 0xb1, 0x1a, 0x4e, 0x76, 0x97, 0xde, 0x87, 0x5b, 0xca, 0x7c, 0x7b, 0x5e, 0x6d, 0x79, 0x20, 0x36, - 0x15, 0xc1, 0xd1, 0x8c, 0xf6, 0x9f, 0x41, 0x37, 0x45, 0xed, 0x4c, 0x05, 0x32, 0x3d, 0xcc, 0xeb, - 0x33, 0xc6, 0x3e, 0xf2, 0x3c, 0x2a, 0xa2, 0xa4, 0x6c, 0x2d, 0x9a, 0x32, 0x1f, 0xc2, 0xcd, 0x63, - 0xc4, 0xa4, 0x37, 0x1c, 0xa6, 0x3a, 0x11, 0xc9, 0x6c, 0x0d, 0x4a, 0xc7, 0xc8, 0x15, 0xc6, 0x97, - 0x2c, 0x3e, 0xe4, 0x07, 0xf0, 0x24, 0x42, 0xae, 0xb0, 0xb2, 0x64, 0x89, 0xb1, 0xf9, 0x87, 0x02, - 0x54, 0x54, 0x72, 0xe6, 0x17, 0x8c, 0x47, 0xf1, 0x39, 0xa2, 0xea, 0xe8, 0x29, 0xc8, 0x78, 0x05, - 0x5a, 0x72, 0x64, 0x93, 0x90, 0x61, 0x92, 0xa4, 0xfc, 0xa6, 0xc4, 0x3e, 0x91, 0x48, 0xf1, 0xf8, - 0x26, 0x9e, 0xbf, 0x54, 0xa7, 0xa9, 0x20, 0x8e, 0x3f, 0x8d, 0x78, 0x84, 0x8b, 0x14, 0x5f, 0xb3, - 0x14, 0xc4, 0x8f, 0xba, 0xe6, 0xb7, 0x22, 0xf8, 0x69, 0x90, 0x1f, 0x75, 0x9f, 0xc4, 0x01, 0xb3, - 0x43, 0x82, 0x03, 0xa6, 0x72, 0x3a, 0x08, 0xd4, 0x80, 0x63, 0xcc, 0x5f, 0x16, 0x60, 0x55, 0x3e, - 0x40, 0xf3, 0xde, 0x36, 0xb9, 0x59, 0x8b, 0x58, 0x54, 0x29, 0x42, 0x96, 0xbc, 0x4d, 0xc5, 0x98, - 0xc7, 0xf1, 0xb9, 0x2f, 0xef, 0x07, 0xa5, 0xda, 0xb9, 0x2f, 0x2e, 0x86, 0x57, 0xa0, 0x95, 0x5e, - 0xd0, 0x62, 0x5e, 0xaa, 0xd8, 0x4c, 0xb0, 0x82, 0x6c, 0xa9, 0xa6, 0xe6, 0x0f, 0x79, 0x4b, 0x9f, - 0x3c, 0xbe, 0xae, 0x41, 0x29, 0x4e, 0x94, 0xe1, 0x43, 0x8e, 0x19, 0x25, 0x57, 0x3b, 0x1f, 0x1a, - 0x77, 0xa1, 0xe5, 0x78, 0x1e, 0xe6, 0xcb, 0x9d, 0xc9, 0x3e, 0xf6, 0x92, 0x20, 0xcd, 0x63, 0xcd, - 0xbf, 0x14, 0xa0, 0xbd, 0x4b, 0xc2, 0xe9, 0xc7, 0x78, 0x82, 0x32, 0x19, 0x44, 0x28, 0xa9, 0x6e, - 0x76, 0x3e, 0xe6, 0xd5, 0xea, 0x29, 0x9e, 0x20, 0x19, 0x5a, 0x72, 0x67, 0xab, 0x1c, 0x21, 0xc2, - 0x4a, 0x4f, 0x26, 0xcf, 0x6e, 0x4d, 0x39, 0x79, 0x44, 0x3c, 0x51, 0x97, 0x7b, 0x98, 0xda, 0xc9, - 0x23, 0x5b, 0xd3, 0xaa, 0x78, 0x98, 0x8a, 0x29, 0x65, 0xc8, 0x8a, 0x78, 0x44, 0xcd, 0x1a, 0xb2, - 0x2a, 0x31, 0xdc, 0x90, 0x4d, 0x58, 0x25, 0xa7, 0xa7, 0x11, 0x62, 0xa2, 0x82, 0x2e, 0x59, 0x0a, - 0x4a, 0xd2, 0x5c, 0x35, 0x93, 0xe6, 0x36, 0xc0, 0xd8, 0x47, 0xec, 0xc9, 0x93, 0xa3, 0xbd, 0x73, - 0x14, 0x30, 0x7d, 0x3b, 0xbc, 0x09, 0x55, 0x8d, 0xfa, 0x77, 0x9e, 0x27, 0x5f, 0x87, 0xd6, 0x23, - 0xcf, 0x3b, 0x7e, 0xea, 0x84, 0xda, 0x1f, 0x5d, 0xa8, 0x0c, 0x76, 0x0f, 0x07, 0xd2, 0x25, 0x25, - 0x6e, 0x80, 0x02, 0xf9, 0x6d, 0xb4, 0x8f, 0xd8, 0x11, 0x62, 0x14, 0xbb, 0xc9, 0x6d, 0x74, 0x07, - 0x2a, 0x0a, 0xc3, 0x57, 0xfa, 0x72, 0xa8, 0xd3, 0xac, 0x02, 0xef, 0xff, 0x71, 0x4d, 0x65, 0x64, - 0xd5, 0xdc, 0x1b, 0xfb, 0xd0, 0x9e, 0xf9, 0x12, 0x63, 0xa8, 0xd7, 0x9e, 0xc5, 0x1f, 0x68, 0x7a, - 0x9b, 0x7d, 0xf9, 0x65, 0xa7, 0xaf, 0xbf, 0xec, 0xf4, 0xf7, 0xfc, 0x90, 0x4d, 0x8d, 0x3d, 0x68, - 0xe5, 0xbf, 0x59, 0x18, 0xcf, 0xeb, 0xe2, 0x68, 0xc1, 0x97, 0x8c, 0xa5, 0x6c, 0xf6, 0xa1, 0x3d, - 0xf3, 0xf9, 0x42, 0xeb, 0xb3, 0xf8, 0xab, 0xc6, 0x52, 0x46, 0x0f, 0xa1, 0x9e, 0xf9, 0x5e, 0x61, - 0x74, 0x25, 0x93, 0xf9, 0x4f, 0x18, 0x4b, 0x19, 0xec, 0x42, 0x33, 0xf7, 0x09, 0xc1, 0xe8, 0x29, - 0x7b, 0x16, 0x7c, 0x57, 0x58, 0xca, 0x64, 0x07, 0xea, 0x99, 0x97, 0x7c, 0xad, 0xc5, 0xfc, 0xe7, - 0x82, 0xde, 0xad, 0x05, 0x33, 0x2a, 0xf1, 0xef, 0x43, 0x7b, 0xe6, 0x79, 0x5f, 0xbb, 0x64, 0xf1, - 0xab, 0xff, 0x52, 0x65, 0x3e, 0x15, 0x5b, 0x94, 0xe9, 0xde, 0x32, 0x5b, 0x34, 0xff, 0x98, 0xdf, - 0x7b, 0x61, 0xf1, 0xa4, 0xd2, 0x6a, 0x0f, 0x5a, 0xf9, 0x77, 0x7c, 0xcd, 0x6c, 0xe1, 0xeb, 0xfe, - 0xe5, 0xfb, 0x9d, 0x7b, 0xd2, 0x4f, 0xf7, 0x7b, 0xd1, 0x4b, 0xff, 0x52, 0x46, 0x8f, 0x00, 0x54, - 0xaf, 0xe6, 0xe1, 0x20, 0x71, 0xf4, 0x5c, 0x8f, 0x98, 0x38, 0x7a, 0x41, 0x5f, 0xf7, 0x10, 0x40, - 0xb6, 0x58, 0x1e, 0x89, 0x99, 0x71, 0x53, 0xab, 0x31, 0xd3, 0xd7, 0xf5, 0xba, 0xf3, 0x13, 0x73, - 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x11, 0x40, 0xda, 0xba, 0x69, 0x06, 0x73, 0xcd, 0xdc, 0x25, - 0x3e, 0x68, 0x64, 0x1b, 0x35, 0x43, 0xd9, 0xba, 0xa0, 0x79, 0xbb, 0x84, 0x45, 0x7b, 0xa6, 0x10, - 0xcf, 0x1f, 0xb6, 0xd9, 0xfa, 0xbc, 0x37, 0x57, 0x8c, 0x1b, 0x0f, 0xa0, 0x91, 0xad, 0xc0, 0xb5, - 0x16, 0x0b, 0xaa, 0xf2, 0x5e, 0xae, 0x0a, 0x37, 0x1e, 0x42, 0x2b, 0x5f, 0x7d, 0xeb, 0x23, 0xb5, - 0xb0, 0x26, 0xef, 0xa9, 0xb7, 0xa5, 0x0c, 0xf9, 0x3b, 0x00, 0x69, 0x95, 0xae, 0xdd, 0x37, 0x57, - 0xb7, 0xcf, 0x48, 0xdd, 0x87, 0xf6, 0x4c, 0xf5, 0xad, 0x2d, 0x5e, 0x5c, 0x94, 0x2f, 0x75, 0xdd, - 0xbb, 0x00, 0x69, 0x56, 0xd6, 0xd2, 0xe7, 0xf2, 0x74, 0xaf, 0xa9, 0xdf, 0xdd, 0x24, 0xdd, 0x2e, - 0x34, 0x73, 0xad, 0xa9, 0x4e, 0x33, 0x8b, 0xfa, 0xd5, 0xcb, 0x92, 0x6f, 0xbe, 0x8f, 0xd3, 0x9e, - 0x5b, 0xd8, 0xdd, 0x5d, 0x76, 0x7e, 0xb2, 0xcd, 0x83, 0xde, 0xb9, 0x05, 0x0d, 0xc5, 0xf7, 0xc4, - 0x73, 0xb6, 0x41, 0xc8, 0xc4, 0xf3, 0x82, 0xbe, 0x61, 0x29, 0xa3, 0x03, 0x68, 0xef, 0xeb, 0xda, - 0x4f, 0xd5, 0xa5, 0x4a, 0x9d, 0x05, 0x75, 0x78, 0xaf, 0xb7, 0x68, 0x4a, 0x05, 0xd5, 0xa7, 0xd0, - 0x99, 0xab, 0x49, 0x8d, 0xad, 0xe4, 0xf5, 0x73, 0x61, 0xb1, 0xba, 0x54, 0xad, 0x43, 0x58, 0x9b, - 0x2d, 0x49, 0x8d, 0x17, 0x55, 0xa2, 0x5c, 0x5c, 0xaa, 0x2e, 0x65, 0xf5, 0x3e, 0x54, 0x75, 0x09, - 0x64, 0xa8, 0x57, 0xe6, 0x99, 0x92, 0x68, 0xe9, 0xd2, 0x07, 0x50, 0xcf, 0x54, 0x1c, 0x3a, 0xdb, - 0xcd, 0x17, 0x21, 0x3d, 0xf5, 0x28, 0x9c, 0x50, 0x3e, 0x80, 0x8a, 0xaa, 0x32, 0x8c, 0x8d, 0xe4, - 0x90, 0x67, 0x8a, 0x8e, 0x65, 0x12, 0x77, 0x2e, 0xbe, 0xfd, 0x6e, 0xeb, 0xb9, 0xbf, 0x7d, 0xb7, - 0xf5, 0xdc, 0x2f, 0x9e, 0x6d, 0x15, 0xbe, 0x7d, 0xb6, 0x55, 0xf8, 0xeb, 0xb3, 0xad, 0xc2, 0x3f, - 0x9e, 0x6d, 0x15, 0x7e, 0xf4, 0xd3, 0xff, 0xf0, 0x9f, 0x25, 0x34, 0x0e, 0x18, 0xf6, 0xd1, 0xf6, - 0x39, 0xa6, 0x2c, 0x33, 0x15, 0x9e, 0x8d, 0xe4, 0xdf, 0x4b, 0x32, 0xff, 0x3a, 0xe1, 0x0a, 0x0e, - 0x57, 0x05, 0xfc, 0xce, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x52, 0x45, 0x13, 0x9c, 0xc2, 0x22, - 0x00, 0x00, + // 3055 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47, + 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed, + 0xf5, 0xda, 0xc6, 0x92, 0xbd, 0x76, 0xb0, 0x7e, 0x84, 0x59, 0x24, 0xad, 0x2c, 0xc9, 0xb6, 0xbc, + 0x43, 0xcb, 0xc2, 0x04, 0x04, 0x74, 0xb4, 0xba, 0x6b, 0x47, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab, + 0xb5, 0x92, 0x89, 0x20, 0x38, 0xc1, 0x8d, 0x23, 0x37, 0x7e, 0x80, 0xe0, 0xc6, 0x91, 0x0b, 0x07, + 0x0e, 0x0e, 0x4e, 0x1c, 0x39, 0x11, 0x78, 0x3f, 0x81, 0x2f, 0x20, 0xea, 0xd5, 0x5d, 0x3d, 0x0f, + 0x19, 0x14, 0x1b, 0xc1, 0x65, 0xa2, 0x33, 0x2b, 0x2b, 0x5f, 0x55, 0x99, 0x95, 0x59, 0x35, 0x30, + 0x1c, 0x61, 0x76, 0x92, 0x1e, 0xaf, 0xfb, 0x24, 0xdc, 0x38, 0xf5, 0x98, 0xf7, 0xba, 0x4f, 0x22, + 0xe6, 0xe1, 0x08, 0xd1, 0x64, 0x0a, 0x4e, 0xa8, 0xbf, 0x31, 0xc6, 0xc7, 0xc9, 0x46, 0x4c, 0x09, + 0x23, 0x3e, 0x19, 0xab, 0xaf, 0x64, 0xc3, 0x1b, 0xa1, 0x88, 0xad, 0x0b, 0xc0, 0xaa, 0x8e, 0x68, + 0xec, 0x0f, 0x1a, 0xc4, 0xc7, 0x12, 0x31, 0x68, 0xf8, 0x89, 0xfe, 0x6c, 0xb2, 0x8b, 0x18, 0x25, + 0x0a, 0x78, 0x76, 0x44, 0xc8, 0x68, 0x8c, 0x24, 0x8f, 0xe3, 0xf4, 0xd1, 0x06, 0x0a, 0x63, 0x76, + 0x21, 0x07, 0xed, 0xdf, 0x97, 0x61, 0x75, 0x9b, 0x22, 0x8f, 0xa1, 0x6d, 0xad, 0x80, 0x83, 0xbe, + 0x4c, 0x51, 0xc2, 0xac, 0x17, 0xa0, 0x95, 0x29, 0xe5, 0xe2, 0xa0, 0x5f, 0xba, 0x59, 0xba, 0xd3, + 0x70, 0x9a, 0x19, 0x6e, 0x3f, 0xb0, 0xae, 0x43, 0x0d, 0x9d, 0x23, 0x9f, 0x8f, 0x96, 0xc5, 0xe8, + 0x22, 0x07, 0xf7, 0x03, 0xeb, 0x4d, 0x68, 0x26, 0x8c, 0xe2, 0x68, 0xe4, 0xa6, 0x09, 0xa2, 0xfd, + 0xca, 0xcd, 0xd2, 0x9d, 0xe6, 0xdd, 0xa5, 0x75, 0xae, 0xf2, 0xfa, 0xa1, 0x18, 0x38, 0x4a, 0x10, + 0x75, 0x20, 0xc9, 0xbe, 0xad, 0xdb, 0x50, 0x0b, 0xd0, 0x19, 0xf6, 0x51, 0xd2, 0xaf, 0xde, 0xac, + 0xdc, 0x69, 0xde, 0x6d, 0x49, 0xf2, 0x07, 0x02, 0xe9, 0xe8, 0x41, 0xeb, 0x15, 0xa8, 0x27, 0x8c, + 0x50, 0x6f, 0x84, 0x92, 0xfe, 0x82, 0x20, 0x6c, 0x6b, 0xbe, 0x02, 0xeb, 0x64, 0xc3, 0xd6, 0x73, + 0x50, 0x79, 0xb8, 0xbd, 0xdf, 0x5f, 0x14, 0xd2, 0x41, 0x51, 0xc5, 0xc8, 0x77, 0x38, 0xda, 0xba, + 0x05, 0xed, 0xc4, 0x8b, 0x82, 0x63, 0x72, 0xee, 0xc6, 0x38, 0x88, 0x92, 0x7e, 0xed, 0x66, 0xe9, + 0x4e, 0xdd, 0x69, 0x29, 0xe4, 0x90, 0xe3, 0xec, 0xf7, 0xe0, 0xda, 0x21, 0xf3, 0x28, 0xbb, 0x82, + 0x77, 0xec, 0x23, 0x58, 0x75, 0x50, 0x48, 0xce, 0xae, 0xe4, 0xda, 0x3e, 0xd4, 0x18, 0x0e, 0x11, + 0x49, 0x99, 0x70, 0x6d, 0xdb, 0xd1, 0xa0, 0xfd, 0xc7, 0x12, 0x58, 0x3b, 0xe7, 0xc8, 0x1f, 0x52, + 0xe2, 0xa3, 0x24, 0xf9, 0x3f, 0x2d, 0xd7, 0xcb, 0x50, 0x8b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57, + 0xab, 0xa0, 0xb5, 0xd2, 0xa3, 0xf6, 0x17, 0xb0, 0x72, 0x88, 0x47, 0x91, 0x37, 0x7e, 0x8a, 0xfa, + 0xae, 0xc2, 0x62, 0x22, 0x78, 0x0a, 0x55, 0xdb, 0x8e, 0x82, 0xec, 0x21, 0x58, 0x9f, 0x7b, 0x98, + 0x3d, 0x3d, 0x49, 0xf6, 0xeb, 0xb0, 0x5c, 0xe0, 0x98, 0xc4, 0x24, 0x4a, 0x90, 0x50, 0x80, 0x79, + 0x2c, 0x4d, 0x04, 0xb3, 0x05, 0x47, 0x41, 0x36, 0x81, 0xd5, 0xa3, 0x38, 0xb8, 0x62, 0x34, 0xdd, + 0x85, 0x06, 0x45, 0x09, 0x49, 0x29, 0x8f, 0x81, 0xb2, 0x70, 0xea, 0x8a, 0x74, 0xea, 0x27, 0x38, + 0x4a, 0xcf, 0x1d, 0x3d, 0xe6, 0xe4, 0x64, 0x6a, 0x7f, 0xb2, 0xe4, 0x2a, 0xfb, 0xf3, 0x3d, 0xb8, + 0x36, 0xf4, 0xd2, 0xe4, 0x2a, 0xba, 0xda, 0xef, 0xf3, 0xbd, 0x9d, 0xa4, 0xe1, 0x95, 0x26, 0xff, + 0xa1, 0x04, 0xf5, 0xed, 0x38, 0x3d, 0x4a, 0xbc, 0x11, 0xb2, 0xbe, 0x03, 0x4d, 0x46, 0x98, 0x37, + 0x76, 0x53, 0x0e, 0x0a, 0xf2, 0xaa, 0x03, 0x02, 0x25, 0x09, 0x5e, 0x80, 0x56, 0x8c, 0xa8, 0x1f, + 0xa7, 0x8a, 0xa2, 0x7c, 0xb3, 0x72, 0xa7, 0xea, 0x34, 0x25, 0x4e, 0x92, 0xac, 0xc3, 0xb2, 0x18, + 0x73, 0x71, 0xe4, 0x9e, 0x22, 0x1a, 0xa1, 0x71, 0x48, 0x02, 0x24, 0x36, 0x47, 0xd5, 0xe9, 0x89, + 0xa1, 0xfd, 0xe8, 0xe3, 0x6c, 0xc0, 0x7a, 0x15, 0x7a, 0x19, 0x3d, 0xdf, 0xf1, 0x82, 0xba, 0x2a, + 0xa8, 0xbb, 0x8a, 0xfa, 0x48, 0xa1, 0xed, 0x5f, 0x42, 0xe7, 0xb3, 0x13, 0x4a, 0x18, 0x1b, 0xe3, + 0x68, 0xf4, 0xc0, 0x63, 0x1e, 0x0f, 0xcd, 0x18, 0x51, 0x4c, 0x82, 0x44, 0x69, 0xab, 0x41, 0xeb, + 0x35, 0xe8, 0x31, 0x49, 0x8b, 0x02, 0x57, 0xd3, 0x94, 0x05, 0xcd, 0x52, 0x36, 0x30, 0x54, 0xc4, + 0x2f, 0x41, 0x27, 0x27, 0xe6, 0xc1, 0xad, 0xf4, 0x6d, 0x67, 0xd8, 0xcf, 0x70, 0x88, 0xec, 0x33, + 0xe1, 0x2b, 0xb1, 0xc8, 0xd6, 0x6b, 0xd0, 0xc8, 0xfd, 0x50, 0x12, 0x3b, 0xa4, 0x23, 0x77, 0x88, + 0x76, 0xa7, 0x53, 0xcf, 0x9c, 0xf2, 0x01, 0x74, 0x59, 0xa6, 0xb8, 0x1b, 0x78, 0xcc, 0x2b, 0x6e, + 0xaa, 0xa2, 0x55, 0x4e, 0x87, 0x15, 0x60, 0xfb, 0x7d, 0x68, 0x0c, 0x71, 0x90, 0x48, 0xc1, 0x7d, + 0xa8, 0xf9, 0x29, 0xa5, 0x28, 0x62, 0xda, 0x64, 0x05, 0x5a, 0x2b, 0xb0, 0x30, 0xc6, 0x21, 0x66, + 0xca, 0x4c, 0x09, 0xd8, 0x04, 0xe0, 0x00, 0x85, 0x84, 0x5e, 0x08, 0x87, 0xad, 0xc0, 0x82, 0xb9, + 0xb8, 0x12, 0xb0, 0x9e, 0x85, 0x46, 0xe8, 0x9d, 0x67, 0x8b, 0xca, 0x47, 0xea, 0xa1, 0x77, 0x2e, + 0x95, 0xef, 0x43, 0xed, 0x91, 0x87, 0xc7, 0x7e, 0xc4, 0x94, 0x57, 0x34, 0x98, 0x0b, 0xac, 0x9a, + 0x02, 0xff, 0x5a, 0x86, 0xa6, 0x94, 0x28, 0x15, 0x5e, 0x81, 0x05, 0xdf, 0xf3, 0x4f, 0x32, 0x91, + 0x02, 0xb0, 0x6e, 0x6b, 0x45, 0xca, 0x66, 0x86, 0xcb, 0x35, 0xd5, 0xaa, 0x6d, 0x00, 0x24, 0x8f, + 0xbd, 0x58, 0xe9, 0x56, 0x99, 0x43, 0xdc, 0xe0, 0x34, 0x52, 0xdd, 0xb7, 0xa0, 0x25, 0xf7, 0x9d, + 0x9a, 0x52, 0x9d, 0x33, 0xa5, 0x29, 0xa9, 0xe4, 0xa4, 0x5b, 0xd0, 0x4e, 0x13, 0xe4, 0x9e, 0x60, + 0x44, 0x3d, 0xea, 0x9f, 0x5c, 0xf4, 0x17, 0xe4, 0x01, 0x94, 0x26, 0x68, 0x4f, 0xe3, 0xac, 0xbb, + 0xb0, 0xc0, 0x73, 0x4b, 0xd2, 0x5f, 0x14, 0x67, 0xdd, 0x73, 0x26, 0x4b, 0x61, 0xea, 0xba, 0xf8, + 0xdd, 0x89, 0x18, 0xbd, 0x70, 0x24, 0xe9, 0xe0, 0x1d, 0x80, 0x1c, 0x69, 0x2d, 0x41, 0xe5, 0x14, + 0x5d, 0xa8, 0x38, 0xe4, 0x9f, 0xdc, 0x39, 0x67, 0xde, 0x38, 0xd5, 0x5e, 0x97, 0xc0, 0x7b, 0xe5, + 0x77, 0x4a, 0xb6, 0x0f, 0xdd, 0xad, 0xf1, 0x29, 0x26, 0xc6, 0xf4, 0x15, 0x58, 0x08, 0xbd, 0x2f, + 0x08, 0xd5, 0x9e, 0x14, 0x80, 0xc0, 0xe2, 0x88, 0x50, 0xcd, 0x42, 0x00, 0x56, 0x07, 0xca, 0x24, + 0x16, 0xfe, 0x6a, 0x38, 0x65, 0x12, 0xe7, 0x82, 0xaa, 0x86, 0x20, 0xfb, 0x9f, 0x55, 0x80, 0x5c, + 0x8a, 0xe5, 0xc0, 0x00, 0x13, 0x37, 0x41, 0x94, 0x9f, 0xef, 0xee, 0xf1, 0x05, 0x43, 0x89, 0x4b, + 0x91, 0x9f, 0xd2, 0x04, 0x9f, 0xf1, 0xf5, 0xe3, 0x66, 0x5f, 0x93, 0x66, 0x4f, 0xe8, 0xe6, 0x5c, + 0xc7, 0xe4, 0x50, 0xce, 0xdb, 0xe2, 0xd3, 0x1c, 0x3d, 0xcb, 0xda, 0x87, 0x6b, 0x39, 0xcf, 0xc0, + 0x60, 0x57, 0xbe, 0x8c, 0xdd, 0x72, 0xc6, 0x2e, 0xc8, 0x59, 0xed, 0xc0, 0x32, 0x26, 0xee, 0x97, + 0x29, 0x4a, 0x0b, 0x8c, 0x2a, 0x97, 0x31, 0xea, 0x61, 0xf2, 0x43, 0x31, 0x21, 0x67, 0x33, 0x84, + 0x1b, 0x86, 0x95, 0x3c, 0xdc, 0x0d, 0x66, 0xd5, 0xcb, 0x98, 0xad, 0x66, 0x5a, 0xf1, 0x7c, 0x90, + 0x73, 0xfc, 0x08, 0x56, 0x31, 0x71, 0x1f, 0x7b, 0x98, 0x4d, 0xb2, 0x5b, 0xf8, 0x16, 0x23, 0xf9, + 0x89, 0x56, 0xe4, 0x25, 0x8d, 0x0c, 0x11, 0x1d, 0x15, 0x8c, 0x5c, 0xfc, 0x16, 0x23, 0x0f, 0xc4, + 0x84, 0x9c, 0xcd, 0x26, 0xf4, 0x30, 0x99, 0xd4, 0xa6, 0x76, 0x19, 0x93, 0x2e, 0x26, 0x45, 0x4d, + 0xb6, 0xa0, 0x97, 0x20, 0x9f, 0x11, 0x6a, 0x6e, 0x82, 0xfa, 0x65, 0x2c, 0x96, 0x14, 0x7d, 0xc6, + 0xc3, 0xfe, 0x29, 0xb4, 0xf6, 0xd2, 0x11, 0x62, 0xe3, 0xe3, 0x2c, 0x19, 0x3c, 0xb5, 0xfc, 0x63, + 0xff, 0xbb, 0x0c, 0xcd, 0xed, 0x11, 0x25, 0x69, 0x5c, 0xc8, 0xc9, 0x32, 0x48, 0x27, 0x73, 0xb2, + 0x20, 0x11, 0x39, 0x59, 0x12, 0xbf, 0x0d, 0xad, 0x50, 0x84, 0xae, 0xa2, 0x97, 0x79, 0xa8, 0x37, + 0x15, 0xd4, 0x4e, 0x33, 0x34, 0x92, 0xd9, 0x3a, 0x40, 0x8c, 0x83, 0x44, 0xcd, 0x91, 0xe9, 0xa8, + 0xab, 0xca, 0x2d, 0x9d, 0xa2, 0x9d, 0x46, 0x9c, 0x65, 0xeb, 0x37, 0xa1, 0x79, 0xcc, 0x9d, 0xa4, + 0x26, 0x14, 0x92, 0x51, 0xee, 0x3d, 0x07, 0x8e, 0xf3, 0x20, 0xdc, 0x83, 0xf6, 0x89, 0x74, 0x99, + 0x9a, 0x24, 0xf7, 0xd0, 0x2d, 0x65, 0x49, 0x6e, 0xef, 0xba, 0xe9, 0x59, 0xb9, 0x00, 0xad, 0x13, + 0x03, 0x35, 0x38, 0x84, 0xde, 0x14, 0xc9, 0x8c, 0x1c, 0x74, 0xc7, 0xcc, 0x41, 0xcd, 0xbb, 0x96, + 0x14, 0x64, 0xce, 0x34, 0xf3, 0xd2, 0x6f, 0xcb, 0xd0, 0xfa, 0x14, 0xb1, 0xc7, 0x84, 0x9e, 0x4a, + 0x7d, 0x2d, 0xa8, 0x46, 0x5e, 0x88, 0x14, 0x47, 0xf1, 0x6d, 0xdd, 0x80, 0x3a, 0x3d, 0x97, 0x09, + 0x44, 0xad, 0x67, 0x8d, 0x9e, 0x8b, 0xc4, 0x60, 0x3d, 0x0f, 0x40, 0xcf, 0xdd, 0xd8, 0xf3, 0x4f, + 0x91, 0xf2, 0x60, 0xd5, 0x69, 0xd0, 0xf3, 0xa1, 0x44, 0xf0, 0xad, 0x40, 0xcf, 0x5d, 0x44, 0x29, + 0xa1, 0x89, 0xca, 0x55, 0x75, 0x7a, 0xbe, 0x23, 0x60, 0x35, 0x37, 0xa0, 0x24, 0x8e, 0x51, 0x20, + 0x72, 0xb4, 0x98, 0xfb, 0x40, 0x22, 0xb8, 0x54, 0xa6, 0xa5, 0x2e, 0x4a, 0xa9, 0x2c, 0x97, 0xca, + 0x72, 0xa9, 0x35, 0x39, 0x93, 0x99, 0x52, 0x59, 0x26, 0xb5, 0x2e, 0xa5, 0x32, 0x43, 0x2a, 0xcb, + 0xa5, 0x36, 0xf4, 0x5c, 0x25, 0xd5, 0xfe, 0x4d, 0x09, 0x56, 0x27, 0x0b, 0x3f, 0x55, 0x9b, 0xbe, + 0x0d, 0x2d, 0x5f, 0xac, 0x57, 0x61, 0x4f, 0xf6, 0xa6, 0x56, 0xd2, 0x69, 0xfa, 0xc6, 0x36, 0xbe, + 0x07, 0xed, 0x48, 0x3a, 0x38, 0xdb, 0x9a, 0x95, 0x7c, 0x5d, 0x4c, 0xdf, 0x3b, 0xad, 0xc8, 0x80, + 0xec, 0x00, 0xac, 0xcf, 0x29, 0x66, 0xe8, 0x90, 0x51, 0xe4, 0x85, 0x4f, 0xa3, 0xba, 0xb7, 0xa0, + 0x2a, 0xaa, 0x15, 0xbe, 0x4c, 0x2d, 0x47, 0x7c, 0xdb, 0x2f, 0xc3, 0x72, 0x41, 0x8a, 0xb2, 0x75, + 0x09, 0x2a, 0x63, 0x14, 0x09, 0xee, 0x6d, 0x87, 0x7f, 0xda, 0x1e, 0xf4, 0x1c, 0xe4, 0x05, 0x4f, + 0x4f, 0x1b, 0x25, 0xa2, 0x92, 0x8b, 0xb8, 0x03, 0x96, 0x29, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x19, + 0x5a, 0x3f, 0x84, 0xde, 0xf6, 0x98, 0x24, 0xe8, 0x90, 0x05, 0x38, 0x7a, 0x1a, 0xed, 0xc8, 0x2f, + 0x60, 0xf9, 0x33, 0x76, 0xf1, 0x39, 0x67, 0x96, 0xe0, 0xaf, 0xd0, 0x53, 0xb2, 0x8f, 0x92, 0xc7, + 0xda, 0x3e, 0x4a, 0x1e, 0xf3, 0xe6, 0xc6, 0x27, 0xe3, 0x34, 0x8c, 0x44, 0x28, 0xb4, 0x1d, 0x05, + 0xd9, 0x5b, 0xd0, 0x92, 0x35, 0xf4, 0x01, 0x09, 0xd2, 0x31, 0x9a, 0x19, 0x83, 0x6b, 0x00, 0xb1, + 0x47, 0xbd, 0x10, 0x31, 0x44, 0xe5, 0x1e, 0x6a, 0x38, 0x06, 0xc6, 0xfe, 0x5d, 0x19, 0x56, 0xe4, + 0x7d, 0xc3, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x03, 0xa8, 0x9f, 0x90, 0x84, 0x19, 0x0c, 0x33, 0x98, + 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0x16, 0x2e, 0x01, 0x2a, 0x97, 0x5f, 0x02, 0x4c, 0xb5, + 0xf9, 0xd5, 0xe9, 0x36, 0x9f, 0x47, 0x9b, 0x26, 0xc2, 0x32, 0xc6, 0x1b, 0x4e, 0x43, 0x61, 0xf6, + 0x03, 0xeb, 0x36, 0x74, 0x47, 0x5c, 0x4b, 0xf7, 0x84, 0x90, 0x53, 0x37, 0xf6, 0xd8, 0x89, 0x08, + 0xf5, 0x86, 0xd3, 0x16, 0xe8, 0x3d, 0x42, 0x4e, 0x87, 0x1e, 0x3b, 0xb1, 0xde, 0x85, 0x8e, 0x2a, + 0x03, 0x43, 0xe1, 0xa2, 0x44, 0x1d, 0x7e, 0x2a, 0x8a, 0x4c, 0xef, 0x39, 0xed, 0x53, 0x03, 0x4a, + 0xec, 0xeb, 0x70, 0xed, 0x01, 0x4a, 0x18, 0x25, 0x17, 0x45, 0xc7, 0xd8, 0xdf, 0x07, 0xd8, 0x8f, + 0x18, 0xa2, 0x8f, 0x3c, 0x1f, 0x25, 0xd6, 0x1b, 0x26, 0xa4, 0x8a, 0xa3, 0xa5, 0x75, 0x79, 0xdd, + 0x93, 0x0d, 0x38, 0x06, 0x8d, 0xbd, 0x0e, 0x8b, 0x0e, 0x49, 0x79, 0x3a, 0x7a, 0x51, 0x7f, 0xa9, + 0x79, 0x2d, 0x35, 0x4f, 0x20, 0x1d, 0x35, 0x66, 0xef, 0xe9, 0x16, 0x36, 0x67, 0xa7, 0x96, 0x68, + 0x1d, 0x1a, 0x58, 0xe3, 0x54, 0x56, 0x99, 0x16, 0x9d, 0x93, 0xd8, 0xef, 0xc3, 0xb2, 0xe4, 0x24, + 0x39, 0x6b, 0x36, 0x2f, 0xc2, 0x22, 0xd5, 0x6a, 0x94, 0xf2, 0x7b, 0x1e, 0x45, 0xa4, 0xc6, 0xb8, + 0x3f, 0x3e, 0xc1, 0x09, 0xcb, 0x0d, 0xd1, 0xfe, 0x58, 0x86, 0x1e, 0x1f, 0x28, 0xf0, 0xb4, 0x3f, + 0x84, 0xd6, 0xa6, 0x33, 0xfc, 0x14, 0xe1, 0xd1, 0xc9, 0x31, 0xcf, 0x9e, 0xdf, 0x2b, 0xc2, 0xca, + 0x60, 0x4b, 0x69, 0x6b, 0x0c, 0x39, 0x05, 0x3a, 0xfb, 0x23, 0x58, 0xdd, 0x0c, 0x02, 0x13, 0xa5, + 0xb5, 0x7e, 0x03, 0x1a, 0x91, 0xc1, 0xce, 0x38, 0xb3, 0x0a, 0xd4, 0x39, 0x91, 0xfd, 0x33, 0x58, + 0x7e, 0x18, 0x8d, 0x71, 0x84, 0xb6, 0x87, 0x47, 0x07, 0x28, 0xcb, 0x45, 0x16, 0x54, 0x79, 0xcd, + 0x26, 0x78, 0xd4, 0x1d, 0xf1, 0xcd, 0x83, 0x33, 0x3a, 0x76, 0xfd, 0x38, 0x4d, 0xd4, 0x65, 0xcf, + 0x62, 0x74, 0xbc, 0x1d, 0xa7, 0x09, 0x3f, 0x5c, 0x78, 0x71, 0x41, 0xa2, 0xf1, 0x85, 0x88, 0xd0, + 0xba, 0x53, 0xf3, 0xe3, 0xf4, 0x61, 0x34, 0xbe, 0xb0, 0xbf, 0x2b, 0x3a, 0x70, 0x84, 0x02, 0xc7, + 0x8b, 0x02, 0x12, 0x3e, 0x40, 0x67, 0x86, 0x84, 0xac, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x12, 0xb4, + 0x36, 0x47, 0x28, 0x62, 0x0f, 0x10, 0xf3, 0xf0, 0x58, 0x74, 0x74, 0x67, 0x88, 0x26, 0x98, 0x44, + 0x2a, 0xdc, 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x61, 0xe6, 0x06, 0x1e, 0x0a, 0x49, 0x24, 0xb8, 0xd4, + 0x1d, 0xe0, 0xa8, 0x07, 0x02, 0x63, 0xbd, 0x0c, 0x5d, 0x79, 0x19, 0xe7, 0x9e, 0x78, 0x51, 0x30, + 0xe6, 0x81, 0x5e, 0x11, 0xa1, 0xd9, 0x91, 0xe8, 0x3d, 0x85, 0xb5, 0x5e, 0x81, 0x25, 0x15, 0x86, + 0x39, 0x65, 0x55, 0x50, 0x76, 0x15, 0xbe, 0x40, 0x9a, 0xc6, 0x31, 0xa1, 0x2c, 0x71, 0x13, 0xe4, + 0xfb, 0x24, 0x8c, 0x55, 0x3b, 0xd4, 0xd5, 0xf8, 0x43, 0x89, 0xb6, 0x47, 0xb0, 0xbc, 0xcb, 0xed, + 0x54, 0x96, 0xe4, 0xdb, 0xaa, 0x13, 0xa2, 0xd0, 0x3d, 0x1e, 0x13, 0xff, 0xd4, 0xe5, 0xc9, 0x51, + 0x79, 0x98, 0x17, 0x5c, 0x5b, 0x1c, 0x79, 0x88, 0xbf, 0x12, 0x9d, 0x3f, 0xa7, 0x3a, 0x21, 0x2c, + 0x1e, 0xa7, 0x23, 0x37, 0xa6, 0xe4, 0x18, 0x29, 0x13, 0xbb, 0x21, 0x0a, 0xf7, 0x24, 0x7e, 0xc8, + 0xd1, 0xf6, 0x9f, 0x4b, 0xb0, 0x52, 0x94, 0xa4, 0x52, 0xfd, 0x06, 0xac, 0x14, 0x45, 0xa9, 0xe3, + 0x5f, 0x96, 0x97, 0x3d, 0x53, 0xa0, 0x2c, 0x04, 0xee, 0x41, 0x5b, 0x5c, 0xdd, 0xba, 0x81, 0xe4, + 0x54, 0x2c, 0x7a, 0xcc, 0x75, 0x71, 0x5a, 0x9e, 0xb9, 0x4a, 0xef, 0xc2, 0x0d, 0x65, 0xbe, 0x3b, + 0xad, 0xb6, 0xdc, 0x10, 0xab, 0x8a, 0xe0, 0x60, 0x42, 0xfb, 0x4f, 0xa0, 0x9f, 0xa3, 0xb6, 0x2e, + 0x04, 0x32, 0xdf, 0xcc, 0xcb, 0x13, 0xc6, 0x6e, 0x06, 0x01, 0x15, 0x51, 0x52, 0x75, 0x66, 0x0d, + 0xd9, 0xf7, 0xe1, 0xfa, 0x21, 0x62, 0xd2, 0x1b, 0x1e, 0x53, 0x9d, 0x88, 0x64, 0xb6, 0x04, 0x95, + 0x43, 0xe4, 0x0b, 0xe3, 0x2b, 0x0e, 0xff, 0xe4, 0x1b, 0xf0, 0x28, 0x41, 0xbe, 0xb0, 0xb2, 0xe2, + 0x88, 0x6f, 0xfb, 0x4f, 0x25, 0xa8, 0xa9, 0xe4, 0xcc, 0x0f, 0x98, 0x80, 0xe2, 0x33, 0x44, 0xd5, + 0xd6, 0x53, 0x90, 0xf5, 0x12, 0x74, 0xe4, 0x97, 0x4b, 0x62, 0x86, 0x49, 0x96, 0xf2, 0xdb, 0x12, + 0xfb, 0x50, 0x22, 0xc5, 0xe5, 0x9b, 0xb8, 0xfe, 0x52, 0x9d, 0xa6, 0x82, 0x38, 0xfe, 0x51, 0xc2, + 0x23, 0x5c, 0xa4, 0xf8, 0x86, 0xa3, 0x20, 0xbe, 0xd5, 0x35, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, + 0xab, 0x87, 0x24, 0x8d, 0x98, 0x1b, 0x13, 0x1c, 0x31, 0x95, 0xd3, 0x41, 0xa0, 0x86, 0x1c, 0x63, + 0xff, 0xba, 0x04, 0x8b, 0xf2, 0x02, 0x9a, 0xf7, 0xb6, 0xd9, 0xc9, 0x5a, 0xc6, 0xa2, 0x4a, 0x11, + 0xb2, 0xe4, 0x69, 0x2a, 0xbe, 0x79, 0x1c, 0x9f, 0x85, 0xf2, 0x7c, 0x50, 0xaa, 0x9d, 0x85, 0xe2, + 0x60, 0x78, 0x09, 0x3a, 0xf9, 0x01, 0x2d, 0xc6, 0xa5, 0x8a, 0xed, 0x0c, 0x2b, 0xc8, 0xe6, 0x6a, + 0x6a, 0xff, 0x98, 0xb7, 0xf4, 0xd9, 0xe5, 0xeb, 0x12, 0x54, 0xd2, 0x4c, 0x19, 0xfe, 0xc9, 0x31, + 0xa3, 0xec, 0x68, 0xe7, 0x9f, 0xd6, 0x6d, 0xe8, 0x78, 0x41, 0x80, 0xf9, 0x74, 0x6f, 0xbc, 0x8b, + 0x83, 0x2c, 0x48, 0x8b, 0x58, 0xfb, 0x6f, 0x25, 0xe8, 0x6e, 0x93, 0xf8, 0xe2, 0x43, 0x3c, 0x46, + 0x46, 0x06, 0x11, 0x4a, 0xaa, 0x93, 0x9d, 0x7f, 0xf3, 0x6a, 0xf5, 0x11, 0x1e, 0x23, 0x19, 0x5a, + 0x72, 0x65, 0xeb, 0x1c, 0x21, 0xc2, 0x4a, 0x0f, 0x66, 0xd7, 0x6e, 0x6d, 0x39, 0x78, 0x40, 0x02, + 0x51, 0x97, 0x07, 0x98, 0xba, 0xd9, 0x25, 0x5b, 0xdb, 0xa9, 0x05, 0x98, 0x8a, 0x21, 0x65, 0xc8, + 0x82, 0xb8, 0x44, 0x35, 0x0d, 0x59, 0x94, 0x18, 0x6e, 0xc8, 0x2a, 0x2c, 0x92, 0x47, 0x8f, 0x12, + 0xc4, 0x44, 0x05, 0x5d, 0x71, 0x14, 0x94, 0xa5, 0xb9, 0xba, 0x91, 0xe6, 0x56, 0xc0, 0xda, 0x45, + 0xec, 0xe1, 0xc3, 0x83, 0x9d, 0x33, 0x14, 0x31, 0x7d, 0x3a, 0xbc, 0x0e, 0x75, 0x8d, 0xfa, 0x6f, + 0xae, 0x27, 0x5f, 0x85, 0xce, 0x66, 0x10, 0x1c, 0x3e, 0xf6, 0x62, 0xed, 0x8f, 0x3e, 0xd4, 0x86, + 0xdb, 0xfb, 0x43, 0xe9, 0x92, 0x0a, 0x37, 0x40, 0x81, 0xfc, 0x34, 0xda, 0x45, 0xec, 0x00, 0x31, + 0x8a, 0xfd, 0xec, 0x34, 0xba, 0x05, 0x35, 0x85, 0xe1, 0x33, 0x43, 0xf9, 0xa9, 0xd3, 0xac, 0x02, + 0xed, 0x1f, 0x80, 0xf5, 0x23, 0x5e, 0x57, 0x21, 0x59, 0x54, 0x2b, 0x49, 0xaf, 0x42, 0xef, 0x4c, + 0x60, 0x5d, 0x59, 0x70, 0x18, 0xcb, 0xd0, 0x95, 0x03, 0x22, 0x06, 0x85, 0xec, 0x23, 0x58, 0x96, + 0x65, 0xa0, 0xe4, 0x73, 0x05, 0x16, 0xdc, 0x87, 0xd9, 0x7a, 0x56, 0x1d, 0xf1, 0x7d, 0xf7, 0x2f, + 0x3d, 0x75, 0x54, 0xa8, 0x5b, 0x07, 0x6b, 0x17, 0xba, 0x13, 0x4f, 0x44, 0x96, 0xba, 0x86, 0x9a, + 0xfd, 0x72, 0x34, 0x58, 0x5d, 0x97, 0x4f, 0x4e, 0xeb, 0xfa, 0xc9, 0x69, 0x7d, 0x27, 0x8c, 0xd9, + 0x85, 0xb5, 0x03, 0x9d, 0xe2, 0x63, 0x8a, 0xf5, 0xac, 0xae, 0xda, 0x66, 0x3c, 0xb1, 0xcc, 0x65, + 0xb3, 0x0b, 0xdd, 0x89, 0x77, 0x15, 0xad, 0xcf, 0xec, 0xe7, 0x96, 0xb9, 0x8c, 0xee, 0x43, 0xd3, + 0x78, 0x48, 0xb1, 0xfa, 0x92, 0xc9, 0xf4, 0xdb, 0xca, 0x5c, 0x06, 0xdb, 0xd0, 0x2e, 0xbc, 0x6d, + 0x58, 0x03, 0x65, 0xcf, 0x8c, 0x07, 0x8f, 0xb9, 0x4c, 0xb6, 0xa0, 0x69, 0x3c, 0x31, 0x68, 0x2d, + 0xa6, 0xdf, 0x31, 0x06, 0x37, 0x66, 0x8c, 0xa8, 0x13, 0x69, 0x17, 0xba, 0x13, 0xef, 0x0e, 0xda, + 0x25, 0xb3, 0x9f, 0x23, 0xe6, 0x2a, 0xf3, 0xb1, 0x58, 0x22, 0xa3, 0xad, 0x34, 0x96, 0x68, 0xfa, + 0x95, 0x61, 0xf0, 0xdc, 0xec, 0x41, 0xa5, 0xd5, 0x0e, 0x74, 0x8a, 0x0f, 0x0c, 0x9a, 0xd9, 0xcc, + 0x67, 0x87, 0xcb, 0xd7, 0xbb, 0xf0, 0xd6, 0x90, 0xaf, 0xf7, 0xac, 0x27, 0x88, 0xb9, 0x8c, 0x36, + 0x01, 0x54, 0x13, 0x19, 0xe0, 0x28, 0x73, 0xf4, 0x54, 0xf3, 0x9a, 0x39, 0x7a, 0x46, 0xc3, 0x79, + 0x1f, 0x40, 0xf6, 0x7e, 0x01, 0x49, 0x99, 0x75, 0x5d, 0xab, 0x31, 0xd1, 0x70, 0x0e, 0xfa, 0xd3, + 0x03, 0x53, 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x01, 0x40, 0xde, 0x53, 0x6a, 0x06, 0x53, 0x5d, + 0xe6, 0x25, 0x3e, 0x68, 0x99, 0x1d, 0xa4, 0xa5, 0x6c, 0x9d, 0xd1, 0x55, 0x5e, 0xc2, 0xa2, 0x3b, + 0xd1, 0x21, 0x14, 0x37, 0xdb, 0x64, 0xe3, 0x30, 0x98, 0xea, 0x12, 0xac, 0x7b, 0xd0, 0x32, 0x5b, + 0x03, 0xad, 0xc5, 0x8c, 0x76, 0x61, 0x50, 0x68, 0x0f, 0xac, 0xfb, 0xd0, 0x29, 0xb6, 0x05, 0x7a, + 0x4b, 0xcd, 0x6c, 0x16, 0x06, 0xea, 0xd2, 0xcb, 0x20, 0x7f, 0x0b, 0x20, 0x6f, 0x1f, 0xb4, 0xfb, + 0xa6, 0x1a, 0x8a, 0x09, 0xa9, 0xbb, 0xd0, 0x9d, 0x68, 0x0b, 0xb4, 0xc5, 0xb3, 0xbb, 0x85, 0xb9, + 0xae, 0x7b, 0x1b, 0x20, 0x3f, 0x2e, 0xb4, 0xf4, 0xa9, 0x03, 0x64, 0xd0, 0xd6, 0x17, 0x82, 0x92, + 0x6e, 0x1b, 0xda, 0x85, 0x9e, 0x59, 0xa7, 0x99, 0x59, 0x8d, 0xf4, 0x65, 0xc9, 0xb7, 0xd8, 0x60, + 0x6a, 0xcf, 0xcd, 0x6c, 0x3b, 0x2f, 0xdb, 0x3f, 0x66, 0x57, 0xa3, 0x57, 0x6e, 0x46, 0xa7, 0xf3, + 0x2d, 0xf1, 0x6c, 0x76, 0x2e, 0x46, 0x3c, 0xcf, 0x68, 0x68, 0xe6, 0x32, 0xda, 0x83, 0xee, 0xae, + 0x2e, 0x4a, 0x55, 0xc1, 0xac, 0xd4, 0x99, 0xd1, 0x20, 0x0c, 0x06, 0xb3, 0x86, 0x54, 0x50, 0x7d, + 0x0c, 0xbd, 0xa9, 0x62, 0xd9, 0x5a, 0xcb, 0xae, 0x65, 0x67, 0x56, 0xd1, 0x73, 0xd5, 0xda, 0x87, + 0xa5, 0xc9, 0x5a, 0xd9, 0x7a, 0x5e, 0x25, 0xca, 0xd9, 0x35, 0xf4, 0x5c, 0x56, 0xef, 0x42, 0x5d, + 0xd7, 0x66, 0x96, 0xba, 0xfe, 0x9e, 0xa8, 0xd5, 0xe6, 0x4e, 0xbd, 0x07, 0x4d, 0xa3, 0x14, 0xd2, + 0xd9, 0x6e, 0xba, 0x3a, 0x1a, 0xa8, 0xdb, 0xea, 0x8c, 0xf2, 0x1e, 0xd4, 0x54, 0xf9, 0x63, 0xad, + 0x64, 0x9b, 0xdc, 0xa8, 0x86, 0x2e, 0xdb, 0x61, 0xbb, 0x88, 0x19, 0x45, 0x8d, 0x16, 0x3a, 0x5d, + 0xe7, 0xe8, 0x14, 0x5b, 0x18, 0x51, 0x6b, 0xb1, 0x09, 0x2d, 0xb3, 0xac, 0xd1, 0x4b, 0x3a, 0xa3, + 0xd4, 0x99, 0xa7, 0xc9, 0xd6, 0xf9, 0xd7, 0xdf, 0xac, 0x3d, 0xf3, 0x8f, 0x6f, 0xd6, 0x9e, 0xf9, + 0xd5, 0x93, 0xb5, 0xd2, 0xd7, 0x4f, 0xd6, 0x4a, 0x7f, 0x7f, 0xb2, 0x56, 0xfa, 0xd7, 0x93, 0xb5, + 0xd2, 0x4f, 0x7e, 0xfe, 0x3f, 0xfe, 0x0f, 0x87, 0xa6, 0x11, 0xc3, 0x21, 0xda, 0x38, 0xc3, 0x94, + 0x19, 0x43, 0xf1, 0xe9, 0x48, 0xfe, 0x19, 0xc7, 0xf8, 0x8f, 0x0e, 0xd7, 0xf2, 0x78, 0x51, 0xc0, + 0x6f, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x4e, 0xa6, 0xdc, 0xf0, 0x23, 0x00, 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -5446,6 +5533,79 @@ func (m *Metrics) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *VolumeStatsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeStatsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeStatsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.VolumeGuestPath) > 0 { + i -= len(m.VolumeGuestPath) + copy(dAtA[i:], m.VolumeGuestPath) + i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ResizeVolumeRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeVolumeRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ResizeVolumeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Size_ != 0 { + i = encodeVarintAgent(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x10 + } + if len(m.VolumeGuestPath) > 0 { + i -= len(m.VolumeGuestPath) + copy(dAtA[i:], m.VolumeGuestPath) + i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { offset -= sovAgent(v) base := offset @@ -6737,6 +6897,41 @@ func (m *Metrics) Size() (n int) { return n } +func (m *VolumeStatsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.VolumeGuestPath) + if l > 0 { + n += 1 + l + sovAgent(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ResizeVolumeRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.VolumeGuestPath) + if l > 0 { + n += 1 + l + sovAgent(uint64(l)) + } + if m.Size_ != 0 { + n += 1 + sovAgent(uint64(m.Size_)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovAgent(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -7551,6 +7746,29 @@ func (this *Metrics) String() string { }, "") return s } +func (this *VolumeStatsRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeStatsRequest{`, + `VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *ResizeVolumeRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ResizeVolumeRequest{`, + `VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`, + `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func valueToStringAgent(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -7592,6 +7810,8 @@ type AgentServiceService interface { CopyFile(ctx context.Context, req *CopyFileRequest) (*types.Empty, error) GetOOMEvent(ctx context.Context, req *GetOOMEventRequest) (*OOMEvent, error) AddSwap(ctx context.Context, req *AddSwapRequest) (*types.Empty, error) + GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error) + ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error) } func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc AgentServiceService) { @@ -7813,6 +8033,20 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag } return svc.AddSwap(ctx, &req) }, + "GetVolumeStats": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req VolumeStatsRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.GetVolumeStats(ctx, &req) + }, + "ResizeVolume": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req ResizeVolumeRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.ResizeVolume(ctx, &req) + }, }) } @@ -8073,6 +8307,22 @@ func (c *agentServiceClient) AddSwap(ctx context.Context, req *AddSwapRequest) ( } return &resp, nil } + +func (c *agentServiceClient) GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error) { + var resp VolumeStatsResponse + if err := c.client.Call(ctx, "grpc.AgentService", "GetVolumeStats", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *agentServiceClient) ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error) { + var resp types.Empty + if err := c.client.Call(ctx, "grpc.AgentService", "ResizeVolume", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -15399,6 +15649,191 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { } return nil } +func (m *VolumeStatsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeStatsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeStatsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeGuestPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ResizeVolumeRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeVolumeRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeVolumeRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeGuestPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipAgent(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go new file mode 100644 index 000000000..6ae4f148e --- /dev/null +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go @@ -0,0 +1,1164 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto + +package grpc + +import ( + bytes "bytes" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type VolumeUsage_Unit int32 + +const ( + VolumeUsage_UNKNOWN VolumeUsage_Unit = 0 + VolumeUsage_BYTES VolumeUsage_Unit = 1 + VolumeUsage_INODES VolumeUsage_Unit = 2 +) + +var VolumeUsage_Unit_name = map[int32]string{ + 0: "UNKNOWN", + 1: "BYTES", + 2: "INODES", +} + +var VolumeUsage_Unit_value = map[string]int32{ + "UNKNOWN": 0, + "BYTES": 1, + "INODES": 2, +} + +func (x VolumeUsage_Unit) String() string { + return proto.EnumName(VolumeUsage_Unit_name, int32(x)) +} + +func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{1, 0} +} + +// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto) +type VolumeStatsResponse struct { + // This field is OPTIONAL. + Usage []*VolumeUsage `protobuf:"bytes,1,rep,name=usage,proto3" json:"usage,omitempty"` + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the VOLUME_CONDITION node + // capability is supported. + VolumeCondition *VolumeCondition `protobuf:"bytes,2,opt,name=volume_condition,json=volumeCondition,proto3" json:"volume_condition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeStatsResponse) Reset() { *m = VolumeStatsResponse{} } +func (*VolumeStatsResponse) ProtoMessage() {} +func (*VolumeStatsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{0} +} +func (m *VolumeStatsResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeStatsResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeStatsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeStatsResponse.Merge(m, src) +} +func (m *VolumeStatsResponse) XXX_Size() int { + return m.Size() +} +func (m *VolumeStatsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeStatsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeStatsResponse proto.InternalMessageInfo + +type VolumeUsage struct { + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Available uint64 `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + Total uint64 `protobuf:"varint,2,opt,name=total,proto3" json:"total,omitempty"` + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Used uint64 `protobuf:"varint,3,opt,name=used,proto3" json:"used,omitempty"` + // Units by which values are measured. This field is REQUIRED. + Unit VolumeUsage_Unit `protobuf:"varint,4,opt,name=unit,proto3,enum=grpc.VolumeUsage_Unit" json:"unit,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeUsage) Reset() { *m = VolumeUsage{} } +func (*VolumeUsage) ProtoMessage() {} +func (*VolumeUsage) Descriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{1} +} +func (m *VolumeUsage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeUsage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeUsage) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeUsage.Merge(m, src) +} +func (m *VolumeUsage) XXX_Size() int { + return m.Size() +} +func (m *VolumeUsage) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeUsage.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeUsage proto.InternalMessageInfo + +// VolumeCondition represents the current condition of a volume. +type VolumeCondition struct { + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + // This field is REQUIRED. + Abnormal bool `protobuf:"varint,1,opt,name=abnormal,proto3" json:"abnormal,omitempty"` + // The message describing the condition of the volume. + // This field is REQUIRED. + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCondition) Reset() { *m = VolumeCondition{} } +func (*VolumeCondition) ProtoMessage() {} +func (*VolumeCondition) Descriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{2} +} +func (m *VolumeCondition) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeCondition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeCondition.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeCondition) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCondition.Merge(m, src) +} +func (m *VolumeCondition) XXX_Size() int { + return m.Size() +} +func (m *VolumeCondition) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCondition.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCondition proto.InternalMessageInfo + +func init() { + proto.RegisterEnum("grpc.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) + proto.RegisterType((*VolumeStatsResponse)(nil), "grpc.VolumeStatsResponse") + proto.RegisterType((*VolumeUsage)(nil), "grpc.VolumeUsage") + proto.RegisterType((*VolumeCondition)(nil), "grpc.VolumeCondition") +} + +func init() { + proto.RegisterFile("github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto", fileDescriptor_5658ecc8e79d3214) +} + +var fileDescriptor_5658ecc8e79d3214 = []byte{ + // 421 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0xbf, 0x6e, 0xd4, 0x40, + 0x10, 0xc6, 0x6f, 0x12, 0x5f, 0x92, 0x9b, 0x93, 0xc8, 0xb1, 0xfc, 0x91, 0x15, 0xa1, 0xd5, 0xc9, + 0x0d, 0xa7, 0x48, 0xd8, 0xd2, 0xf1, 0x02, 0x28, 0x10, 0x21, 0x84, 0xe4, 0x48, 0x7b, 0x1c, 0x08, + 0x0a, 0xd0, 0xda, 0x59, 0xcc, 0x2a, 0xf6, 0xae, 0xe5, 0x5d, 0xbb, 0x4e, 0xc9, 0xa3, 0xd0, 0xd0, + 0x53, 0x52, 0xa6, 0xa4, 0xa4, 0xe4, 0xfc, 0x14, 0x94, 0xc8, 0x6b, 0x02, 0x86, 0x54, 0xe9, 0xbe, + 0xdf, 0xf7, 0xcd, 0xfe, 0x99, 0xd1, 0x60, 0x9c, 0x49, 0xfb, 0xa1, 0x4e, 0xc2, 0x54, 0x17, 0xd1, + 0x19, 0xb7, 0xfc, 0x41, 0xaa, 0x95, 0xe5, 0x52, 0x89, 0xca, 0x5c, 0x61, 0x53, 0xa5, 0x51, 0x2e, + 0x13, 0x13, 0x95, 0x95, 0xb6, 0x3a, 0xd5, 0xf9, 0x6f, 0x65, 0xa2, 0xd4, 0xc8, 0xd0, 0x49, 0xe2, + 0x65, 0x55, 0x99, 0x1e, 0x04, 0x99, 0xce, 0x74, 0x1f, 0x26, 0xf5, 0xfb, 0xa8, 0x23, 0x07, 0x4e, + 0xf5, 0x95, 0xc1, 0x39, 0xe0, 0xad, 0x97, 0x3a, 0xaf, 0x0b, 0xb1, 0xb2, 0xdc, 0x1a, 0x26, 0x4c, + 0xa9, 0x95, 0x11, 0xe4, 0x3e, 0x8e, 0x6b, 0xc3, 0x33, 0xe1, 0xc3, 0x7c, 0x7b, 0x31, 0x5d, 0xde, + 0x0c, 0xbb, 0x1b, 0xc3, 0xbe, 0x72, 0xdd, 0x05, 0xac, 0xcf, 0xc9, 0x23, 0x9c, 0x35, 0xce, 0x7d, + 0x97, 0x6a, 0x75, 0x2a, 0xad, 0xd4, 0xca, 0xdf, 0x9a, 0xc3, 0x62, 0xba, 0xbc, 0x33, 0x3c, 0xf3, + 0xf8, 0x32, 0x64, 0xfb, 0xcd, 0xbf, 0x46, 0xf0, 0x19, 0x70, 0x3a, 0xb8, 0x98, 0xdc, 0xc3, 0x09, + 0x6f, 0xb8, 0xcc, 0x79, 0x92, 0x77, 0xcf, 0xc3, 0xc2, 0x63, 0x7f, 0x0d, 0x72, 0x1b, 0xc7, 0x56, + 0x5b, 0x9e, 0xbb, 0x47, 0x3c, 0xd6, 0x03, 0x21, 0xe8, 0xd5, 0x46, 0x9c, 0xfa, 0xdb, 0xce, 0x74, + 0x9a, 0x1c, 0xa2, 0x57, 0x2b, 0x69, 0x7d, 0x6f, 0x0e, 0x8b, 0x1b, 0xcb, 0xbb, 0x57, 0x3a, 0x08, + 0xd7, 0x4a, 0x5a, 0xe6, 0x6a, 0x82, 0x43, 0xf4, 0x3a, 0x22, 0x53, 0xdc, 0x5d, 0xc7, 0xcf, 0xe3, + 0x93, 0x57, 0xf1, 0x6c, 0x44, 0x26, 0x38, 0x3e, 0x7a, 0xfd, 0xe2, 0x78, 0x35, 0x03, 0x82, 0xb8, + 0xf3, 0x2c, 0x3e, 0x79, 0x72, 0xbc, 0x9a, 0x6d, 0x05, 0x4f, 0x71, 0xff, 0xbf, 0x9e, 0xc8, 0x01, + 0xee, 0xf1, 0x44, 0xe9, 0xaa, 0xe0, 0xb9, 0xfb, 0xf1, 0x1e, 0xfb, 0xc3, 0xc4, 0xc7, 0xdd, 0x42, + 0x18, 0x37, 0xcb, 0xee, 0xcb, 0x13, 0x76, 0x89, 0x47, 0x1f, 0xe1, 0x62, 0x43, 0x47, 0xdf, 0x37, + 0x74, 0xf4, 0x73, 0x43, 0xe1, 0xbc, 0xa5, 0xf0, 0xa9, 0xa5, 0xf0, 0xa5, 0xa5, 0xf0, 0xb5, 0xa5, + 0x70, 0xd1, 0x52, 0xf8, 0xd6, 0x52, 0xf8, 0xd1, 0x52, 0x78, 0xf3, 0xf6, 0x9a, 0x0b, 0x52, 0xd5, + 0xca, 0xca, 0x42, 0x44, 0x8d, 0xac, 0xec, 0x20, 0x2a, 0xcf, 0xb2, 0x88, 0x67, 0x42, 0xd9, 0xc1, + 0xf2, 0x74, 0x63, 0x49, 0x76, 0x1c, 0x3f, 0xfc, 0x15, 0x00, 0x00, 0xff, 0xff, 0xce, 0x52, 0xbe, + 0x85, 0x89, 0x02, 0x00, 0x00, +} + +func (this *VolumeStatsResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*VolumeStatsResponse) + if !ok { + that2, ok := that.(VolumeStatsResponse) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if len(this.Usage) != len(that1.Usage) { + return false + } + for i := range this.Usage { + if !this.Usage[i].Equal(that1.Usage[i]) { + return false + } + } + if !this.VolumeCondition.Equal(that1.VolumeCondition) { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *VolumeUsage) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*VolumeUsage) + if !ok { + that2, ok := that.(VolumeUsage) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Available != that1.Available { + return false + } + if this.Total != that1.Total { + return false + } + if this.Used != that1.Used { + return false + } + if this.Unit != that1.Unit { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *VolumeCondition) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*VolumeCondition) + if !ok { + that2, ok := that.(VolumeCondition) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Abnormal != that1.Abnormal { + return false + } + if this.Message != that1.Message { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (m *VolumeStatsResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeStatsResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeStatsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.VolumeCondition != nil { + { + size, err := m.VolumeCondition.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCsi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if len(m.Usage) > 0 { + for iNdEx := len(m.Usage) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Usage[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCsi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *VolumeUsage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeUsage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeUsage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Unit != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Unit)) + i-- + dAtA[i] = 0x20 + } + if m.Used != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Used)) + i-- + dAtA[i] = 0x18 + } + if m.Total != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Total)) + i-- + dAtA[i] = 0x10 + } + if m.Available != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Available)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *VolumeCondition) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeCondition) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeCondition) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Message) > 0 { + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintCsi(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0x12 + } + if m.Abnormal { + i-- + if m.Abnormal { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintCsi(dAtA []byte, offset int, v uint64) int { + offset -= sovCsi(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func NewPopulatedVolumeStatsResponse(r randyCsi, easy bool) *VolumeStatsResponse { + this := &VolumeStatsResponse{} + if r.Intn(5) != 0 { + v1 := r.Intn(5) + this.Usage = make([]*VolumeUsage, v1) + for i := 0; i < v1; i++ { + this.Usage[i] = NewPopulatedVolumeUsage(r, easy) + } + } + if r.Intn(5) != 0 { + this.VolumeCondition = NewPopulatedVolumeCondition(r, easy) + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedCsi(r, 3) + } + return this +} + +func NewPopulatedVolumeUsage(r randyCsi, easy bool) *VolumeUsage { + this := &VolumeUsage{} + this.Available = uint64(uint64(r.Uint32())) + this.Total = uint64(uint64(r.Uint32())) + this.Used = uint64(uint64(r.Uint32())) + this.Unit = VolumeUsage_Unit([]int32{0, 1, 2}[r.Intn(3)]) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedCsi(r, 5) + } + return this +} + +func NewPopulatedVolumeCondition(r randyCsi, easy bool) *VolumeCondition { + this := &VolumeCondition{} + this.Abnormal = bool(bool(r.Intn(2) == 0)) + this.Message = string(randStringCsi(r)) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedCsi(r, 3) + } + return this +} + +type randyCsi interface { + Float32() float32 + Float64() float64 + Int63() int64 + Int31() int32 + Uint32() uint32 + Intn(n int) int +} + +func randUTF8RuneCsi(r randyCsi) rune { + ru := r.Intn(62) + if ru < 10 { + return rune(ru + 48) + } else if ru < 36 { + return rune(ru + 55) + } + return rune(ru + 61) +} +func randStringCsi(r randyCsi) string { + v2 := r.Intn(100) + tmps := make([]rune, v2) + for i := 0; i < v2; i++ { + tmps[i] = randUTF8RuneCsi(r) + } + return string(tmps) +} +func randUnrecognizedCsi(r randyCsi, maxFieldNumber int) (dAtA []byte) { + l := r.Intn(5) + for i := 0; i < l; i++ { + wire := r.Intn(4) + if wire == 3 { + wire = 5 + } + fieldNumber := maxFieldNumber + r.Intn(100) + dAtA = randFieldCsi(dAtA, r, fieldNumber, wire) + } + return dAtA +} +func randFieldCsi(dAtA []byte, r randyCsi, fieldNumber int, wire int) []byte { + key := uint32(fieldNumber)<<3 | uint32(wire) + switch wire { + case 0: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + v3 := r.Int63() + if r.Intn(2) == 0 { + v3 *= -1 + } + dAtA = encodeVarintPopulateCsi(dAtA, uint64(v3)) + case 1: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + case 2: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + ll := r.Intn(100) + dAtA = encodeVarintPopulateCsi(dAtA, uint64(ll)) + for j := 0; j < ll; j++ { + dAtA = append(dAtA, byte(r.Intn(256))) + } + default: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + } + return dAtA +} +func encodeVarintPopulateCsi(dAtA []byte, v uint64) []byte { + for v >= 1<<7 { + dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80)) + v >>= 7 + } + dAtA = append(dAtA, uint8(v)) + return dAtA +} +func (m *VolumeStatsResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Usage) > 0 { + for _, e := range m.Usage { + l = e.Size() + n += 1 + l + sovCsi(uint64(l)) + } + } + if m.VolumeCondition != nil { + l = m.VolumeCondition.Size() + n += 1 + l + sovCsi(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *VolumeUsage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Available != 0 { + n += 1 + sovCsi(uint64(m.Available)) + } + if m.Total != 0 { + n += 1 + sovCsi(uint64(m.Total)) + } + if m.Used != 0 { + n += 1 + sovCsi(uint64(m.Used)) + } + if m.Unit != 0 { + n += 1 + sovCsi(uint64(m.Unit)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *VolumeCondition) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Abnormal { + n += 2 + } + l = len(m.Message) + if l > 0 { + n += 1 + l + sovCsi(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovCsi(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozCsi(x uint64) (n int) { + return sovCsi(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *VolumeStatsResponse) String() string { + if this == nil { + return "nil" + } + repeatedStringForUsage := "[]*VolumeUsage{" + for _, f := range this.Usage { + repeatedStringForUsage += strings.Replace(f.String(), "VolumeUsage", "VolumeUsage", 1) + "," + } + repeatedStringForUsage += "}" + s := strings.Join([]string{`&VolumeStatsResponse{`, + `Usage:` + repeatedStringForUsage + `,`, + `VolumeCondition:` + strings.Replace(this.VolumeCondition.String(), "VolumeCondition", "VolumeCondition", 1) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeUsage) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeUsage{`, + `Available:` + fmt.Sprintf("%v", this.Available) + `,`, + `Total:` + fmt.Sprintf("%v", this.Total) + `,`, + `Used:` + fmt.Sprintf("%v", this.Used) + `,`, + `Unit:` + fmt.Sprintf("%v", this.Unit) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeCondition) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeCondition{`, + `Abnormal:` + fmt.Sprintf("%v", this.Abnormal) + `,`, + `Message:` + fmt.Sprintf("%v", this.Message) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func valueToStringCsi(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *VolumeStatsResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeStatsResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeStatsResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCsi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCsi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Usage = append(m.Usage, &VolumeUsage{}) + if err := m.Usage[len(m.Usage)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeCondition", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCsi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCsi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.VolumeCondition == nil { + m.VolumeCondition = &VolumeCondition{} + } + if err := m.VolumeCondition.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCsi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCsi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeUsage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeUsage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeUsage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Available", wireType) + } + m.Available = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Available |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) + } + m.Total = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Total |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Used", wireType) + } + m.Used = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Used |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Unit", wireType) + } + m.Unit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Unit |= VolumeUsage_Unit(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipCsi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCsi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeCondition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeCondition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeCondition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Abnormal", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Abnormal = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCsi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCsi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCsi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCsi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipCsi(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCsi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCsi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCsi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthCsi + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupCsi + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthCsi + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthCsi = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowCsi = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupCsi = fmt.Errorf("proto: unexpected end of group") +) diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go new file mode 100644 index 000000000..87346ddf1 --- /dev/null +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go @@ -0,0 +1,585 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto + +package grpc + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb" + github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" + proto "github.com/gogo/protobuf/proto" + math "math" + math_rand "math/rand" + testing "testing" + time "time" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +func TestVolumeStatsResponseProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVolumeStatsResponseMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func BenchmarkVolumeStatsResponseProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeStatsResponse, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedVolumeStatsResponse(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkVolumeStatsResponseProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeStatsResponse(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &VolumeStatsResponse{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeUsageProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVolumeUsageMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func BenchmarkVolumeUsageProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeUsage, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedVolumeUsage(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkVolumeUsageProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeUsage(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &VolumeUsage{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeConditionProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVolumeConditionMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func BenchmarkVolumeConditionProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeCondition, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedVolumeCondition(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkVolumeConditionProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeCondition(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &VolumeCondition{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeStatsResponseJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeStatsResponse{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestVolumeUsageJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeUsage{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestVolumeConditionJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeCondition{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestVolumeStatsResponseProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeStatsResponseProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeUsageProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeUsageProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeConditionProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeConditionProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeStatsResponseSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + +func BenchmarkVolumeStatsResponseSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeStatsResponse, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedVolumeStatsResponse(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeUsageSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + +func BenchmarkVolumeUsageSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeUsage, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedVolumeUsage(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeConditionSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + +func BenchmarkVolumeConditionSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeCondition, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedVolumeCondition(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeStatsResponseStringer(t *testing.T) { + popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) + p := NewPopulatedVolumeStatsResponse(popr, false) + s1 := p.String() + s2 := fmt.Sprintf("%v", p) + if s1 != s2 { + t.Fatalf("String want %v got %v", s1, s2) + } +} +func TestVolumeUsageStringer(t *testing.T) { + popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) + p := NewPopulatedVolumeUsage(popr, false) + s1 := p.String() + s2 := fmt.Sprintf("%v", p) + if s1 != s2 { + t.Fatalf("String want %v got %v", s1, s2) + } +} +func TestVolumeConditionStringer(t *testing.T) { + popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) + p := NewPopulatedVolumeCondition(popr, false) + s1 := p.String() + s2 := fmt.Sprintf("%v", p) + if s1 != s2 { + t.Fatalf("String want %v got %v", s1, s2) + } +} + +//These tests are generated by github.com/gogo/protobuf/plugin/testgen diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go index 264c74d7b..919f2e8a5 100644 --- a/src/runtime/virtcontainers/pkg/mock/mock.go +++ b/src/runtime/virtcontainers/pkg/mock/mock.go @@ -12,12 +12,13 @@ import ( "net/url" "os" + "path" + "strings" + "github.com/containerd/ttrpc" gpb "github.com/gogo/protobuf/types" aTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" - "path" - "strings" ) const VSockPrefix = "mock://" @@ -232,6 +233,14 @@ func (p *HybridVSockTTRPCMockImp) AddSwap(ctx context.Context, req *pb.AddSwapRe return &gpb.Empty{}, nil } +func (p *HybridVSockTTRPCMockImp) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequest) (*pb.VolumeStatsResponse, error) { + return &pb.VolumeStatsResponse{}, nil +} + +func (p *HybridVSockTTRPCMockImp) ResizeVolume(ctx context.Context, req *pb.ResizeVolumeRequest) (*gpb.Empty, error) { + return &gpb.Empty{}, nil +} + func (p *HybridVSockTTRPCMockImp) PullImage(ctx context.Context, req *pb.PullImageRequest) (*gpb.Empty, error) { return &gpb.Empty{}, nil } diff --git a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go index af307e19b..ed0072dc7 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go +++ b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go @@ -256,6 +256,13 @@ func (s *Sandbox) GetHypervisorPid() (int, error) { return 0, nil } +func (s *Sandbox) GuestVolumeStats(ctx context.Context, path string) ([]byte, error) { + return nil, nil +} +func (s *Sandbox) ResizeGuestVolume(ctx context.Context, path string, size uint64) error { + return nil +} + func (s *Sandbox) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) { return nil, nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 0985380d8..242b3645d 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1840,8 +1840,8 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { coreID := fmt.Sprintf("%d", hc.Properties.Core) threadID := fmt.Sprintf("%d", hc.Properties.Thread) - // If CPU type is IBM pSeries or Z, we do not set socketID and threadID - if machine.Type == "pseries" || machine.Type == "s390-ccw-virtio" { + // If CPU type is IBM pSeries, Z or arm virt, we do not set socketID and threadID + if machine.Type == "pseries" || machine.Type == "s390-ccw-virtio" || machine.Type == "virt" { socketID = "" threadID = "" dieID = "" diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 8e76bf55c..3f52ab756 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -158,9 +158,8 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { func (q *qemuAmd64) capabilities() types.Capabilities { var caps types.Capabilities - if (q.qemuMachine.Type == QemuQ35 || - q.qemuMachine.Type == QemuVirt) && - q.protection == noneProtection { + if q.qemuMachine.Type == QemuQ35 || + q.qemuMachine.Type == QemuVirt { caps.SetBlockDeviceHotplugSupport() } diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index c5c5e8057..62fec60a7 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -277,9 +277,7 @@ func (q *qemuArchBase) kernelParameters(debug bool) []Param { func (q *qemuArchBase) capabilities() types.Capabilities { var caps types.Capabilities - if q.protection == noneProtection { - caps.SetBlockDeviceHotplugSupport() - } + caps.SetBlockDeviceHotplugSupport() caps.SetMultiQueueSupport() caps.SetFsSharingSupport() return caps diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index e78fcb701..e18f2264b 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -101,8 +101,7 @@ func (q *qemuPPC64le) capabilities() types.Capabilities { var caps types.Capabilities // pseries machine type supports hotplugging drives - if q.qemuMachine.Type == QemuPseries && - q.protection == noneProtection { + if q.qemuMachine.Type == QemuPseries { caps.SetBlockDeviceHotplugSupport() } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 6852a1d42..8d96fb50a 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -2261,6 +2261,43 @@ func (s *Sandbox) GetAgentURL() (string, error) { return s.agent.getAgentURL() } +// GuestVolumeStats return the filesystem stat of a given volume in the guest. +func (s *Sandbox) GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error) { + guestMountPath, err := s.guestMountPath(volumePath) + if err != nil { + return nil, err + } + return s.agent.getGuestVolumeStats(ctx, guestMountPath) +} + +// ResizeGuestVolume resizes a volume in the guest. +func (s *Sandbox) ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error { + // TODO: https://github.com/kata-containers/kata-containers/issues/3694. + guestMountPath, err := s.guestMountPath(volumePath) + if err != nil { + return err + } + return s.agent.resizeGuestVolume(ctx, guestMountPath, size) +} + +func (s *Sandbox) guestMountPath(volumePath string) (string, error) { + // verify the device even exists + if _, err := os.Stat(volumePath); err != nil { + s.Logger().WithError(err).WithField("volume", volumePath).Error("Cannot get stats for volume that doesn't exist") + return "", err + } + + // verify that we have a mount in this sandbox who's source maps to this + for _, c := range s.containers { + for _, m := range c.mounts { + if volumePath == m.Source { + return m.GuestDeviceMount, nil + } + } + } + return "", fmt.Errorf("mount %s not found in sandbox", volumePath) +} + // getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets' // cpus and mems as a string in canonical linux CPU/mems list format func (s *Sandbox) getSandboxCPUSet() (string, string, error) { diff --git a/src/tools/agent-ctl/src/client.rs b/src/tools/agent-ctl/src/client.rs index 64f45bdbf..a054cf71c 100644 --- a/src/tools/agent-ctl/src/client.rs +++ b/src/tools/agent-ctl/src/client.rs @@ -175,6 +175,11 @@ static AGENT_CMDS: &[AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_sandbox_get_oom_event, }, + AgentCmd { + name: "GetVolumeStats", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_get_volume_stats, + }, AgentCmd { name: "ListInterfaces", st: ServiceType::Agent, @@ -1703,6 +1708,30 @@ fn agent_cmd_sandbox_get_oom_event( Ok(()) } +fn agent_cmd_sandbox_get_volume_stats( + ctx: &Context, + client: &AgentServiceClient, + _health: &HealthClient, + _image: &ImageClient, + _options: &mut Options, + args: &str, +) -> Result<()> { + let req: VolumeStatsRequest = utils::make_request(args)?; + + let ctx = clone_context(ctx); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .get_volume_stats(ctx, &req) + .map_err(|e| anyhow!(e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + fn agent_cmd_sandbox_copy_file( ctx: &Context, client: &AgentServiceClient, diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 2def8f1c1..16cb75510 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -89,6 +89,7 @@ Options: -c : Path to config file to build the kernel. -d : Enable bash debug. -e : Enable experimental kernel. + -E : Enable arch-specific experimental kernel, arch info offered by "-a". -f : Enable force generate config when setup. -g : GPU vendor, intel or nvidia. -h : Display this help. @@ -462,7 +463,7 @@ install_kata() { } main() { - while getopts "a:b:c:defg:hk:p:st:v:x:" opt; do + while getopts "a:b:c:deEfg:hk:p:t:v:x:" opt; do case "$opt" in a) arch_target="${OPTARG}" @@ -480,6 +481,9 @@ main() { e) build_type="experimental" ;; + E) + build_type="arch-experimental" + ;; f) force_setup_generate_config="true" ;; @@ -525,6 +529,17 @@ main() { if [ -z "$kernel_version" ]; then if [[ ${build_type} == "experimental" ]]; then kernel_version=$(get_from_kata_deps "assets.kernel-experimental.tag") + elif [[ ${build_type} == "arch-experimental" ]]; then + case "${arch_target}" in + "aarch64") + build_type="arm-experimental" + kernel_version=$(get_from_kata_deps "assets.arm-kernel-experimental.version") + ;; + *) + info "No arch-specific experimental kernel supported, using experimental one instead" + kernel_version=$(get_from_kata_deps "assets.kernel-experimental.tag") + ;; + esac elif [[ "${conf_guest}" == "tdx" ]]; then kernel_version=$(get_from_kata_deps "assets.kernel.tdx.tag") else diff --git a/tools/packaging/kernel/configs/fragments/arm64/base.conf b/tools/packaging/kernel/configs/fragments/arm64/base.conf index 5ce4ac8c8..b5202d876 100644 --- a/tools/packaging/kernel/configs/fragments/arm64/base.conf +++ b/tools/packaging/kernel/configs/fragments/arm64/base.conf @@ -37,7 +37,6 @@ CONFIG_ARM64_PAN=y CONFIG_ARM64_CNP=y CONFIG_ARM64_PMEM=y CONFIG_ARM64_RAS_EXTN=y -CONFIG_ARM64_UAO=y # end of ARMv8.2 architectural feature CONFIG_NO_HZ_FULL=y diff --git a/tools/packaging/kernel/configs/fragments/build-type/arm-experimental/no.conf b/tools/packaging/kernel/configs/fragments/build-type/arm-experimental/no.conf new file mode 100644 index 000000000..e69de29bb diff --git a/tools/packaging/kernel/configs/fragments/common/fs.conf b/tools/packaging/kernel/configs/fragments/common/fs.conf index 4a702e092..ae4e3255f 100644 --- a/tools/packaging/kernel/configs/fragments/common/fs.conf +++ b/tools/packaging/kernel/configs/fragments/common/fs.conf @@ -26,7 +26,6 @@ CONFIG_FS_POSIX_ACL=y CONFIG_EXPORTFS=y CONFIG_EXPORTFS_BLOCK_OPS=y CONFIG_FILE_LOCKING=y -CONFIG_MANDATORY_FILE_LOCKING=y # A bunch of these are required for systemd at least. CONFIG_FSNOTIFY=y CONFIG_DNOTIFY=y diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0001-arm64-kernel-Handle-disabled-present-cpus-in-MADT-GI.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0001-arm64-kernel-Handle-disabled-present-cpus-in-MADT-GI.patch new file mode 100644 index 000000000..b07f9af4a --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0001-arm64-kernel-Handle-disabled-present-cpus-in-MADT-GI.patch @@ -0,0 +1,145 @@ +From 790af0565140c9df7394c195c22960d92f117c30 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 1 Dec 2021 14:58:33 +0800 +Subject: [PATCH 1/7] arm64: kernel: Handle disabled[(+)present] cpus in + MADT/GICC during init + +With ACPI enabled, cpus get identified by the presence of the GICC +entry in the MADT Table. Each GICC entry part of MADT presents cpu as +enabled or disabled. As of now, the disabled cpus are skipped as +physical cpu hotplug is not supported. These remain disabled even after +the kernel has booted. + +To support virtual cpu hotplug(in which case disabled vcpus could be +hotplugged even after kernel has booted), QEMU will populate MADT Table +with appropriate details of GICC entry for each possible(present+disabled) +vcpu. Now, during the init time vcpus will be identified as present or +disabled. To achieve this, below changes have been made with respect to +the present/possible vcpu handling along with the mentioned reasoning: + +1. Identify all possible(present+disabled) vcpus at boot/init time + and set their present mask and possible mask. In the existing code, + cpus are being marked present quite late within smp_prepare_cpus() + function, which gets called in context to the kernel thread. Since + the cpu hotplug is not supported, present cpus are always equal to + the possible cpus. But with cpu hotplug enabled, this assumption is + not true. Hence, present cpus should be marked while MADT GICC entries + are bring parsed for each vcpu. +2. Set possible cpus to include disabled. This needs to be done now + while parsing MADT GICC entries corresponding to each vcpu as the + disabled vcpu info is available only at this point as for hotplug + case possible vcpus is not equal to present vcpus. +3. We will store the parsed madt/gicc entry even for the disabled vcpus + during init time. This is needed as some modules like PMU registers + IRQs for each possible vcpus during init time. Therefore, a valid + entry of the MADT GICC should be present for all possible vcpus. +4. Refactoring related to DT/OF is also done to align it with the init + changes to support vcpu hotplug. + +Signed-off-by: Salil Mehta +Signed-off-by: Xiongfeng Wang +--- + arch/arm64/kernel/smp.c | 28 +++++++++++++++++++++------- + 1 file changed, 21 insertions(+), 7 deletions(-) + +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 6f6ff072acbd..4b317e71b1c4 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -524,13 +524,12 @@ static int __init smp_cpu_setup(int cpu) + if (ops->cpu_init(cpu)) + return -ENODEV; + +- set_cpu_possible(cpu, true); +- + return 0; + } + + static bool bootcpu_valid __initdata; + static unsigned int cpu_count = 1; ++static unsigned int disabled_cpu_count; + + #ifdef CONFIG_ACPI + static struct acpi_madt_generic_interrupt cpu_madt_gicc[NR_CPUS]; +@@ -549,10 +548,17 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) + static void __init + acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + { ++ unsigned int total_cpu_count = disabled_cpu_count + cpu_count; + u64 hwid = processor->arm_mpidr; + + if (!(processor->flags & ACPI_MADT_ENABLED)) { ++#ifndef CONFIG_ACPI_HOTPLUG_CPU + pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid); ++#else ++ cpu_madt_gicc[total_cpu_count] = *processor; ++ set_cpu_possible(total_cpu_count, true); ++ disabled_cpu_count++; ++#endif + return; + } + +@@ -561,7 +567,7 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + return; + } + +- if (is_mpidr_duplicate(cpu_count, hwid)) { ++ if (is_mpidr_duplicate(total_cpu_count, hwid)) { + pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid); + return; + } +@@ -582,9 +588,9 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + return; + + /* map the logical cpu id to cpu MPIDR */ +- set_cpu_logical_map(cpu_count, hwid); ++ set_cpu_logical_map(total_cpu_count, hwid); + +- cpu_madt_gicc[cpu_count] = *processor; ++ cpu_madt_gicc[total_cpu_count] = *processor; + + /* + * Set-up the ACPI parking protocol cpu entries +@@ -595,7 +601,10 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + * initialize the cpu if the parking protocol is + * the only available enable method). + */ +- acpi_set_mailbox_entry(cpu_count, processor); ++ acpi_set_mailbox_entry(total_cpu_count, processor); ++ ++ set_cpu_possible(total_cpu_count, true); ++ set_cpu_present(total_cpu_count, true); + + cpu_count++; + } +@@ -629,6 +638,9 @@ static void __init acpi_parse_and_init_cpus(void) + acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, + acpi_parse_gic_cpu_interface, 0); + ++ pr_debug("possible cpus(%u) present cpus(%u) disabled cpus(%u)\n", ++ cpu_count+disabled_cpu_count, cpu_count, disabled_cpu_count); ++ + /* + * In ACPI, SMP and CPU NUMA information is provided in separate + * static tables, namely the MADT and the SRAT. +@@ -699,6 +711,9 @@ static void __init of_parse_and_init_cpus(void) + set_cpu_logical_map(cpu_count, hwid); + + early_map_cpu_to_node(cpu_count, of_node_to_nid(dn)); ++ ++ set_cpu_possible(cpu_count, true); ++ set_cpu_present(cpu_count, true); + next: + cpu_count++; + } +@@ -783,7 +798,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) + if (err) + continue; + +- set_cpu_present(cpu, true); + numa_store_cpu_info(cpu); + } + } +-- +2.17.1 + diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0002-arm64-kernel-Bound-the-total-present-disabled-cpus-w.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0002-arm64-kernel-Bound-the-total-present-disabled-cpus-w.patch new file mode 100644 index 000000000..392740d72 --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0002-arm64-kernel-Bound-the-total-present-disabled-cpus-w.patch @@ -0,0 +1,86 @@ +From 2bd0439913fde8598113cc3959764a877c0bd1ad Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 1 Dec 2021 16:01:17 +0800 +Subject: [PATCH 2/7] arm64: kernel: Bound the total(present+disabled) cpus + with nr_cpu_ids + +Bound the total number of identified cpus(including disabled cpus) by +maximum allowed limit by the kernel. Max value is either specified as +part of the kernel parameters 'nr_cpus' or specified during compile +time using CONFIG_NR_CPUS. + +Signed-off-by: Salil Mehta +Signed-off-by: Xiongfeng Wang +--- + arch/arm64/kernel/smp.c | 18 ++++++++++++------ + 1 file changed, 12 insertions(+), 6 deletions(-) + +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 4b317e71b1c4..18a0576f2721 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -528,6 +528,7 @@ static int __init smp_cpu_setup(int cpu) + } + + static bool bootcpu_valid __initdata; ++static bool cpus_clipped __initdata = false; + static unsigned int cpu_count = 1; + static unsigned int disabled_cpu_count; + +@@ -551,6 +552,11 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + unsigned int total_cpu_count = disabled_cpu_count + cpu_count; + u64 hwid = processor->arm_mpidr; + ++ if (total_cpu_count > nr_cpu_ids) { ++ cpus_clipped = true; ++ return; ++ } ++ + if (!(processor->flags & ACPI_MADT_ENABLED)) { + #ifndef CONFIG_ACPI_HOTPLUG_CPU + pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid); +@@ -584,9 +590,6 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + return; + } + +- if (cpu_count >= NR_CPUS) +- return; +- + /* map the logical cpu id to cpu MPIDR */ + set_cpu_logical_map(total_cpu_count, hwid); + +@@ -704,8 +707,10 @@ static void __init of_parse_and_init_cpus(void) + continue; + } + +- if (cpu_count >= NR_CPUS) ++ if (cpu_count >= NR_CPUS) { ++ cpus_clipped = true; + goto next; ++ } + + pr_debug("cpu logical map 0x%llx\n", hwid); + set_cpu_logical_map(cpu_count, hwid); +@@ -726,6 +731,7 @@ static void __init of_parse_and_init_cpus(void) + */ + void __init smp_init_cpus(void) + { ++ unsigned int total_cpu_count = disabled_cpu_count + cpu_count; + int i; + + if (acpi_disabled) +@@ -733,9 +739,9 @@ void __init smp_init_cpus(void) + else + acpi_parse_and_init_cpus(); + +- if (cpu_count > nr_cpu_ids) ++ if (cpus_clipped) + pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", +- cpu_count, nr_cpu_ids); ++ total_cpu_count, nr_cpu_ids); + + if (!bootcpu_valid) { + pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); +-- +2.17.1 + diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0003-arm64-kernel-Init-cpu-operations-for-all-possible-vc.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0003-arm64-kernel-Init-cpu-operations-for-all-possible-vc.patch new file mode 100644 index 000000000..7145b5fbe --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0003-arm64-kernel-Init-cpu-operations-for-all-possible-vc.patch @@ -0,0 +1,116 @@ +From 58ceaa003bab7d2613f01ec58925a75e1f731240 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Thu, 2 Dec 2021 13:57:51 +0800 +Subject: [PATCH 3/7] arm64: kernel: Init cpu operations for all possible vcpus + +Currently, cpu-operations are only initialized for the cpus which +already have logical cpuid to hwid assoication established. And this +only happens for the cpus which are present during boot time. + +To support virtual cpu hotplug, we shall initialze the cpu-operations +for all possible(present+disabled) vcpus. This means logical cpuid to +hwid/mpidr association might not exists(i.e. might be INVALID_HWID) +during init. Later, when the vcpu is actually hotplugged logical cpuid +is allocated and associated with the hwid/mpidr. + +This patch does some refactoring to support above change. + +Signed-off-by: Salil Mehta +Signed-off-by: Xiongfeng Wang +--- + arch/arm64/kernel/smp.c | 39 +++++++++++++++------------------------ + 1 file changed, 15 insertions(+), 24 deletions(-) + +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 18a0576f2721..fed4415e8cfe 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -518,13 +518,16 @@ static int __init smp_cpu_setup(int cpu) + const struct cpu_operations *ops; + + if (init_cpu_ops(cpu)) +- return -ENODEV; ++ goto out; + + ops = get_cpu_ops(cpu); + if (ops->cpu_init(cpu)) +- return -ENODEV; ++ goto out; + + return 0; ++out: ++ __cpu_logical_map[cpu] = INVALID_HWID; ++ return -ENODEV; + } + + static bool bootcpu_valid __initdata; +@@ -562,7 +565,8 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid); + #else + cpu_madt_gicc[total_cpu_count] = *processor; +- set_cpu_possible(total_cpu_count, true); ++ if (!smp_cpu_setup(total_cpu_count)) ++ set_cpu_possible(total_cpu_count, true); + disabled_cpu_count++; + #endif + return; +@@ -606,9 +610,10 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) + */ + acpi_set_mailbox_entry(total_cpu_count, processor); + +- set_cpu_possible(total_cpu_count, true); +- set_cpu_present(total_cpu_count, true); +- ++ if (!smp_cpu_setup(total_cpu_count)) { ++ set_cpu_possible(total_cpu_count, true); ++ set_cpu_present(total_cpu_count, true); ++ } + cpu_count++; + } + +@@ -716,9 +721,10 @@ static void __init of_parse_and_init_cpus(void) + set_cpu_logical_map(cpu_count, hwid); + + early_map_cpu_to_node(cpu_count, of_node_to_nid(dn)); +- +- set_cpu_possible(cpu_count, true); +- set_cpu_present(cpu_count, true); ++ if (!smp_cpu_setup(cpu_count)) { ++ set_cpu_possible(cpu_count, true); ++ set_cpu_present(cpu_count, true); ++ } + next: + cpu_count++; + } +@@ -732,7 +738,6 @@ static void __init of_parse_and_init_cpus(void) + void __init smp_init_cpus(void) + { + unsigned int total_cpu_count = disabled_cpu_count + cpu_count; +- int i; + + if (acpi_disabled) + of_parse_and_init_cpus(); +@@ -747,20 +752,6 @@ void __init smp_init_cpus(void) + pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); + return; + } +- +- /* +- * We need to set the cpu_logical_map entries before enabling +- * the cpus so that cpu processor description entries (DT cpu nodes +- * and ACPI MADT entries) can be retrieved by matching the cpu hwid +- * with entries in cpu_logical_map while initializing the cpus. +- * If the cpu set-up fails, invalidate the cpu_logical_map entry. +- */ +- for (i = 1; i < nr_cpu_ids; i++) { +- if (cpu_logical_map(i) != INVALID_HWID) { +- if (smp_cpu_setup(i)) +- set_cpu_logical_map(i, INVALID_HWID); +- } +- } + } + + void __init smp_prepare_cpus(unsigned int max_cpus) +-- +2.17.1 + diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0004-arm64-kernel-Arch-specific-ACPI-hooks-like-logical-c.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0004-arm64-kernel-Arch-specific-ACPI-hooks-like-logical-c.patch new file mode 100644 index 000000000..f21df43a0 --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0004-arm64-kernel-Arch-specific-ACPI-hooks-like-logical-c.patch @@ -0,0 +1,125 @@ +From 6b7b492fc89e97e5ee51f9d033000fb6483a5298 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 1 Dec 2021 16:21:50 +0800 +Subject: [PATCH 4/7] arm64: kernel: Arch specific ACPI hooks(like logical + cpuid<->hwid etc.) + +To support virtual cpu hotplug, some arch specifc hooks must be +facilitated. These hooks are called by the generic ACPI cpu hotplug +framework during a vcpu hot-(un)plug event handling. The changes +required involve: + +1. Allocation of the logical cpuid corresponding to the hwid/mpidr +2. Mapping of logical cpuid to hwid/mpidr and marking present +3. Removing vcpu from present mask during hot-unplug +4. For arm64, all possible cpus are registered within topology_init() + Hence, we need to override the weak ACPI call of arch_register_cpu() + (which returns -ENODEV) and return success. +5. NUMA node mapping set for this vcpu using SRAT Table info during init + time will be discarded as the logical cpu-ids used at that time + might not be correct. This mapping will be set again using the + proximity/node info obtained by evaluating _PXM ACPI method. + +Note, during hot unplug of vcpu, we do not unmap the association between +the logical cpuid and hwid/mpidr. This remains persistent. + +Signed-off-by: Salil Mehta +Signed-off-by: Xiongfeng Wang +--- + arch/arm64/kernel/smp.c | 80 +++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 80 insertions(+) + +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index fed4415e8cfe..8ab68ec01090 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -543,6 +543,86 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) + return &cpu_madt_gicc[cpu]; + } + ++#ifdef CONFIG_ACPI_HOTPLUG_CPU ++int arch_register_cpu(int num) ++{ ++ return 0; ++} ++ ++static int set_numa_node_for_cpu(acpi_handle handle, int cpu) ++{ ++#ifdef CONFIG_ACPI_NUMA ++ int node_id; ++ ++ /* will evaluate _PXM */ ++ node_id = acpi_get_node(handle); ++ if (node_id != NUMA_NO_NODE) ++ set_cpu_numa_node(cpu, node_id); ++#endif ++ return 0; ++} ++ ++static void unset_numa_node_for_cpu(int cpu) ++{ ++#ifdef CONFIG_ACPI_NUMA ++ set_cpu_numa_node(cpu, NUMA_NO_NODE); ++#endif ++} ++ ++static int allocate_logical_cpuid(u64 physid) ++{ ++ int first_invalid_idx = -1; ++ bool first = true; ++ int i; ++ ++ for_each_possible_cpu(i) { ++ /* ++ * logical cpuid<->hwid association remains persistent once ++ * established ++ */ ++ if (cpu_logical_map(i) == physid) ++ return i; ++ ++ if ((cpu_logical_map(i) == INVALID_HWID) && first) { ++ first_invalid_idx = i; ++ first = false; ++ } ++ } ++ ++ return first_invalid_idx; ++} ++ ++int acpi_unmap_cpu(int cpu) ++{ ++ set_cpu_present(cpu, false); ++ unset_numa_node_for_cpu(cpu); ++ ++ return 0; ++} ++ ++int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, ++ int *cpuid) ++{ ++ int cpu; ++ ++ cpu = allocate_logical_cpuid(physid); ++ if (cpu < 0) { ++ pr_warn("Unable to map logical cpuid to physid 0x%llx\n", ++ physid); ++ return -ENOSPC; ++ } ++ ++ /* map the logical cpu id to cpu MPIDR */ ++ __cpu_logical_map[cpu] = physid; ++ set_numa_node_for_cpu(handle, cpu); ++ ++ set_cpu_present(cpu, true); ++ *cpuid = cpu; ++ ++ return 0; ++} ++#endif ++ + /* + * acpi_map_gic_cpu_interface - parse processor MADT entry + * +-- +2.17.1 + diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0005-cpu-numa-fix-failure-when-hot-remove-cpu.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0005-cpu-numa-fix-failure-when-hot-remove-cpu.patch new file mode 100644 index 000000000..793e02fda --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0005-cpu-numa-fix-failure-when-hot-remove-cpu.patch @@ -0,0 +1,82 @@ +From 5c979f026c1319c712e7fa4882ec3a4ef3e2101b Mon Sep 17 00:00:00 2001 +From: Jianyong Wu +Date: Fri, 3 Dec 2021 17:11:39 +0800 +Subject: [PATCH 5/7] cpu/numa: fix failure when hot-remove cpu + +when hot-remove cpu, the map from cpu to numa will set to NUMA_NO_NODE +which will lead to failure as the map is used by others. thus we need a +specific map to descrip the unpluged cpu. +Here we introduce a new map to descrip the unpluged cpu map. + +Singed-off-by: Jianyong Wu +--- + arch/arm64/include/asm/smp.h | 2 ++ + arch/arm64/kernel/setup.c | 14 ++++++++++++++ + arch/arm64/kernel/smp.c | 5 ++++- + 3 files changed, 20 insertions(+), 1 deletion(-) + +diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h +index fc55f5a57a06..7949f6090eed 100644 +--- a/arch/arm64/include/asm/smp.h ++++ b/arch/arm64/include/asm/smp.h +@@ -47,6 +47,8 @@ DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); + */ + extern u64 __cpu_logical_map[NR_CPUS]; + extern u64 cpu_logical_map(unsigned int cpu); ++extern u64 get_acpicpu_numa_node(unsigned int cpu); ++extern int set_acpicpu_numa_node(unsigned int cpu, unsigned int node); + + static inline void set_cpu_logical_map(unsigned int cpu, u64 hwid) + { +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index be5f85b0a24d..68d7a7894e10 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -284,6 +284,20 @@ static int __init reserve_memblock_reserved_regions(void) + } + arch_initcall(reserve_memblock_reserved_regions); + ++u64 __acpicpu_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; ++ ++u64 get_acpicpu_numa_node(unsigned int cpu) ++{ ++ return __acpicpu_node_map[cpu]; ++} ++ ++int set_acpicpu_numa_node(unsigned int cpu, unsigned int node) ++{ ++ __acpicpu_node_map[cpu] = node; ++ ++ return 0; ++} ++ + u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID }; + + u64 cpu_logical_map(unsigned int cpu) +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 8ab68ec01090..0c07921b0b61 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -557,7 +557,10 @@ static int set_numa_node_for_cpu(acpi_handle handle, int cpu) + /* will evaluate _PXM */ + node_id = acpi_get_node(handle); + if (node_id != NUMA_NO_NODE) ++ { ++ set_acpicpu_numa_node(cpu, node_id); + set_cpu_numa_node(cpu, node_id); ++ } + #endif + return 0; + } +@@ -565,7 +568,7 @@ static int set_numa_node_for_cpu(acpi_handle handle, int cpu) + static void unset_numa_node_for_cpu(int cpu) + { + #ifdef CONFIG_ACPI_NUMA +- set_cpu_numa_node(cpu, NUMA_NO_NODE); ++ set_acpicpu_numa_node(cpu, NUMA_NO_NODE); + #endif + } + +-- +2.17.1 + diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0006-arm64-mm-avoid-fixmap-race-condition-when-create-pud.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0006-arm64-mm-avoid-fixmap-race-condition-when-create-pud.patch new file mode 100644 index 000000000..bfbc79153 --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0006-arm64-mm-avoid-fixmap-race-condition-when-create-pud.patch @@ -0,0 +1,69 @@ +From e3a11f2f7ccb0dbbb8cf95944e89b34fd928107a Mon Sep 17 00:00:00 2001 +From: Jianyong Wu +Date: Mon, 6 Dec 2021 10:52:37 +0800 +Subject: [PATCH 6/7] arm64/mm: avoid fixmap race condition when create pud + mapping + +The 'fixmap' is a global resource and is used recursively by +create pud mapping(), leading to a potential race condition in the +presence of a concurrent call to alloc_init_pud(): + +kernel_init thread virtio-mem workqueue thread +================== =========================== + + alloc_init_pud(...) alloc_init_pud(...) + pudp = pud_set_fixmap_offset(...) pudp = pud_set_fixmap_offset(...) + READ_ONCE(*pudp) + pud_clear_fixmap(...) + READ_ONCE(*pudp) // CRASH! + +As kernel may sleep during creating pud mapping, introduce a mutex lock to +serialise use of the fixmap entries by alloc_init_pud(). However, there is +no need for locking in early boot stage and it doesn't work well with +KASLR enabled when early boot. So, enable lock when system_state doesn't +equal to "SYSTEM_BOOTING". + +Signed-off-by: Jianyong Wu +Reviewed-by: Catalin Marinas +Fixes: f4710445458c ("arm64: mm: use fixmap when creating page tables") +Link: https://lore.kernel.org/r/20220201114400.56885-1-jianyong.wu@arm.com +Signed-off-by: Will Deacon +--- + arch/arm64/mm/mmu.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c +index cfd9deb347c3..432fab4ce2b4 100644 +--- a/arch/arm64/mm/mmu.c ++++ b/arch/arm64/mm/mmu.c +@@ -63,6 +63,7 @@ static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; + static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; + + static DEFINE_SPINLOCK(swapper_pgdir_lock); ++static DEFINE_SPINLOCK(fixmap_lock); + + void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) + { +@@ -328,6 +329,11 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, + } + BUG_ON(p4d_bad(p4d)); + ++ /* ++ * We only have one fixmap entry per page-table level, so take ++ * the fixmap lock until we're done. ++ */ ++ spin_lock(&fixmap_lock); + pudp = pud_set_fixmap_offset(p4dp, addr); + do { + pud_t old_pud = READ_ONCE(*pudp); +@@ -358,6 +364,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, + } while (pudp++, addr = next, addr != end); + + pud_clear_fixmap(); ++ spin_unlock(&fixmap_lock); + } + + static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, +-- +2.17.1 + diff --git a/tools/packaging/kernel/patches/5.15.x/arm-experimental/0007-virtio-mem-enable-virtio-mem-on-arm64.patch b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0007-virtio-mem-enable-virtio-mem-on-arm64.patch new file mode 100644 index 000000000..bff84a1e9 --- /dev/null +++ b/tools/packaging/kernel/patches/5.15.x/arm-experimental/0007-virtio-mem-enable-virtio-mem-on-arm64.patch @@ -0,0 +1,67 @@ +From b1a3d86afbccb5485d2a53cc7e4e097a40f9d443 Mon Sep 17 00:00:00 2001 +From: Jianyong Wu +Date: Tue, 14 Dec 2021 14:18:39 +0800 +Subject: [PATCH 7/7] virtio-mem: enable virtio-mem on arm64 + +It seems that virtio-mem works on arm64 now and can be enabled. + +Signed-off-by: Jianyong Wu +--- + arch/arm64/mm/mmu.c | 12 +++++++----- + drivers/virtio/Kconfig | 2 +- + 2 files changed, 8 insertions(+), 6 deletions(-) + +diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c +index 432fab4ce2b4..809fe52d3035 100644 +--- a/arch/arm64/mm/mmu.c ++++ b/arch/arm64/mm/mmu.c +@@ -63,7 +63,7 @@ static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; + static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; + + static DEFINE_SPINLOCK(swapper_pgdir_lock); +-static DEFINE_SPINLOCK(fixmap_lock); ++static DEFINE_MUTEX(fixmap_lock); + + void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) + { +@@ -330,10 +330,11 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, + BUG_ON(p4d_bad(p4d)); + + /* +- * We only have one fixmap entry per page-table level, so take +- * the fixmap lock until we're done. ++ * No need for locking during early boot. And it doesn't work as ++ * expected with KASLR enabled. + */ +- spin_lock(&fixmap_lock); ++ if (system_state != SYSTEM_BOOTING) ++ mutex_lock(&fixmap_lock); + pudp = pud_set_fixmap_offset(p4dp, addr); + do { + pud_t old_pud = READ_ONCE(*pudp); +@@ -364,7 +365,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, + } while (pudp++, addr = next, addr != end); + + pud_clear_fixmap(); +- spin_unlock(&fixmap_lock); ++ if (system_state != SYSTEM_BOOTING) ++ mutex_unlock(&fixmap_lock); + } + + static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, +diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig +index ce1b3f6ec325..ebabff45935c 100644 +--- a/drivers/virtio/Kconfig ++++ b/drivers/virtio/Kconfig +@@ -96,7 +96,7 @@ config VIRTIO_BALLOON + config VIRTIO_MEM + tristate "Virtio mem driver" + default m +- depends on X86_64 ++ depends on X86_64 || ARM64 + depends on VIRTIO + depends on MEMORY_HOTPLUG_SPARSE + depends on MEMORY_HOTREMOVE +-- +2.17.1 + diff --git a/versions.yaml b/versions.yaml index de7122fca..ebca53c8d 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "b0324f85571c441f840e9bdeb25410514a00bb74" + version: "v22.0" firecracker: description: "Firecracker micro-VMM" @@ -162,6 +162,11 @@ assets: url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" tag: "v5.13.10" + arm-kernel-experimental: + description: "Linux kernel with cpu/mem hotplug support on arm64" + url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" + version: "v5.15.7" + externals: description: "Third-party projects used by the system"