From 9a01d4e4469adc2d4b6b95145da8fa9ba778cae0 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Wed, 18 May 2022 15:42:28 +0800 Subject: [PATCH 01/40] dragonball: add more unit test for virtio-blk device. Added more unit tests for virtio-blk device. Fixes: #4899 Signed-off-by: wllenyj --- .../src/device_manager/blk_dev_mgr.rs | 584 ++++++++++++++++++ 1 file changed, 584 insertions(+) diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 0493a818a..66890d7f9 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -776,3 +776,587 @@ impl Default for BlockDeviceMgr { } } } + +#[cfg(test)] +mod tests { + use test_utils::skip_if_not_root; + use vmm_sys_util::tempfile::TempFile; + + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + #[test] + fn test_block_device_type() { + let dev_type = BlockDeviceType::get_type("spool:/device1"); + assert_eq!(dev_type, BlockDeviceType::Spool); + let dev_type = BlockDeviceType::get_type("/device1"); + assert_eq!(dev_type, BlockDeviceType::RawBlock); + } + + #[test] + fn test_create_block_devices_configs() { + let mgr = BlockDeviceMgr::default(); + assert!(!mgr.has_root_block_device()); + assert!(!mgr.has_part_uuid_root()); + assert!(!mgr.is_read_only_root()); + assert_eq!(mgr.get_index_of_drive_id(""), None); + assert_eq!(mgr.info_list.len(), 0); + } + + #[test] + fn test_add_non_root_block_device() { + skip_if_not_root!(); + let dummy_file = TempFile::new().unwrap(); + let dummy_path = dummy_file.as_path().to_owned(); + let dummy_id = String::from("1"); + let dummy_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path.clone(), + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: dummy_id.clone(), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!(BlockDeviceMgr::insert_device( + vm.device_manager_mut(), + ctx, + dummy_block_device.clone(), + ) + .is_ok()); + + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + assert!(!vm.device_manager().block_manager.has_root_block_device()); + assert!(!vm.device_manager().block_manager.has_part_uuid_root()); + assert!(!vm.device_manager().block_manager.is_read_only_root()); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + assert_eq!( + vm.device_manager().block_manager.info_list[0] + .config + .device_type(), + BlockDeviceType::RawBlock + ); + assert_eq!( + vm.device_manager().block_manager.info_list[0] + .config + .queue_sizes(), + [128u16] + ); + + let dev_config = vm.device_manager().block_manager.iter().next().unwrap(); + assert_eq!(dev_config.config, dummy_block_device); + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_path(&dummy_path) + .is_some()); + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_id(&dummy_id) + .is_some()); + } + + #[test] + fn test_update_blk_device_ratelimiters() { + skip_if_not_root!(); + //Init vm for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + ); + + let dummy_file = TempFile::new().unwrap(); + let dummy_path = dummy_file.as_path().to_owned(); + + let dummy_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: true, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device) + .unwrap(); + + let cfg = BlockDeviceConfigUpdateInfo { + drive_id: String::from("1"), + rate_limiter: None, + }; + + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + ); + + vm.device_manager_mut() + .block_manager + .attach_devices(&mut device_op_ctx) + .unwrap(); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + + //Patch while the epoll handler is invalid. + let expected_error = "could not send patch message to the block epoll handler".to_string(); + + assert_eq!( + BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg) + .unwrap_err() + .to_string(), + expected_error + ); + + //Invalid drive id + let cfg2 = BlockDeviceConfigUpdateInfo { + drive_id: String::from("2"), + rate_limiter: None, + }; + + let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id); + + assert_eq!( + BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2) + .unwrap_err() + .to_string(), + expected_error + ); + } + + #[test] + fn test_add_one_root_block_device() { + skip_if_not_root!(); + let dummy_file = TempFile::new().unwrap(); + let dummy_path = dummy_file.as_path().to_owned(); + let dummy_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: true, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!(BlockDeviceMgr::insert_device( + vm.device_manager_mut(), + ctx, + dummy_block_device.clone(), + ) + .is_ok()); + + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + assert!(vm.device_manager().block_manager.has_root_block); + assert!(!vm.device_manager().block_manager.has_part_uuid_root); + assert!(vm.device_manager().block_manager.read_only_root); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); + + let dev_config = vm.device_manager().block_manager.iter().next().unwrap(); + assert_eq!(dev_config.config, dummy_block_device); + assert!(vm.device_manager().block_manager.is_read_only_root()); + } + + #[test] + fn test_add_two_root_block_devices_configs() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device_1 = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let root_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!( + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2) + .is_err() + ); + } + + #[test] + // Test BlockDevicesConfigs::add when you first add the root device and then the other devices. + fn test_add_root_block_device_first() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let dummy_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_3 = TempFile::new().unwrap(); + let dummy_path_3 = dummy_file_3.as_path().to_owned(); + let dummy_block_device_3 = BlockDeviceConfigInfo { + path_on_host: dummy_path_3, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("3"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + vm.device_manager_mut() + .block_manager + .create(root_block_device.clone()) + .unwrap(); + vm.device_manager_mut() + .block_manager + .create(dummy_block_device_2.clone()) + .unwrap(); + vm.device_manager_mut() + .block_manager + .create(dummy_block_device_3.clone()) + .unwrap(); + + assert!(vm.device_manager().block_manager.has_root_block_device(),); + assert!(!vm.device_manager().block_manager.has_part_uuid_root()); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 3); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap(); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap(); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap(); + } + + #[test] + // Test BlockDevicesConfigs::add when you add other devices first and then the root device. + fn test_root_block_device_add_last() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let dummy_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_3 = TempFile::new().unwrap(); + let dummy_path_3 = dummy_file_3.as_path().to_owned(); + let dummy_block_device_3 = BlockDeviceConfigInfo { + path_on_host: dummy_path_3, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("3"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + .unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone()) + .unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone()) + .unwrap(); + + assert!(vm.device_manager().block_manager.has_root_block_device(),); + assert!(!vm.device_manager().block_manager.has_part_uuid_root()); + assert_eq!(vm.device_manager().block_manager.info_list.len(), 3); + + let mut block_dev_iter = vm.device_manager().block_manager.iter(); + // The root device should be first in the list no matter of the order in + // which the devices were added. + assert_eq!( + block_dev_iter.next().unwrap().config.drive_id, + root_block_device.drive_id + ); + assert_eq!( + block_dev_iter.next().unwrap().config.drive_id, + dummy_block_device_2.drive_id + ); + assert_eq!( + block_dev_iter.next().unwrap().config.drive_id, + dummy_block_device_3.drive_id + ); + } + + #[test] + fn test_block_device_update() { + skip_if_not_root!(); + let dummy_file_1 = TempFile::new().unwrap(); + let dummy_path_1 = dummy_file_1.as_path().to_owned(); + let root_block_device = BlockDeviceConfigInfo { + path_on_host: dummy_path_1.clone(), + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let dummy_file_2 = TempFile::new().unwrap(); + let dummy_path_2 = dummy_file_2.as_path().to_owned(); + let mut dummy_block_device_2 = BlockDeviceConfigInfo { + path_on_host: dummy_path_2.clone(), + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + + let mut vm = crate::vm::tests::create_vm_instance(); + + // Add 2 block devices. + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + .unwrap(); + + // Get index zero. + assert_eq!( + vm.device_manager() + .block_manager + .get_index_of_drive_id(&String::from("1")) + .unwrap(), + 0 + ); + + // Get None. + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_id(&String::from("foo")) + .is_none()); + + // Test several update cases using dummy_block_device_2. + // Validate `dummy_block_device_2` is already in the list + assert!(vm + .device_manager() + .block_manager + .get_index_of_drive_id(&dummy_block_device_2.drive_id) + .is_some()); + // Update OK. + dummy_block_device_2.is_read_only = true; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + .unwrap(); + + let index = vm + .device_manager() + .block_manager + .get_index_of_drive_id(&dummy_block_device_2.drive_id) + .unwrap(); + // Validate update was successful. + assert!( + vm.device_manager().block_manager.info_list[index] + .config + .is_read_only + ); + + // Update with invalid path. + let dummy_filename_3 = String::from("test_update_3"); + let dummy_path_3 = PathBuf::from(dummy_filename_3); + dummy_block_device_2.path_on_host = dummy_path_3; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!(BlockDeviceMgr::insert_device( + vm.device_manager_mut(), + ctx, + dummy_block_device_2.clone(), + ) + .is_err()); + + // Update with 2 root block devices. + dummy_block_device_2.path_on_host = dummy_path_2.clone(); + dummy_block_device_2.is_root_device = true; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + assert!( + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,) + .is_err(), + ); + + // Switch roots and add a PARTUUID for the new one. + let root_block_device_old = BlockDeviceConfigInfo { + path_on_host: dummy_path_1, + device_type: BlockDeviceType::RawBlock, + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("1"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + let root_block_device_new = BlockDeviceConfigInfo { + path_on_host: dummy_path_2, + device_type: BlockDeviceType::RawBlock, + is_root_device: true, + part_uuid: Some("0eaa91a0-01".to_string()), + is_read_only: false, + is_direct: false, + no_drop: false, + drive_id: String::from("2"), + rate_limiter: None, + num_queues: BlockDeviceConfigInfo::default_num_queues(), + queue_size: 128, + use_shared_irq: None, + use_generic_irq: None, + }; + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap(); + let ctx = DeviceOpContext::create_boot_ctx(&vm, None); + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap(); + assert!(vm.device_manager().block_manager.has_part_uuid_root); + } +} From 3896c7a22bf3b9bb7e477af675141a3cc94d90e2 Mon Sep 17 00:00:00 2001 From: Sidhartha Mani Date: Fri, 3 Mar 2023 17:46:40 -0800 Subject: [PATCH 02/40] protocol: add updateEphemeralMounts proto - adds a new rpc call to the agent service named `updateEphemeralMounts` - this call takes a list of grpc.Storage objects Signed-off-by: Sidhartha Mani --- src/libs/protocols/protos/agent.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index 504cae83a..da4377f0d 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -34,6 +34,7 @@ service AgentService { rpc SignalProcess(SignalProcessRequest) returns (google.protobuf.Empty); rpc WaitProcess(WaitProcessRequest) returns (WaitProcessResponse); // wait & reap like waitpid(2) rpc UpdateContainer(UpdateContainerRequest) returns (google.protobuf.Empty); + rpc UpdateEphemeralMounts(UpdateEphemeralMountsRequest) returns (google.protobuf.Empty); rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse); rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty); rpc ResumeContainer(ResumeContainerRequest) returns (google.protobuf.Empty); @@ -316,6 +317,10 @@ message UpdateRoutesRequest { Routes routes = 1; } +message UpdateEphemeralMountsRequest { + repeated Storage storages = 1; +} + message ListInterfacesRequest { } From 16e2c3cc55b11fde7724b3ad66871bebe107e384 Mon Sep 17 00:00:00 2001 From: Sidhartha Mani Date: Fri, 3 Mar 2023 17:48:04 -0800 Subject: [PATCH 03/40] agent: implement update_ephemeral_mounts api - implement update_ephemeral_mounts rpc - for each mountpoint passed in, remount it with new options Signed-off-by: Sidhartha Mani --- src/agent/src/mount.rs | 64 ++++++++++++++++++++++++++++++++++++++++++ src/agent/src/rpc.rs | 19 ++++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 17ea3859c..bc13a6896 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -240,6 +240,70 @@ async fn ephemeral_storage_handler( Ok("".to_string()) } +// update_ephemeral_mounts takes a list of ephemeral mounts and remounts them +// with mount options passed by the caller +#[instrument] +pub async fn update_ephemeral_mounts( + logger: Logger, + storages: Vec, + sandbox: Arc>, +) -> Result<()> { + for (_, storage) in storages.iter().enumerate() { + let handler_name = storage.driver.clone(); + let logger = logger.new(o!( + "msg" => "updating tmpfs storage", + "subsystem" => "storage", + "storage-type" => handler_name.to_owned())); + + match handler_name.as_str() { + DRIVER_EPHEMERAL_TYPE => { + fs::create_dir_all(Path::new(&storage.mount_point))?; + + if storage.options.is_empty() { + continue; + } else { + // assume that fsGid has already been set + let mut opts = Vec::<&str>::new(); + for (_, opt) in storage.options.iter().enumerate() { + if opt.starts_with(FS_GID) { + continue; + } + opts.push(opt) + } + let mount_path = Path::new(&storage.mount_point); + let src_path = Path::new(&storage.source); + + let (flags, options) = parse_mount_flags_and_options(opts); + + info!(logger, "mounting storage"; + "mount-source" => src_path.display(), + "mount-destination" => mount_path.display(), + "mount-fstype" => storage.fstype.as_str(), + "mount-options" => options.as_str(), + ); + + return baremount( + src_path, + mount_path, + storage.fstype.as_str(), + flags, + options.as_str(), + &logger, + ); + } + } + _ => { + return Err(anyhow!( + "Unsupported storage type for syncing mounts {}. Only ephemeral storage update is supported", + storage.driver.to_owned() + )); + } + }; + } + + Ok(()) +} + #[instrument] async fn overlayfs_storage_handler( logger: &Logger, diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 3c0bc54c7..3a927b3b5 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -51,7 +51,7 @@ use crate::device::{ }; use crate::linux_abi::*; use crate::metrics::get_metrics; -use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST}; +use crate::mount::{add_storages, baremount, update_ephemeral_mounts, STORAGE_HANDLER_LIST}; use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS}; use crate::network::setup_guest_dns; use crate::pci; @@ -997,6 +997,23 @@ impl agent_ttrpc::AgentService for AgentService { }) } + async fn update_ephemeral_mounts( + &self, + ctx: &TtrpcContext, + req: protocols::agent::UpdateEphemeralMountsRequest, + ) -> ttrpc::Result { + trace_rpc_call!(ctx, "update_mounts", req); + is_allowed!(req); + + match update_ephemeral_mounts(sl!(), req.storages.to_vec(), self.sandbox.clone()).await { + Ok(_) => Ok(Empty::new()), + Err(e) => Err(ttrpc_error!( + ttrpc::Code::INTERNAL, + format!("Failed to update mounts: {:?}", e), + )), + } + } + async fn get_ip_tables( &self, ctx: &TtrpcContext, From 42b8867148d2287b2a40f23792a0ba23d97eea4f Mon Sep 17 00:00:00 2001 From: Tingzhou Yuan Date: Tue, 6 Dec 2022 06:54:34 +0000 Subject: [PATCH 04/40] runtime-rs: impl volume-stats trait for sandbox Implements get-volume-stats trait for sandbox, handler for shim-mgmt and add RPC calls to agent. Also added type conversions in trans.rs Fixes #5369 Signed-off-by: Tingzhou Yuan --- src/runtime-rs/Cargo.lock | 1 + src/runtime-rs/crates/agent/src/kata/agent.rs | 3 ++- src/runtime-rs/crates/agent/src/kata/trans.rs | 25 +++++++++++++++++-- src/runtime-rs/crates/agent/src/lib.rs | 5 ++-- src/runtime-rs/crates/agent/src/types.rs | 10 ++++++++ .../crates/runtimes/common/src/sandbox.rs | 1 + .../crates/runtimes/src/shim_mgmt/handlers.rs | 25 ++++++++++++++++++- .../runtimes/virt_container/src/sandbox.rs | 9 +++++++ 8 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index e38ae3510..e2525363b 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -2470,6 +2470,7 @@ dependencies = [ "slog", "slog-scope", "tokio", + "url", "virt_container", "wasm_container", ] diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index b403f0f44..6baab2dd1 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -115,5 +115,6 @@ impl_agent!( copy_file | crate::CopyFileRequest | crate::Empty | None, get_oom_event | crate::Empty | crate::OomEventResponse | Some(0), get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None, - set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None + set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None, + get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None ); diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 4ac0c45ec..f2f725535 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -8,7 +8,7 @@ use std::convert::Into; use protocols::{ agent::{self, OOMEvent}, - empty, health, types, + csi, empty, health, types, }; use crate::{ @@ -24,7 +24,7 @@ use crate::{ SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, StatsContainerResponse, Storage, StringUser, ThrottlingData, TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, - WaitProcessRequest, WriteStreamRequest, + VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, WriteStreamRequest, }, OomEventResponse, WaitProcessResponse, WriteStreamResponse, }; @@ -846,3 +846,24 @@ impl From for OomEventResponse { } } } + +impl From for agent::VolumeStatsRequest { + fn from(from: VolumeStatsRequest) -> Self { + Self { + volume_guest_path: from.volume_guest_path, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for VolumeStatsResponse { + fn from(from: csi::VolumeStatsResponse) -> Self { + let result: String = format!( + "Usage: {:?}\nVolume Condition: {:?}", + from.get_usage(), + from.get_volume_condition() + ); + Self { data: result } + } +} diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index a3d1da72a..fd4dde255 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -23,8 +23,8 @@ pub use types::{ ReseedRandomDevRequest, ResizeVolumeRequest, Route, Routes, SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, StatsContainerResponse, Storage, TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, - VersionCheckResponse, WaitProcessRequest, WaitProcessResponse, WriteStreamRequest, - WriteStreamResponse, + VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, + WaitProcessResponse, WriteStreamRequest, WriteStreamResponse, }; use anyhow::Result; @@ -88,4 +88,5 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { async fn get_oom_event(&self, req: Empty) -> Result; async fn get_ip_tables(&self, req: GetIPTablesRequest) -> Result; async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result; + async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result; } diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index 0cd509ff5..7937166bd 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -568,6 +568,16 @@ pub struct ResizeVolumeRequest { pub size: u64, } +#[derive(PartialEq, Clone, Default, Debug)] +pub struct VolumeStatsRequest { + pub volume_guest_path: String, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct VolumeStatsResponse { + pub data: String, +} + #[cfg(test)] mod test { use std::convert::TryFrom; diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index 3fee8165d..7b1bcd8d5 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -26,4 +26,5 @@ pub trait Sandbox: Send + Sync { // utils async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result>; async fn get_iptables(&self, is_ipv6: bool) -> Result>; + async fn direct_volume_stats(&self, volume_path: &str) -> Result; } diff --git a/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs b/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs index dea9725a5..b2ee0fd13 100644 --- a/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs +++ b/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs @@ -11,8 +11,9 @@ use anyhow::{anyhow, Result}; use common::Sandbox; use hyper::{Body, Method, Request, Response, StatusCode}; use std::sync::Arc; +use url::Url; -use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL}; +use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL, DIRECT_VOLUMN_PATH_KEY, DIRECT_VOLUMN_STATS_URL,}; // main router for response, this works as a multiplexer on // http arrival which invokes the corresponding handler function @@ -34,6 +35,7 @@ pub(crate) async fn handler_mux( (&Method::PUT, IP6_TABLE_URL) | (&Method::GET, IP6_TABLE_URL) => { ipv6_table_handler(sandbox, req).await } + (&Method::POST, DIRECT_VOLUMN_STATS_URL) => direct_volume_stats_handler(sandbox, req).await, _ => Ok(not_found(req).await), } } @@ -101,3 +103,24 @@ async fn generic_ip_table_handler( _ => Err(anyhow!("IP Tables only takes PUT and GET")), } } + +async fn direct_volume_stats_handler( + sandbox: Arc, + req: Request, +) -> Result> { + let params = Url::parse(&req.uri().to_string()) + .unwrap() + .query_pairs() + .into_owned() + .collect::>(); + let volume_path = params.get(DIRECT_VOLUMN_PATH_KEY).unwrap(); + let result = sandbox.direct_volume_stats(volume_path).await; + match result { + Ok(stats) => Ok(Response::new(Body::from(stats))), + Err(e) => { + let builder = Response::builder().status(StatusCode::INTERNAL_SERVER_ERROR); + let resp = builder.body(Body::from(e.to_string())).unwrap(); + Ok(resp) + } + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 0d6e4765e..41fdab15f 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use agent::{ self, kata::KataAgent, types::KernelModule, Agent, GetIPTablesRequest, SetIPTablesRequest, + VolumeStatsRequest, }; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -329,6 +330,14 @@ impl Sandbox for VirtSandbox { self.agent.agent_sock().await } + async fn direct_volume_stats(&self, volume_guest_path: &str) -> Result { + let req: agent::VolumeStatsRequest = VolumeStatsRequest { + volume_guest_path: volume_guest_path.to_string(), + }; + let result = self.agent.get_volume_stats(req).await?.data; + Ok(result) + } + async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result> { info!(sl!(), "sb: set_iptables invoked"); let req = SetIPTablesRequest { is_ipv6, data }; From e029988bc2b7f0ea978d58f448d4f284b2f18db4 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Thu, 9 Mar 2023 15:51:51 +0800 Subject: [PATCH 05/40] bugfix: add get_ns_path API for Hypervisor For external hypervisors(qemu, cloud-hypervisor, ...), the ns they launch vm in is different from internal hypervisor(dragonball). And when we doing CreateContainer hook, we will rely on the netns path. So we add a get_ns_path API. Fixes: #6442 Signed-off-by: Yushuo --- .../crates/hypervisor/src/ch/inner_hypervisor.rs | 15 ++++++++++++++- src/runtime-rs/crates/hypervisor/src/ch/mod.rs | 5 +++++ .../hypervisor/src/dragonball/inner_hypervisor.rs | 5 +++++ .../crates/hypervisor/src/dragonball/mod.rs | 5 +++++ .../hypervisor/src/dragonball/vmm_instance.rs | 7 +++++++ src/runtime-rs/crates/hypervisor/src/lib.rs | 1 + .../crates/hypervisor/src/qemu/inner.rs | 5 +++++ src/runtime-rs/crates/hypervisor/src/qemu/mod.rs | 5 +++++ .../src/container_manager/manager.rs | 3 ++- 9 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 9cd05de91..3a2631c35 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -473,7 +473,20 @@ impl CloudHypervisorInner { } pub(crate) async fn get_vmm_master_tid(&self) -> Result { - todo!() + if let Some(pid) = self.pid { + Ok(pid) + } else { + Err(anyhow!("could not get vmm master tid")) + } + } + + pub(crate) async fn get_ns_path(&self) -> Result { + if let Some(pid) = self.pid { + let ns_path = format!("/proc/{}/ns", pid); + Ok(ns_path) + } else { + Err(anyhow!("could not get ns path")) + } } pub(crate) async fn check(&self) -> Result<()> { diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 7805d2601..f8f44710e 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -123,6 +123,11 @@ impl Hypervisor for CloudHypervisor { inner.get_vmm_master_tid().await } + async fn get_ns_path(&self) -> Result { + let inner = self.inner.read().await; + inner.get_ns_path().await + } + async fn check(&self) -> Result<()> { let inner = self.inner.read().await; inner.check().await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs index fc3bea7a5..adc016807 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs @@ -132,6 +132,11 @@ impl DragonballInner { Ok(master_tid) } + pub(crate) async fn get_ns_path(&self) -> Result { + let ns_path = self.vmm_instance.get_ns_path(); + Ok(ns_path) + } + pub(crate) async fn check(&self) -> Result<()> { Ok(()) } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index d096abf54..951af2bc7 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -122,6 +122,11 @@ impl Hypervisor for Dragonball { inner.get_vmm_master_tid().await } + async fn get_ns_path(&self) -> Result { + let inner = self.inner.read().await; + inner.get_ns_path().await + } + async fn check(&self) -> Result<()> { let inner = self.inner.read().await; inner.check().await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 8e7d6332f..4c40b40c5 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -81,6 +81,13 @@ impl VmmInstance { result } + pub fn get_ns_path(&self) -> String { + let info_binding = self.vmm_shared_info.clone(); + let info = info_binding.read().unwrap(); + let result = format!("/proc/{}/task/{}/ns", info.pid, info.master_tid); + result + } + pub fn get_vcpu_tids(&self) -> Vec<(u8, u32)> { let info = self.vmm_shared_info.clone(); let result = info.read().unwrap().tids.clone(); diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index e7f18e513..364cce0f2 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -88,6 +88,7 @@ pub trait Hypervisor: Send + Sync { async fn get_thread_ids(&self) -> Result; async fn get_pids(&self) -> Result>; async fn get_vmm_master_tid(&self) -> Result; + async fn get_ns_path(&self) -> Result; async fn cleanup(&self) -> Result<()>; async fn check(&self) -> Result<()>; async fn get_jailer_root(&self) -> Result; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 995a9c590..0c0efbde6 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -94,6 +94,11 @@ impl QemuInner { todo!() } + pub(crate) async fn get_ns_path(&self) -> Result { + info!(sl!(), "QemuInner::get_ns_path()"); + todo!() + } + pub(crate) async fn cleanup(&self) -> Result<()> { info!(sl!(), "QemuInner::cleanup()"); todo!() diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 0192e2a8e..c73fb23d4 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -108,6 +108,11 @@ impl Hypervisor for Qemu { inner.get_vmm_master_tid().await } + async fn get_ns_path(&self) -> Result { + let inner = self.inner.read().await; + inner.get_ns_path().await + } + async fn cleanup(&self) -> Result<()> { let inner = self.inner.read().await; inner.cleanup().await diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index 53bd36894..f85e7901a 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -73,7 +73,8 @@ impl ContainerManager for VirtContainerManager { // * should be run after the vm is started, before container is created, and after CreateRuntime Hooks // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createcontainer-hooks let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?; - let vmm_netns_path = format!("/proc/{}/task/{}/ns/{}", self.pid, vmm_master_tid, "net"); + let vmm_ns_path = self.hypervisor.get_ns_path().await?; + let vmm_netns_path = format!("{}/{}", vmm_ns_path, "net"); let state = oci::State { version: spec.version.clone(), id: config.container_id.clone(), From 30e235f0a1ec420d796712e56c1669785316e24a Mon Sep 17 00:00:00 2001 From: Tingzhou Yuan Date: Tue, 6 Dec 2022 07:08:43 +0000 Subject: [PATCH 06/40] runtime-rs: impl volume-resize trait for sandbox Implements resize-volume handlers in shim-mgmt, trait for sandbox and add RPC calls to agent. Note the actual rpc handler for the resize request is currently not implemented, refer to issue #3694. Fixes #5369 Signed-off-by: Tingzhou Yuan --- .../shim-interface/src/shim_mgmt/client.rs | 6 +-- src/runtime-rs/Cargo.lock | 1 + src/runtime-rs/crates/agent/src/kata/agent.rs | 3 +- src/runtime-rs/crates/agent/src/kata/trans.rs | 24 ++++++++--- src/runtime-rs/crates/agent/src/lib.rs | 1 + src/runtime-rs/crates/runtimes/Cargo.toml | 2 + .../crates/runtimes/common/src/sandbox.rs | 1 + .../crates/runtimes/src/shim_mgmt/handlers.rs | 42 ++++++++++++++----- .../runtimes/virt_container/src/sandbox.rs | 16 ++++++- 9 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/libs/shim-interface/src/shim_mgmt/client.rs b/src/libs/shim-interface/src/shim_mgmt/client.rs index ace72c1d6..f112903c5 100644 --- a/src/libs/shim-interface/src/shim_mgmt/client.rs +++ b/src/libs/shim-interface/src/shim_mgmt/client.rs @@ -53,7 +53,7 @@ impl MgmtClient { .method(Method::GET) .uri(url) .body(Body::empty())?; - return self.send_request(req).await; + self.send_request(req).await } /// The HTTP Post method for client @@ -72,7 +72,7 @@ impl MgmtClient { .uri(url) .header("content-type", content_type) .body(body)?; - return self.send_request(req).await; + self.send_request(req).await } /// The http PUT method for client @@ -82,7 +82,7 @@ impl MgmtClient { .method(Method::PUT) .uri(url) .body(Body::from(data))?; - return self.send_request(req).await; + self.send_request(req).await } async fn send_request(&self, req: Request) -> Result> { diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index e2525363b..3e2d68fd1 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -2452,6 +2452,7 @@ dependencies = [ name = "runtimes" version = "0.1.0" dependencies = [ + "agent", "anyhow", "common", "hyper", diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index 6baab2dd1..aa0df0857 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -116,5 +116,6 @@ impl_agent!( get_oom_event | crate::Empty | crate::OomEventResponse | Some(0), get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None, set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None, - get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None + get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None, + resize_volume | crate::ResizeVolumeRequest | crate::Empty | None ); diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index f2f725535..7d33a0992 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -20,11 +20,12 @@ use crate::{ GetIPTablesResponse, GuestDetailsResponse, HealthCheckResponse, HugetlbStats, IPAddress, IPFamily, Interface, Interfaces, KernelModule, MemHotplugByProbeRequest, MemoryData, MemoryStats, NetworkStats, OnlineCPUMemRequest, PidsStats, ReadStreamRequest, - ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, - SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, SignalProcessRequest, - StatsContainerResponse, Storage, StringUser, ThrottlingData, TtyWinResizeRequest, - UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, - VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, WriteStreamRequest, + ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, ResizeVolumeRequest, + Route, Routes, SetGuestDateTimeRequest, SetIPTablesRequest, SetIPTablesResponse, + SignalProcessRequest, StatsContainerResponse, Storage, StringUser, ThrottlingData, + TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, + VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, + WriteStreamRequest, }, OomEventResponse, WaitProcessResponse, WriteStreamResponse, }; @@ -860,10 +861,21 @@ impl From for agent::VolumeStatsRequest { impl From for VolumeStatsResponse { fn from(from: csi::VolumeStatsResponse) -> Self { let result: String = format!( - "Usage: {:?}\nVolume Condition: {:?}", + "Usage: {:?} Volume Condition: {:?}", from.get_usage(), from.get_volume_condition() ); Self { data: result } } } + +impl From for agent::ResizeVolumeRequest { + fn from(from: ResizeVolumeRequest) -> Self { + Self { + volume_guest_path: from.volume_guest_path, + size: from.size, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index fd4dde255..ea3bab78f 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -89,4 +89,5 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { async fn get_ip_tables(&self, req: GetIPTablesRequest) -> Result; async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result; async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result; + async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result; } diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml index 142c44ed0..3a6ab0a1b 100644 --- a/src/runtime-rs/crates/runtimes/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/Cargo.toml @@ -15,7 +15,9 @@ hyper = { version = "0.14.20", features = ["stream", "server", "http1"] } hyperlocal = "0.8" serde_json = "1.0.88" nix = "0.25.0" +url = "2.3.1" +agent = { path = "../agent" } common = { path = "./common" } kata-types = { path = "../../../libs/kata-types" } kata-sys-util = { path = "../../../libs/kata-sys-util" } diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index 7b1bcd8d5..0aee04922 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -27,4 +27,5 @@ pub trait Sandbox: Send + Sync { async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result>; async fn get_iptables(&self, is_ipv6: bool) -> Result>; async fn direct_volume_stats(&self, volume_path: &str) -> Result; + async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()>; } diff --git a/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs b/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs index b2ee0fd13..b0c79cd40 100644 --- a/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs +++ b/src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs @@ -7,13 +7,17 @@ // This defines the handlers corresponding to the url when a request is sent to destined url, // the handler function should be invoked, and the corresponding data will be in the response -use anyhow::{anyhow, Result}; +use agent::ResizeVolumeRequest; +use anyhow::{anyhow, Context, Result}; use common::Sandbox; use hyper::{Body, Method, Request, Response, StatusCode}; use std::sync::Arc; use url::Url; -use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL, DIRECT_VOLUMN_PATH_KEY, DIRECT_VOLUMN_STATS_URL,}; +use shim_interface::shim_mgmt::{ + AGENT_URL, DIRECT_VOLUME_PATH_KEY, DIRECT_VOLUME_RESIZE_URL, DIRECT_VOLUME_STATS_URL, + IP6_TABLE_URL, IP_TABLE_URL, +}; // main router for response, this works as a multiplexer on // http arrival which invokes the corresponding handler function @@ -35,7 +39,10 @@ pub(crate) async fn handler_mux( (&Method::PUT, IP6_TABLE_URL) | (&Method::GET, IP6_TABLE_URL) => { ipv6_table_handler(sandbox, req).await } - (&Method::POST, DIRECT_VOLUMN_STATS_URL) => direct_volume_stats_handler(sandbox, req).await, + (&Method::POST, DIRECT_VOLUME_STATS_URL) => direct_volume_stats_handler(sandbox, req).await, + (&Method::POST, DIRECT_VOLUME_RESIZE_URL) => { + direct_volume_resize_handler(sandbox, req).await + } _ => Ok(not_found(req).await), } } @@ -109,18 +116,33 @@ async fn direct_volume_stats_handler( req: Request, ) -> Result> { let params = Url::parse(&req.uri().to_string()) - .unwrap() + .map_err(|e| anyhow!(e))? .query_pairs() .into_owned() .collect::>(); - let volume_path = params.get(DIRECT_VOLUMN_PATH_KEY).unwrap(); + let volume_path = params + .get(DIRECT_VOLUME_PATH_KEY) + .context("shim-mgmt: volume path key not found in request params")?; let result = sandbox.direct_volume_stats(volume_path).await; match result { Ok(stats) => Ok(Response::new(Body::from(stats))), - Err(e) => { - let builder = Response::builder().status(StatusCode::INTERNAL_SERVER_ERROR); - let resp = builder.body(Body::from(e.to_string())).unwrap(); - Ok(resp) - } + _ => Err(anyhow!("handler: Failed to get volume stats")), + } +} + +async fn direct_volume_resize_handler( + sandbox: Arc, + req: Request, +) -> Result> { + let body = hyper::body::to_bytes(req.into_body()).await?; + + // unserialize json body into resizeRequest struct + let resize_req: ResizeVolumeRequest = + serde_json::from_slice(&body).context("shim-mgmt: deserialize resizeRequest failed")?; + let result = sandbox.direct_volume_resize(resize_req).await; + + match result { + Ok(_) => Ok(Response::new(Body::from(""))), + _ => Err(anyhow!("handler: Failed to resize volume")), } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 41fdab15f..1b170ce31 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -334,8 +334,20 @@ impl Sandbox for VirtSandbox { let req: agent::VolumeStatsRequest = VolumeStatsRequest { volume_guest_path: volume_guest_path.to_string(), }; - let result = self.agent.get_volume_stats(req).await?.data; - Ok(result) + let result = self + .agent + .get_volume_stats(req) + .await + .context("sandbox: failed to process direct volume stats query")?; + Ok(result.data) + } + + async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()> { + self.agent + .resize_volume(resize_req) + .await + .context("sandbox: failed to resize direct-volume")?; + Ok(()) } async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result> { From e7bca62c32fbab982b2f1032b6c6b0b099cd4a77 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Wed, 8 Mar 2023 17:16:26 +0800 Subject: [PATCH 07/40] bugfix: modify tty_win info in runtime when handling ResizePtyRequest Currently, we only create the new exec process in runtime, this will cause error when the following requests needing to be handled: - Task: exec process - Task: resize process pty - ... The agent do not do_exec_process when we handle ExecProcess, thus we can not find any process information in the guest when we handle ResizeProcessPty. This will report an error. In this commit, the handling process is modified to the: * Modify process tty_win information in runtime * If the exec process is not running, we just return. And the truly pty_resize will happen when start_process Fixes: #6248 Signed-off-by: Yushuo --- .../src/container_manager/container.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 501002e42..2f9f03b0e 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -344,20 +344,33 @@ impl Container { height: u32, ) -> Result<()> { let logger = logger_with_process(process); - let inner = self.inner.read().await; + let mut inner = self.inner.write().await; if inner.init_process.get_status().await != ProcessStatus::Running { warn!(logger, "container is not running"); return Ok(()); } - self.agent - .tty_win_resize(agent::TtyWinResizeRequest { - process_id: process.clone().into(), - row: height, - column: width, - }) - .await - .context("resize pty")?; - Ok(()) + + if process.exec_id.is_empty() { + inner.init_process.height = height; + inner.init_process.width = width; + } else if let Some(exec) = inner.exec_processes.get_mut(&process.exec_id) { + exec.process.height = height; + exec.process.width = width; + + // for some case, resize_pty request should be handled while the process has not been started in agent + // just return here, and truly resize_pty will happen in start_process + if exec.process.get_status().await != ProcessStatus::Running { + return Ok(()); + } + } else { + return Err(anyhow!( + "could not find process {} in container {}", + process.exec_id(), + process.container_id() + )); + } + + inner.win_resize_process(process, height, width).await } pub async fn stats(&self) -> Result> { From 844bf053b2aa9a5de354dbad43d1bf747f46c536 Mon Sep 17 00:00:00 2001 From: Li Hongyu Date: Tue, 22 Nov 2022 11:54:43 +0000 Subject: [PATCH 08/40] runtime-rs: add the missing default trait Some structs in the runtime-rs don't implement Default trait. This commit adds the missing Default. Fixes: #5463 Signed-off-by: Li Hongyu --- src/libs/kata-types/src/config/agent.rs | 24 +++++++++++++++++-- src/libs/kata-types/src/config/default.rs | 1 + src/runtime-rs/crates/agent/src/types.rs | 13 +++++++++- .../src/container_manager/container_inner.rs | 7 ++++-- .../runtimes/virt_container/src/sandbox.rs | 19 ++------------- src/runtime-rs/crates/shim/src/args.rs | 7 +++--- src/runtime-rs/crates/shim/src/shim.rs | 8 +++---- src/runtime-rs/crates/shim/src/shim_start.rs | 20 ++++++++-------- 8 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 875e1835a..98ec142f4 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -9,14 +9,16 @@ use crate::config::{ConfigOps, TomlConfig}; pub use vendor::AgentVendor; -use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT}; +use super::default::{ + DEFAULT_AGENT_DIAL_TIMEOUT_MS, DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT, +}; use crate::eother; /// agent name of Kata agent. pub const AGENT_NAME_KATA: &str = "kata"; /// Kata agent configuration information. -#[derive(Debug, Default, Deserialize, Serialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct Agent { /// If enabled, the agent will log additional debug messages to the system log. #[serde(default, rename = "enable_debug")] @@ -81,6 +83,24 @@ pub struct Agent { pub container_pipe_size: u32, } +impl std::default::Default for Agent { + fn default() -> Self { + Self { + debug: true, + enable_tracing: false, + debug_console_enabled: false, + server_port: DEFAULT_AGENT_VSOCK_PORT, + log_port: DEFAULT_AGENT_LOG_PORT, + dial_timeout_ms: DEFAULT_AGENT_DIAL_TIMEOUT_MS, + reconnect_timeout_ms: 3_000, + request_timeout_ms: 30_000, + health_check_request_timeout_ms: 90_000, + kernel_modules: Default::default(), + container_pipe_size: 0, + } + } +} + fn default_server_port() -> u32 { DEFAULT_AGENT_VSOCK_PORT } diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 7108c4add..1d5e2b4d3 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -24,6 +24,7 @@ pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024; pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025; pub const DEFAULT_AGENT_DBG_CONSOLE_PORT: u32 = 1026; pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA; +pub const DEFAULT_AGENT_DIAL_TIMEOUT_MS: u32 = 10; pub const DEFAULT_RUNTIME_NAME: &str = RUNTIME_NAME_VIRTCONTAINER; pub const DEFAULT_HYPERVISOR: &str = HYPERVISOR_NAME_DRAGONBALL; diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index 0cd509ff5..64f19c1b3 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -9,6 +9,8 @@ use std::convert::TryFrom; use serde::{Deserialize, Serialize}; +pub const DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT: u32 = 10; + #[derive(PartialEq, Clone, Default)] pub struct Empty {} @@ -164,7 +166,7 @@ impl ContainerProcessID { } } -#[derive(PartialEq, Clone, Debug, Default)] +#[derive(PartialEq, Clone, Debug)] pub struct RemoveContainerRequest { pub container_id: String, pub timeout: u32, @@ -179,6 +181,15 @@ impl RemoveContainerRequest { } } +impl std::default::Default for RemoveContainerRequest { + fn default() -> Self { + Self { + container_id: "".to_string(), + timeout: DEFAULT_REMOVE_CONTAINER_REQUEST_TIMEOUT, + } + } +} + #[derive(PartialEq, Clone, Default)] pub struct SignalProcessRequest { pub process_id: ContainerProcessID, diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index b041af076..bb6c2ed07 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -164,9 +164,12 @@ impl ContainerInner { let exit_status = self.get_exit_status().await; let _locked_exit_status = exit_status.read().await; info!(self.logger, "container terminated"); - let timeout: u32 = 10; + let remove_request = agent::RemoveContainerRequest { + container_id: cid.to_string(), + ..Default::default() + }; self.agent - .remove_container(agent::RemoveContainerRequest::new(cid, timeout)) + .remove_container(remove_request) .await .or_else(|e| { if force { diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 0d6e4765e..e689f05df 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -18,10 +18,7 @@ use common::{ use containerd_shim_protos::events::task::TaskOOM; use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL}; use kata_sys_util::hooks::HookStates; -use kata_types::config::{ - default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT}, - TomlConfig, -}; +use kata_types::config::TomlConfig; use resource::{ manager::ManagerArgs, network::{NetworkConfig, NetworkWithNetNsConfig}, @@ -380,19 +377,7 @@ impl Persist for VirtSandbox { HYPERVISOR_DRAGONBALL => Ok(Arc::new(Dragonball::restore((), h).await?)), _ => Err(anyhow!("Unsupported hypervisor {}", &h.hypervisor_type)), }?; - let agent = Arc::new(KataAgent::new(kata_types::config::Agent { - debug: true, - enable_tracing: false, - server_port: DEFAULT_AGENT_VSOCK_PORT, - log_port: DEFAULT_AGENT_LOG_PORT, - dial_timeout_ms: 10, - reconnect_timeout_ms: 3_000, - request_timeout_ms: 30_000, - health_check_request_timeout_ms: 90_000, - kernel_modules: Default::default(), - container_pipe_size: 0, - debug_console_enabled: false, - })); + let agent = Arc::new(KataAgent::new(kata_types::config::Agent::default())); let sid = sandbox_args.sid; let args = ManagerArgs { sid: sid.clone(), diff --git a/src/runtime-rs/crates/shim/src/args.rs b/src/runtime-rs/crates/shim/src/args.rs index e759c458c..1ab5b8afa 100644 --- a/src/runtime-rs/crates/shim/src/args.rs +++ b/src/runtime-rs/crates/shim/src/args.rs @@ -117,12 +117,11 @@ mod tests { result: Result<()>, } - let default_id = "1dfc0567".to_string(); - let default_namespace = "ns1".to_string(); + let default_id = "default_id".to_string(); + let default_namespace = "default_namespace".to_string(); let default_address = bind_address.to_string(); let default_publish_binary = "containerd".to_string(); let default_bundle = path.to_string(); - let default_debug = false; let mut arg = Args { id: default_id.clone(), @@ -130,7 +129,7 @@ mod tests { address: default_address.clone(), publish_binary: default_publish_binary.clone(), bundle: default_bundle.clone(), - debug: default_debug, + ..Default::default() }; let tests = &[ diff --git a/src/runtime-rs/crates/shim/src/shim.rs b/src/runtime-rs/crates/shim/src/shim.rs index 71941aee8..acd76d6bf 100644 --- a/src/runtime-rs/crates/shim/src/shim.rs +++ b/src/runtime-rs/crates/shim/src/shim.rs @@ -93,12 +93,12 @@ mod tests { std::env::set_current_dir(bundle_path).unwrap(); let args = Args { - id: "1dfc0567".to_string(), - namespace: "test_namespace".into(), - address: "containerd_socket".into(), + id: "default_id".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let executor = ShimExecutor::new(args); diff --git a/src/runtime-rs/crates/shim/src/shim_start.rs b/src/runtime-rs/crates/shim/src/shim_start.rs index 67fa8c5be..06a7f3ae5 100644 --- a/src/runtime-rs/crates/shim/src/shim_start.rs +++ b/src/runtime-rs/crates/shim/src/shim_start.rs @@ -153,12 +153,12 @@ mod tests { std::env::set_current_dir(bundle_path).unwrap(); let args = Args { - id: "sandbox1".into(), - namespace: "ns".into(), - address: "address".into(), + id: "default_id".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let mut executor = ShimExecutor::new(args); @@ -185,11 +185,11 @@ mod tests { let args = Args { id: sandbox_id.to_owned(), - namespace: "ns1".into(), - address: "containerd_socket".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let executor = ShimExecutor::new(args); @@ -204,11 +204,11 @@ mod tests { let args = Args { id: container_id, - namespace: "ns1".into(), - address: "containerd_socket".into(), + namespace: "default_namespace".into(), + address: "default_address".into(), publish_binary: "containerd".into(), bundle: bundle_path2.to_str().unwrap().into(), - debug: false, + ..Default::default() }; let executor2 = ShimExecutor::new(args); From a6c67a161e918b512c21fc048b33fd85b5586cbd Mon Sep 17 00:00:00 2001 From: Sidhartha Mani Date: Fri, 3 Mar 2023 17:48:36 -0800 Subject: [PATCH 09/40] runtime: add support for ephemeral mounts to occupy entire sandbox memory On hotplug of memory as containers are started, remount all ephemeral mounts with size option set to the total sandbox memory Fixes: #6417 Signed-off-by: Sidhartha Mani --- src/runtime/virtcontainers/agent.go | 3 + src/runtime/virtcontainers/kata_agent.go | 87 ++- src/runtime/virtcontainers/mock_agent.go | 5 + .../pkg/agent/protocols/grpc/agent.pb.go | 683 +++++++++++------ .../pkg/agent/protocols/grpc/oci.pb.go | 720 ++++++++++++------ src/runtime/virtcontainers/pkg/mock/mock.go | 4 + src/runtime/virtcontainers/sandbox.go | 62 ++ src/runtime/virtcontainers/sandbox_test.go | 59 ++ 8 files changed, 1116 insertions(+), 507 deletions(-) diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index dc5d1d9ad..6ff8f91ac 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -153,6 +153,9 @@ type agent interface { // listInterfaces will tell the agent to list interfaces of an existed Sandbox listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error) + // updateEphemeralMounts will tell the agent to update tmpfs mounts in the Sandbox. + updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error + // updateRoutes will tell the agent to update route table for an existed Sandbox. updateRoutes(ctx context.Context, routes []*pbTypes.Route) ([]*pbTypes.Route, error) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 60661fcd3..66784404f 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -112,40 +112,41 @@ var ( ) const ( - grpcCheckRequest = "grpc.CheckRequest" - grpcExecProcessRequest = "grpc.ExecProcessRequest" - grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" - grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" - grpcCreateContainerRequest = "grpc.CreateContainerRequest" - grpcStartContainerRequest = "grpc.StartContainerRequest" - grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" - grpcSignalProcessRequest = "grpc.SignalProcessRequest" - grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" - grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" - grpcListInterfacesRequest = "grpc.ListInterfacesRequest" - grpcListRoutesRequest = "grpc.ListRoutesRequest" - grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" - grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" - grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" - grpcWaitProcessRequest = "grpc.WaitProcessRequest" - grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" - grpcWriteStreamRequest = "grpc.WriteStreamRequest" - grpcCloseStdinRequest = "grpc.CloseStdinRequest" - grpcStatsContainerRequest = "grpc.StatsContainerRequest" - grpcPauseContainerRequest = "grpc.PauseContainerRequest" - grpcResumeContainerRequest = "grpc.ResumeContainerRequest" - grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" - grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" - grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" - grpcCopyFileRequest = "grpc.CopyFileRequest" - grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" - grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" - grpcGetMetricsRequest = "grpc.GetMetricsRequest" - grpcAddSwapRequest = "grpc.AddSwapRequest" - grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" - grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" - grpcGetIPTablesRequest = "grpc.GetIPTablesRequest" - grpcSetIPTablesRequest = "grpc.SetIPTablesRequest" + grpcCheckRequest = "grpc.CheckRequest" + grpcExecProcessRequest = "grpc.ExecProcessRequest" + grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" + grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" + grpcCreateContainerRequest = "grpc.CreateContainerRequest" + grpcStartContainerRequest = "grpc.StartContainerRequest" + grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" + grpcSignalProcessRequest = "grpc.SignalProcessRequest" + grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" + grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" + grpcUpdateEphemeralMountsRequest = "grpc.UpdateEphemeralMountsRequest" + grpcListInterfacesRequest = "grpc.ListInterfacesRequest" + grpcListRoutesRequest = "grpc.ListRoutesRequest" + grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" + grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" + grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" + grpcWaitProcessRequest = "grpc.WaitProcessRequest" + grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" + grpcWriteStreamRequest = "grpc.WriteStreamRequest" + grpcCloseStdinRequest = "grpc.CloseStdinRequest" + grpcStatsContainerRequest = "grpc.StatsContainerRequest" + grpcPauseContainerRequest = "grpc.PauseContainerRequest" + grpcResumeContainerRequest = "grpc.ResumeContainerRequest" + grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" + grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" + grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" + grpcCopyFileRequest = "grpc.CopyFileRequest" + grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" + grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" + grpcGetMetricsRequest = "grpc.GetMetricsRequest" + grpcAddSwapRequest = "grpc.AddSwapRequest" + grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" + grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" + grpcGetIPTablesRequest = "grpc.GetIPTablesRequest" + grpcSetIPTablesRequest = "grpc.SetIPTablesRequest" ) // newKataAgent returns an agent from an agent type. @@ -607,6 +608,21 @@ func (k *kataAgent) updateRoutes(ctx context.Context, routes []*pbTypes.Route) ( return nil, nil } +func (k *kataAgent) updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error { + if storages != nil { + storagesReq := &grpc.UpdateEphemeralMountsRequest{ + Storages: storages, + } + + if _, err := k.sendReq(ctx, storagesReq); err != nil { + k.Logger().WithError(err).Error("update mounts request failed") + return err + } + return nil + } + return nil +} + func (k *kataAgent) addARPNeighbors(ctx context.Context, neighs []*pbTypes.ARPNeighbor) error { if neighs != nil { neighsReq := &grpc.AddARPNeighborsRequest{ @@ -1952,6 +1968,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest)) } + k.reqHandlers[grpcUpdateEphemeralMountsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.UpdateEphemeralMounts(ctx, req.(*grpc.UpdateEphemeralMountsRequest)) + } k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest)) } diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index 67ec52319..72e865c4a 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -121,6 +121,11 @@ func (n *mockAgent) listRoutes(ctx context.Context) ([]*pbTypes.Route, error) { return nil, nil } +// updateEphemeralMounts is the Noop agent updateEphemeralMounts implementation. It does nothing. +func (n *mockAgent) updateEphemeralMounts(ctx context.Context, storages []*grpc.Storage) error { + return nil +} + // check is the Noop agent health checker. It does nothing. func (n *mockAgent) check(ctx context.Context) error { return nil diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index 747eca792..36af23886 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1527,6 +1527,45 @@ func (m *UpdateRoutesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_UpdateRoutesRequest proto.InternalMessageInfo +type UpdateEphemeralMountsRequest struct { + Storages []*Storage `protobuf:"bytes,1,rep,name=storages,proto3" json:"storages,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UpdateEphemeralMountsRequest) Reset() { *m = UpdateEphemeralMountsRequest{} } +func (*UpdateEphemeralMountsRequest) ProtoMessage() {} +func (*UpdateEphemeralMountsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{36} +} +func (m *UpdateEphemeralMountsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *UpdateEphemeralMountsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_UpdateEphemeralMountsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *UpdateEphemeralMountsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UpdateEphemeralMountsRequest.Merge(m, src) +} +func (m *UpdateEphemeralMountsRequest) XXX_Size() int { + return m.Size() +} +func (m *UpdateEphemeralMountsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_UpdateEphemeralMountsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_UpdateEphemeralMountsRequest proto.InternalMessageInfo + type ListInterfacesRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1536,7 +1575,7 @@ type ListInterfacesRequest struct { func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } func (*ListInterfacesRequest) ProtoMessage() {} func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{36} + return fileDescriptor_712ce9a559fda969, []int{37} } func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1574,7 +1613,7 @@ type ListRoutesRequest struct { func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } func (*ListRoutesRequest) ProtoMessage() {} func (*ListRoutesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{37} + return fileDescriptor_712ce9a559fda969, []int{38} } func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1613,7 +1652,7 @@ type ARPNeighbors struct { func (m *ARPNeighbors) Reset() { *m = ARPNeighbors{} } func (*ARPNeighbors) ProtoMessage() {} func (*ARPNeighbors) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{38} + return fileDescriptor_712ce9a559fda969, []int{39} } func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1652,7 +1691,7 @@ type AddARPNeighborsRequest struct { func (m *AddARPNeighborsRequest) Reset() { *m = AddARPNeighborsRequest{} } func (*AddARPNeighborsRequest) ProtoMessage() {} func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{39} + return fileDescriptor_712ce9a559fda969, []int{40} } func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1691,7 +1730,7 @@ type GetIPTablesRequest struct { func (m *GetIPTablesRequest) Reset() { *m = GetIPTablesRequest{} } func (*GetIPTablesRequest) ProtoMessage() {} func (*GetIPTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{40} + return fileDescriptor_712ce9a559fda969, []int{41} } func (m *GetIPTablesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1731,7 +1770,7 @@ type GetIPTablesResponse struct { func (m *GetIPTablesResponse) Reset() { *m = GetIPTablesResponse{} } func (*GetIPTablesResponse) ProtoMessage() {} func (*GetIPTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{41} + return fileDescriptor_712ce9a559fda969, []int{42} } func (m *GetIPTablesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1773,7 +1812,7 @@ type SetIPTablesRequest struct { func (m *SetIPTablesRequest) Reset() { *m = SetIPTablesRequest{} } func (*SetIPTablesRequest) ProtoMessage() {} func (*SetIPTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{42} + return fileDescriptor_712ce9a559fda969, []int{43} } func (m *SetIPTablesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1803,7 +1842,7 @@ func (m *SetIPTablesRequest) XXX_DiscardUnknown() { var xxx_messageInfo_SetIPTablesRequest proto.InternalMessageInfo type SetIPTablesResponse struct { - // raw stdout from iptables-restore or ip6tables-restore ? + // raw stdout from iptables-restore or ip6tables-restore Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1813,7 +1852,7 @@ type SetIPTablesResponse struct { func (m *SetIPTablesResponse) Reset() { *m = SetIPTablesResponse{} } func (*SetIPTablesResponse) ProtoMessage() {} func (*SetIPTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{43} + return fileDescriptor_712ce9a559fda969, []int{44} } func (m *SetIPTablesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1859,7 +1898,7 @@ type OnlineCPUMemRequest struct { func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } func (*OnlineCPUMemRequest) ProtoMessage() {} func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{44} + return fileDescriptor_712ce9a559fda969, []int{45} } func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1899,7 +1938,7 @@ type ReseedRandomDevRequest struct { func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } func (*ReseedRandomDevRequest) ProtoMessage() {} func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{45} + return fileDescriptor_712ce9a559fda969, []int{46} } func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1949,7 +1988,7 @@ type AgentDetails struct { func (m *AgentDetails) Reset() { *m = AgentDetails{} } func (*AgentDetails) ProtoMessage() {} func (*AgentDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{46} + return fileDescriptor_712ce9a559fda969, []int{47} } func (m *AgentDetails) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1995,7 +2034,7 @@ type GuestDetailsRequest struct { func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } func (*GuestDetailsRequest) ProtoMessage() {} func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{47} + return fileDescriptor_712ce9a559fda969, []int{48} } func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2037,7 +2076,7 @@ type GuestDetailsResponse struct { func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } func (*GuestDetailsResponse) ProtoMessage() {} func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{48} + return fileDescriptor_712ce9a559fda969, []int{49} } func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2078,7 +2117,7 @@ type MemHotplugByProbeRequest struct { func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } func (*MemHotplugByProbeRequest) ProtoMessage() {} func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{49} + return fileDescriptor_712ce9a559fda969, []int{50} } func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2120,7 +2159,7 @@ type SetGuestDateTimeRequest struct { func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } func (*SetGuestDateTimeRequest) ProtoMessage() {} func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{50} + return fileDescriptor_712ce9a559fda969, []int{51} } func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2166,7 +2205,7 @@ type FSGroup struct { func (m *FSGroup) Reset() { *m = FSGroup{} } func (*FSGroup) ProtoMessage() {} func (*FSGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{51} + return fileDescriptor_712ce9a559fda969, []int{52} } func (m *FSGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2234,7 +2273,7 @@ type Storage struct { func (m *Storage) Reset() { *m = Storage{} } func (*Storage) ProtoMessage() {} func (*Storage) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{52} + return fileDescriptor_712ce9a559fda969, []int{53} } func (m *Storage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2305,7 +2344,7 @@ type Device struct { func (m *Device) Reset() { *m = Device{} } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{53} + return fileDescriptor_712ce9a559fda969, []int{54} } func (m *Device) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2346,7 +2385,7 @@ type StringUser struct { func (m *StringUser) Reset() { *m = StringUser{} } func (*StringUser) ProtoMessage() {} func (*StringUser) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{54} + return fileDescriptor_712ce9a559fda969, []int{55} } func (m *StringUser) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2403,7 +2442,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (*CopyFileRequest) ProtoMessage() {} func (*CopyFileRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{55} + return fileDescriptor_712ce9a559fda969, []int{56} } func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2441,7 +2480,7 @@ type GetOOMEventRequest struct { func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } func (*GetOOMEventRequest) ProtoMessage() {} func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{56} + return fileDescriptor_712ce9a559fda969, []int{57} } func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2480,7 +2519,7 @@ type OOMEvent struct { func (m *OOMEvent) Reset() { *m = OOMEvent{} } func (*OOMEvent) ProtoMessage() {} func (*OOMEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{57} + return fileDescriptor_712ce9a559fda969, []int{58} } func (m *OOMEvent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2519,7 +2558,7 @@ type AddSwapRequest struct { func (m *AddSwapRequest) Reset() { *m = AddSwapRequest{} } func (*AddSwapRequest) ProtoMessage() {} func (*AddSwapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{58} + return fileDescriptor_712ce9a559fda969, []int{59} } func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2557,7 +2596,7 @@ type GetMetricsRequest struct { func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } func (*GetMetricsRequest) ProtoMessage() {} func (*GetMetricsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{59} + return fileDescriptor_712ce9a559fda969, []int{60} } func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2596,7 +2635,7 @@ type Metrics struct { func (m *Metrics) Reset() { *m = Metrics{} } func (*Metrics) ProtoMessage() {} func (*Metrics) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{60} + return fileDescriptor_712ce9a559fda969, []int{61} } func (m *Metrics) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2636,7 +2675,7 @@ type VolumeStatsRequest struct { func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } func (*VolumeStatsRequest) ProtoMessage() {} func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{61} + return fileDescriptor_712ce9a559fda969, []int{62} } func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2677,7 +2716,7 @@ type ResizeVolumeRequest struct { func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } func (*ResizeVolumeRequest) ProtoMessage() {} func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{62} + return fileDescriptor_712ce9a559fda969, []int{63} } func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2745,6 +2784,7 @@ func init() { proto.RegisterType((*Routes)(nil), "grpc.Routes") proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") + proto.RegisterType((*UpdateEphemeralMountsRequest)(nil), "grpc.UpdateEphemeralMountsRequest") proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") proto.RegisterType((*ARPNeighbors)(nil), "grpc.ARPNeighbors") @@ -2779,208 +2819,211 @@ func init() { } var fileDescriptor_712ce9a559fda969 = []byte{ - // 3207 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x73, 0x1b, 0xc7, - 0xd1, 0xc6, 0x83, 0x04, 0xd0, 0x78, 0x11, 0x4b, 0x8a, 0x82, 0x60, 0x9b, 0x96, 0x57, 0xb6, 0x4c, - 0xd9, 0x9f, 0x48, 0x5b, 0x76, 0x59, 0x7e, 0x94, 0x3f, 0x7d, 0x24, 0x45, 0x93, 0xb4, 0x4d, 0x0b, - 0xdf, 0x42, 0x8c, 0x53, 0x49, 0x25, 0x5b, 0xcb, 0xdd, 0x21, 0x38, 0x26, 0x76, 0x67, 0x3d, 0x3b, - 0x0b, 0x91, 0x4e, 0x55, 0x2a, 0xa7, 0xe4, 0x96, 0x63, 0x6e, 0xf9, 0x03, 0xa9, 0xfc, 0x83, 0x5c, - 0x73, 0x50, 0xe5, 0x94, 0x63, 0x2e, 0x49, 0xc5, 0xfa, 0x09, 0xf9, 0x05, 0xa9, 0x79, 0xed, 0x03, - 0x0f, 0xda, 0x61, 0xa9, 0x2a, 0x17, 0xd4, 0x76, 0x4f, 0x4f, 0xbf, 0xa6, 0xa7, 0xa7, 0x7b, 0x06, - 0xd0, 0x1f, 0x62, 0x76, 0x1a, 0x1f, 0x6f, 0xb8, 0xc4, 0xdf, 0x3c, 0x73, 0x98, 0x73, 0xd7, 0x25, - 0x01, 0x73, 0x70, 0x80, 0x68, 0x34, 0x05, 0x47, 0xd4, 0xdd, 0x1c, 0xe1, 0xe3, 0x68, 0x33, 0xa4, - 0x84, 0x11, 0x97, 0x8c, 0xd4, 0x57, 0xb4, 0xe9, 0x0c, 0x51, 0xc0, 0x36, 0x04, 0x60, 0x94, 0x87, - 0x34, 0x74, 0x7b, 0x35, 0xe2, 0x62, 0x89, 0xe8, 0xd5, 0xdc, 0x48, 0x7f, 0xd6, 0xd9, 0x45, 0x88, - 0x22, 0x05, 0xbc, 0x38, 0x24, 0x64, 0x38, 0x42, 0x92, 0xc7, 0x71, 0x7c, 0xb2, 0x89, 0xfc, 0x90, - 0x5d, 0xc8, 0x41, 0xf3, 0xf7, 0x45, 0x58, 0xdd, 0xa1, 0xc8, 0x61, 0x68, 0x47, 0x2b, 0x60, 0xa1, - 0x6f, 0x62, 0x14, 0x31, 0xe3, 0x55, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x16, 0x6e, 0x16, 0xd6, - 0x6b, 0x56, 0x3d, 0xc1, 0x1d, 0x78, 0xc6, 0x75, 0xa8, 0xa0, 0x73, 0xe4, 0xf2, 0xd1, 0xa2, 0x18, - 0x5d, 0xe4, 0xe0, 0x81, 0x67, 0xbc, 0x03, 0xf5, 0x88, 0x51, 0x1c, 0x0c, 0xed, 0x38, 0x42, 0xb4, - 0x5b, 0xba, 0x59, 0x58, 0xaf, 0xdf, 0x5b, 0xda, 0xe0, 0x2a, 0x6f, 0x0c, 0xc4, 0xc0, 0x51, 0x84, - 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x86, 0x8a, 0x87, 0xc6, 0xd8, 0x45, 0x51, 0xb7, 0x7c, 0xb3, - 0xb4, 0x5e, 0xbf, 0xd7, 0x90, 0xe4, 0x0f, 0x05, 0xd2, 0xd2, 0x83, 0xc6, 0x1d, 0xa8, 0x46, 0x8c, - 0x50, 0x67, 0x88, 0xa2, 0xee, 0x82, 0x20, 0x6c, 0x6a, 0xbe, 0x02, 0x6b, 0x25, 0xc3, 0xc6, 0x4b, - 0x50, 0x7a, 0xb4, 0x73, 0xd0, 0x5d, 0x14, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8, - 0x05, 0xcd, 0xc8, 0x09, 0xbc, 0x63, 0x72, 0x6e, 0x87, 0xd8, 0x0b, 0xa2, 0x6e, 0xe5, 0x66, 0x61, - 0xbd, 0x6a, 0x35, 0x14, 0xb2, 0xcf, 0x71, 0xe6, 0x47, 0x70, 0x6d, 0xc0, 0x1c, 0xca, 0xae, 0xe0, - 0x1d, 0xf3, 0x08, 0x56, 0x2d, 0xe4, 0x93, 0xf1, 0x95, 0x5c, 0xdb, 0x85, 0x0a, 0xc3, 0x3e, 0x22, - 0x31, 0x13, 0xae, 0x6d, 0x5a, 0x1a, 0x34, 0xff, 0x58, 0x00, 0x63, 0xf7, 0x1c, 0xb9, 0x7d, 0x4a, - 0x5c, 0x14, 0x45, 0xff, 0xa5, 0xe5, 0x7a, 0x03, 0x2a, 0xa1, 0x54, 0xa0, 0x5b, 0x16, 0xe4, 0x6a, - 0x15, 0xb4, 0x56, 0x7a, 0xd4, 0xfc, 0x1a, 0x56, 0x06, 0x78, 0x18, 0x38, 0xa3, 0xe7, 0xa8, 0xef, - 0x2a, 0x2c, 0x46, 0x82, 0xa7, 0x50, 0xb5, 0x69, 0x29, 0xc8, 0xec, 0x83, 0xf1, 0x95, 0x83, 0xd9, - 0xf3, 0x93, 0x64, 0xde, 0x85, 0xe5, 0x1c, 0xc7, 0x28, 0x24, 0x41, 0x84, 0x84, 0x02, 0xcc, 0x61, - 0x71, 0x24, 0x98, 0x2d, 0x58, 0x0a, 0x32, 0x09, 0xac, 0x1e, 0x85, 0xde, 0x15, 0x77, 0xd3, 0x3d, - 0xa8, 0x51, 0x14, 0x91, 0x98, 0xf2, 0x3d, 0x50, 0x14, 0x4e, 0x5d, 0x91, 0x4e, 0xfd, 0x02, 0x07, - 0xf1, 0xb9, 0xa5, 0xc7, 0xac, 0x94, 0x4c, 0xc5, 0x27, 0x8b, 0xae, 0x12, 0x9f, 0x1f, 0xc1, 0xb5, - 0xbe, 0x13, 0x47, 0x57, 0xd1, 0xd5, 0xfc, 0x98, 0xc7, 0x76, 0x14, 0xfb, 0x57, 0x9a, 0xfc, 0x87, - 0x02, 0x54, 0x77, 0xc2, 0xf8, 0x28, 0x72, 0x86, 0xc8, 0x78, 0x05, 0xea, 0x8c, 0x30, 0x67, 0x64, - 0xc7, 0x1c, 0x14, 0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x0a, 0x8d, 0x10, 0x51, 0x37, 0x8c, - 0x15, 0x45, 0xf1, 0x66, 0x69, 0xbd, 0x6c, 0xd5, 0x25, 0x4e, 0x92, 0x6c, 0xc0, 0xb2, 0x18, 0xb3, - 0x71, 0x60, 0x9f, 0x21, 0x1a, 0xa0, 0x91, 0x4f, 0x3c, 0x24, 0x82, 0xa3, 0x6c, 0x75, 0xc4, 0xd0, - 0x41, 0xf0, 0x79, 0x32, 0x60, 0xbc, 0x09, 0x9d, 0x84, 0x9e, 0x47, 0xbc, 0xa0, 0x2e, 0x0b, 0xea, - 0xb6, 0xa2, 0x3e, 0x52, 0x68, 0xf3, 0x97, 0xd0, 0x7a, 0x7c, 0x4a, 0x09, 0x63, 0x23, 0x1c, 0x0c, - 0x1f, 0x3a, 0xcc, 0xe1, 0x5b, 0x33, 0x44, 0x14, 0x13, 0x2f, 0x52, 0xda, 0x6a, 0xd0, 0x78, 0x0b, - 0x3a, 0x4c, 0xd2, 0x22, 0xcf, 0xd6, 0x34, 0x45, 0x41, 0xb3, 0x94, 0x0c, 0xf4, 0x15, 0xf1, 0xeb, - 0xd0, 0x4a, 0x89, 0xf9, 0xe6, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x63, 0xec, 0x23, 0x73, 0x2c, 0x7c, - 0x25, 0x16, 0xd9, 0x78, 0x0b, 0x6a, 0xa9, 0x1f, 0x0a, 0x22, 0x42, 0x5a, 0x32, 0x42, 0xb4, 0x3b, - 0xad, 0x6a, 0xe2, 0x94, 0x4f, 0xa0, 0xcd, 0x12, 0xc5, 0x6d, 0xcf, 0x61, 0x4e, 0x3e, 0xa8, 0xf2, - 0x56, 0x59, 0x2d, 0x96, 0x83, 0xcd, 0x8f, 0xa1, 0xd6, 0xc7, 0x5e, 0x24, 0x05, 0x77, 0xa1, 0xe2, - 0xc6, 0x94, 0xa2, 0x80, 0x69, 0x93, 0x15, 0x68, 0xac, 0xc0, 0xc2, 0x08, 0xfb, 0x98, 0x29, 0x33, - 0x25, 0x60, 0x12, 0x80, 0x43, 0xe4, 0x13, 0x7a, 0x21, 0x1c, 0xb6, 0x02, 0x0b, 0xd9, 0xc5, 0x95, - 0x80, 0xf1, 0x22, 0xd4, 0x7c, 0xe7, 0x3c, 0x59, 0x54, 0x3e, 0x52, 0xf5, 0x9d, 0x73, 0xa9, 0x7c, - 0x17, 0x2a, 0x27, 0x0e, 0x1e, 0xb9, 0x01, 0x53, 0x5e, 0xd1, 0x60, 0x2a, 0xb0, 0x9c, 0x15, 0xf8, - 0xe7, 0x22, 0xd4, 0xa5, 0x44, 0xa9, 0xf0, 0x0a, 0x2c, 0xb8, 0x8e, 0x7b, 0x9a, 0x88, 0x14, 0x80, - 0x71, 0x5b, 0x2b, 0x52, 0xcc, 0x66, 0xb8, 0x54, 0x53, 0xad, 0xda, 0x26, 0x40, 0xf4, 0xc4, 0x09, - 0x95, 0x6e, 0xa5, 0x39, 0xc4, 0x35, 0x4e, 0x23, 0xd5, 0x7d, 0x17, 0x1a, 0x32, 0xee, 0xd4, 0x94, - 0xf2, 0x9c, 0x29, 0x75, 0x49, 0x25, 0x27, 0xdd, 0x82, 0x66, 0x1c, 0x21, 0xfb, 0x14, 0x23, 0xea, - 0x50, 0xf7, 0xf4, 0xa2, 0xbb, 0x20, 0x0f, 0xa0, 0x38, 0x42, 0xfb, 0x1a, 0x67, 0xdc, 0x83, 0x05, - 0x9e, 0x5b, 0xa2, 0xee, 0xa2, 0x38, 0xeb, 0x5e, 0xca, 0xb2, 0x14, 0xa6, 0x6e, 0x88, 0xdf, 0xdd, - 0x80, 0xd1, 0x0b, 0x4b, 0x92, 0xf6, 0x3e, 0x00, 0x48, 0x91, 0xc6, 0x12, 0x94, 0xce, 0xd0, 0x85, - 0xda, 0x87, 0xfc, 0x93, 0x3b, 0x67, 0xec, 0x8c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x54, 0xfc, 0xa0, - 0x60, 0xba, 0xd0, 0xde, 0x1e, 0x9d, 0x61, 0x92, 0x99, 0xbe, 0x02, 0x0b, 0xbe, 0xf3, 0x35, 0xa1, - 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84, 0xc2, - 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x47, 0x19, 0x20, 0x95, 0x62, - 0x58, 0xd0, 0xc3, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x7c, 0xc1, 0x50, 0x64, 0x53, 0xe4, - 0xc6, 0x34, 0xc2, 0x63, 0xbe, 0x7e, 0xdc, 0xec, 0x6b, 0xd2, 0xec, 0x09, 0xdd, 0xac, 0xeb, 0x98, - 0x0c, 0xe4, 0xbc, 0x6d, 0x3e, 0xcd, 0xd2, 0xb3, 0x8c, 0x03, 0xb8, 0x96, 0xf2, 0xf4, 0x32, 0xec, - 0x8a, 0x97, 0xb1, 0x5b, 0x4e, 0xd8, 0x79, 0x29, 0xab, 0x5d, 0x58, 0xc6, 0xc4, 0xfe, 0x26, 0x46, - 0x71, 0x8e, 0x51, 0xe9, 0x32, 0x46, 0x1d, 0x4c, 0xfe, 0x5f, 0x4c, 0x48, 0xd9, 0xf4, 0xe1, 0x46, - 0xc6, 0x4a, 0xbe, 0xdd, 0x33, 0xcc, 0xca, 0x97, 0x31, 0x5b, 0x4d, 0xb4, 0xe2, 0xf9, 0x20, 0xe5, - 0xf8, 0x19, 0xac, 0x62, 0x62, 0x3f, 0x71, 0x30, 0x9b, 0x64, 0xb7, 0xf0, 0x3d, 0x46, 0xf2, 0x13, - 0x2d, 0xcf, 0x4b, 0x1a, 0xe9, 0x23, 0x3a, 0xcc, 0x19, 0xb9, 0xf8, 0x3d, 0x46, 0x1e, 0x8a, 0x09, - 0x29, 0x9b, 0x2d, 0xe8, 0x60, 0x32, 0xa9, 0x4d, 0xe5, 0x32, 0x26, 0x6d, 0x4c, 0xf2, 0x9a, 0x6c, - 0x43, 0x27, 0x42, 0x2e, 0x23, 0x34, 0x1b, 0x04, 0xd5, 0xcb, 0x58, 0x2c, 0x29, 0xfa, 0x84, 0x87, - 0xf9, 0x53, 0x68, 0xec, 0xc7, 0x43, 0xc4, 0x46, 0xc7, 0x49, 0x32, 0x78, 0x6e, 0xf9, 0xc7, 0xfc, - 0x57, 0x11, 0xea, 0x3b, 0x43, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xcc, 0xc9, 0x82, - 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x1e, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d, 0xa9, - 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x00, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d, 0xb5, - 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x0e, 0xd4, 0x8f, 0xb9, 0x93, 0xd4, - 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x71, 0xba, 0x09, 0xf7, 0xa1, 0x79, 0x2a, 0x5d, 0xa6, - 0x26, 0xc9, 0x18, 0xba, 0xa5, 0x2c, 0x49, 0xed, 0xdd, 0xc8, 0x7a, 0x56, 0x2e, 0x40, 0xe3, 0x34, - 0x83, 0xea, 0x0d, 0xa0, 0x33, 0x45, 0x32, 0x23, 0x07, 0xad, 0x67, 0x73, 0x50, 0xfd, 0x9e, 0x21, - 0x05, 0x65, 0x67, 0x66, 0xf3, 0xd2, 0x6f, 0x8b, 0xd0, 0xf8, 0x12, 0xb1, 0x27, 0x84, 0x9e, 0x49, - 0x7d, 0x0d, 0x28, 0x07, 0x8e, 0x8f, 0x14, 0x47, 0xf1, 0x6d, 0xdc, 0x80, 0x2a, 0x3d, 0x97, 0x09, - 0x44, 0xad, 0x67, 0x85, 0x9e, 0x8b, 0xc4, 0x60, 0xbc, 0x0c, 0x40, 0xcf, 0xed, 0xd0, 0x71, 0xcf, - 0x90, 0xf2, 0x60, 0xd9, 0xaa, 0xd1, 0xf3, 0xbe, 0x44, 0xf0, 0x50, 0xa0, 0xe7, 0x36, 0xa2, 0x94, - 0xd0, 0x48, 0xe5, 0xaa, 0x2a, 0x3d, 0xdf, 0x15, 0xb0, 0x9a, 0xeb, 0x51, 0x12, 0x86, 0xc8, 0x13, - 0x39, 0x5a, 0xcc, 0x7d, 0x28, 0x11, 0x5c, 0x2a, 0xd3, 0x52, 0x17, 0xa5, 0x54, 0x96, 0x4a, 0x65, - 0xa9, 0xd4, 0x8a, 0x9c, 0xc9, 0xb2, 0x52, 0x59, 0x22, 0xb5, 0x2a, 0xa5, 0xb2, 0x8c, 0x54, 0x96, - 0x4a, 0xad, 0xe9, 0xb9, 0x4a, 0xaa, 0xf9, 0x9b, 0x02, 0xac, 0x4e, 0x16, 0x7e, 0xaa, 0x36, 0x7d, - 0x0f, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0xa6, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30, 0xbe, - 0x0f, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, 0x03, - 0x99, 0x1e, 0x18, 0x5f, 0x51, 0xcc, 0xd0, 0x80, 0x51, 0xe4, 0xf8, 0xcf, 0xa3, 0xba, 0x37, 0xa0, - 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0xc0, 0x72, 0x4e, 0x8a, 0xb2, 0x75, - 0x09, 0x4a, 0x23, 0x14, 0x08, 0xee, 0x4d, 0x8b, 0x7f, 0x9a, 0x0e, 0x74, 0x2c, 0xe4, 0x78, 0xcf, - 0x4f, 0x1b, 0x25, 0xa2, 0x94, 0x8a, 0x58, 0x07, 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x64, - 0xb4, 0x7e, 0x04, 0x9d, 0x9d, 0x11, 0x89, 0xd0, 0x80, 0x79, 0x38, 0x78, 0x1e, 0xed, 0xc8, 0x2f, - 0x60, 0xf9, 0x31, 0xbb, 0xf8, 0x8a, 0x33, 0x8b, 0xf0, 0xb7, 0xe8, 0x39, 0xd9, 0x47, 0xc9, 0x13, - 0x6d, 0x1f, 0x25, 0x4f, 0x78, 0x73, 0xe3, 0x92, 0x51, 0xec, 0x07, 0x62, 0x2b, 0x34, 0x2d, 0x05, - 0x99, 0xdb, 0xd0, 0x90, 0x35, 0xf4, 0x21, 0xf1, 0xe2, 0x11, 0x9a, 0xb9, 0x07, 0xd7, 0x00, 0x42, - 0x87, 0x3a, 0x3e, 0x62, 0x88, 0xca, 0x18, 0xaa, 0x59, 0x19, 0x8c, 0xf9, 0xbb, 0x22, 0xac, 0xc8, - 0xfb, 0x86, 0x81, 0x6c, 0xb3, 0xb5, 0x09, 0x3d, 0xa8, 0x9e, 0x92, 0x88, 0x65, 0x18, 0x26, 0x30, - 0x57, 0x91, 0xf7, 0xe7, 0x92, 0x1b, 0xff, 0xcc, 0x5d, 0x02, 0x94, 0x2e, 0xbf, 0x04, 0x98, 0x6a, - 0xf3, 0xcb, 0xd3, 0x6d, 0x3e, 0xdf, 0x6d, 0x9a, 0x08, 0xcb, 0x3d, 0x5e, 0xb3, 0x6a, 0x0a, 0x73, - 0xe0, 0x19, 0xb7, 0xa1, 0x3d, 0xe4, 0x5a, 0xda, 0xa7, 0x84, 0x9c, 0xd9, 0xa1, 0xc3, 0x4e, 0xc5, - 0x56, 0xaf, 0x59, 0x4d, 0x81, 0xde, 0x27, 0xe4, 0xac, 0xef, 0xb0, 0x53, 0xe3, 0x43, 0x68, 0xa9, - 0x32, 0xd0, 0x17, 0x2e, 0x8a, 0xd4, 0xe1, 0xa7, 0x76, 0x51, 0xd6, 0x7b, 0x56, 0xf3, 0x2c, 0x03, - 0x45, 0xe6, 0x75, 0xb8, 0xf6, 0x10, 0x45, 0x8c, 0x92, 0x8b, 0xbc, 0x63, 0xcc, 0xff, 0x05, 0x38, - 0x08, 0x18, 0xa2, 0x27, 0x8e, 0x8b, 0x22, 0xe3, 0xed, 0x2c, 0xa4, 0x8a, 0xa3, 0xa5, 0x0d, 0x79, - 0xdd, 0x93, 0x0c, 0x58, 0x19, 0x1a, 0x73, 0x03, 0x16, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0x9a, 0xfe, - 0x52, 0xf3, 0x1a, 0x6a, 0x9e, 0x40, 0x5a, 0x6a, 0xcc, 0xdc, 0xd7, 0x2d, 0x6c, 0xca, 0x4e, 0x2d, - 0xd1, 0x06, 0xd4, 0xb0, 0xc6, 0xa9, 0xac, 0x32, 0x2d, 0x3a, 0x25, 0x31, 0x3f, 0x86, 0x65, 0xc9, - 0x49, 0x72, 0xd6, 0x6c, 0x5e, 0x83, 0x45, 0xaa, 0xd5, 0x28, 0xa4, 0xf7, 0x3c, 0x8a, 0x48, 0x8d, - 0x71, 0x7f, 0x7c, 0x81, 0x23, 0x96, 0x1a, 0xa2, 0xfd, 0xb1, 0x0c, 0x1d, 0x3e, 0x90, 0xe3, 0x69, - 0x7e, 0x0a, 0x8d, 0x2d, 0xab, 0xff, 0x25, 0xc2, 0xc3, 0xd3, 0x63, 0x9e, 0x3d, 0xdf, 0xcf, 0xc3, - 0xca, 0x60, 0x43, 0x69, 0x9b, 0x19, 0xb2, 0x72, 0x74, 0xe6, 0x67, 0xb0, 0xba, 0xe5, 0x79, 0x59, - 0x94, 0xd6, 0xfa, 0x6d, 0xa8, 0x05, 0x19, 0x76, 0x99, 0x33, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x77, - 0xc1, 0xd8, 0x43, 0xec, 0xa0, 0xff, 0xd8, 0x39, 0x1e, 0xa5, 0xd6, 0x5f, 0x87, 0x0a, 0x8e, 0x6c, - 0x1c, 0x8e, 0xdf, 0x17, 0x5c, 0xaa, 0xd6, 0x22, 0x8e, 0x0e, 0xc2, 0xf1, 0xfb, 0xe6, 0x1d, 0x58, - 0xce, 0x91, 0x5f, 0x92, 0x56, 0xb6, 0xc0, 0x18, 0xfc, 0x70, 0xce, 0x09, 0x8b, 0x62, 0x86, 0xc5, - 0x1d, 0x58, 0x1e, 0xfc, 0x40, 0x69, 0x3f, 0x83, 0xe5, 0x47, 0xc1, 0x08, 0x07, 0x68, 0xa7, 0x7f, - 0x74, 0x88, 0x92, 0x9c, 0x6a, 0x40, 0x99, 0xd7, 0x9e, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0xc7, - 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xb4, 0x5a, 0x0c, 0x8e, 0x77, 0xc2, 0x38, 0xe2, 0x87, 0x24, 0x2f, - 0x92, 0x48, 0x30, 0xba, 0x10, 0x99, 0xa6, 0x6a, 0x55, 0xdc, 0x30, 0x7e, 0x14, 0x8c, 0x2e, 0xcc, - 0xff, 0x11, 0x37, 0x09, 0x08, 0x79, 0x96, 0x13, 0x78, 0xc4, 0x7f, 0x88, 0xc6, 0x19, 0x09, 0x53, - 0x7a, 0x3f, 0x2d, 0x40, 0x63, 0x6b, 0x88, 0x02, 0xf6, 0x10, 0x31, 0x07, 0x8f, 0x44, 0x67, 0x3a, - 0x46, 0x34, 0xc2, 0x24, 0x50, 0x69, 0x43, 0x83, 0xc6, 0x2b, 0x50, 0xc7, 0x01, 0x66, 0xb6, 0xe7, - 0x20, 0x9f, 0x04, 0x82, 0x4b, 0xd5, 0x02, 0x8e, 0x7a, 0x28, 0x30, 0xc6, 0x1b, 0xd0, 0x96, 0x97, - 0x8a, 0xf6, 0xa9, 0x13, 0x78, 0x23, 0x9e, 0xb0, 0x4a, 0x22, 0xc5, 0xb4, 0x24, 0x7a, 0x5f, 0x61, - 0x8d, 0x3b, 0xb0, 0xa4, 0xd2, 0x49, 0x4a, 0x59, 0x16, 0x94, 0x6d, 0x85, 0xcf, 0x91, 0xc6, 0x61, - 0x48, 0x28, 0x8b, 0xec, 0x08, 0xb9, 0x2e, 0xf1, 0x43, 0xd5, 0xd6, 0xb5, 0x35, 0x7e, 0x20, 0xd1, - 0xe6, 0x10, 0x96, 0xf7, 0xb8, 0x9d, 0xca, 0x92, 0x74, 0x7b, 0xb4, 0x7c, 0xe4, 0xdb, 0xc7, 0x23, - 0xe2, 0x9e, 0xd9, 0x3c, 0xc9, 0x2b, 0x0f, 0xf3, 0xc2, 0x71, 0x9b, 0x23, 0x07, 0xf8, 0x5b, 0x71, - 0x83, 0xc1, 0xa9, 0x4e, 0x09, 0x0b, 0x47, 0xf1, 0xd0, 0x0e, 0x29, 0x39, 0x46, 0xca, 0xc4, 0xb6, - 0x8f, 0xfc, 0x7d, 0x89, 0xef, 0x73, 0xb4, 0xf9, 0xa7, 0x02, 0xac, 0xe4, 0x25, 0xa9, 0xd5, 0xde, - 0x84, 0x95, 0xbc, 0x28, 0x55, 0xc6, 0xc8, 0x32, 0xb9, 0x93, 0x15, 0x28, 0x0b, 0x9a, 0xfb, 0xd0, - 0x14, 0x57, 0xd0, 0xb6, 0x27, 0x39, 0xe5, 0x8b, 0xb7, 0xec, 0xba, 0x58, 0x0d, 0x27, 0xbb, 0x4a, - 0x1f, 0xc2, 0x0d, 0x65, 0xbe, 0x3d, 0xad, 0xb6, 0x0c, 0x88, 0x55, 0x45, 0x70, 0x38, 0xa1, 0xfd, - 0x17, 0xd0, 0x4d, 0x51, 0xdb, 0x17, 0x02, 0x99, 0x6e, 0xca, 0xe5, 0x09, 0x63, 0xb7, 0x3c, 0x8f, - 0x8a, 0xdd, 0x5e, 0xb6, 0x66, 0x0d, 0x99, 0x0f, 0xe0, 0xfa, 0x00, 0x31, 0xe9, 0x0d, 0x87, 0xa9, - 0x8e, 0x4a, 0x32, 0x5b, 0x82, 0xd2, 0x00, 0xb9, 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, - 0x45, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d, 0x86, 0x50, 0xf9, 0x74, 0xb0, 0xc7, 0xeb, 0x26, - 0x1e, 0xd4, 0xb2, 0xce, 0x52, 0x67, 0x6a, 0xd3, 0xaa, 0x08, 0xf8, 0xc0, 0x33, 0x3e, 0x83, 0x65, - 0x39, 0xe4, 0x9e, 0x3a, 0xc1, 0x10, 0xd9, 0x21, 0x19, 0x61, 0x57, 0x86, 0x7e, 0xeb, 0x5e, 0x4f, - 0xa5, 0x21, 0xc5, 0x67, 0x47, 0x90, 0xf4, 0x05, 0x85, 0xd5, 0x19, 0x4e, 0xa2, 0xcc, 0xbf, 0x17, - 0xa0, 0xa2, 0x8e, 0x35, 0x7e, 0x34, 0x7b, 0x14, 0x8f, 0x11, 0x55, 0xc1, 0xae, 0x20, 0xe3, 0x75, - 0x68, 0xc9, 0x2f, 0x9b, 0x84, 0x0c, 0x93, 0xe4, 0xb0, 0x6c, 0x4a, 0xec, 0x23, 0x89, 0x14, 0xd7, - 0x96, 0xe2, 0xe2, 0x50, 0xf5, 0xe8, 0x0a, 0xe2, 0xf8, 0x93, 0x88, 0x2b, 0x25, 0x0e, 0xc7, 0x9a, - 0xa5, 0x20, 0xbe, 0xb9, 0x34, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, 0x73, 0xf9, 0x24, 0x0e, 0x98, - 0x1d, 0x12, 0x1c, 0x30, 0x75, 0x1a, 0x82, 0x40, 0xf5, 0x39, 0xc6, 0x58, 0x87, 0xea, 0x49, 0x64, - 0x0b, 0x6b, 0x44, 0xe5, 0x9b, 0x9c, 0xd0, 0xca, 0x6a, 0xab, 0x72, 0x12, 0x89, 0x0f, 0xf3, 0xd7, - 0x05, 0x58, 0x94, 0x97, 0xfc, 0x46, 0x0b, 0x8a, 0x49, 0xf5, 0x52, 0xc4, 0xa2, 0x12, 0x14, 0x5a, - 0xc9, 0x8a, 0x45, 0x7c, 0xf3, 0x1c, 0x33, 0xf6, 0xe5, 0x19, 0xac, 0x8c, 0x18, 0xfb, 0xe2, 0xf0, - 0x7d, 0x1d, 0x5a, 0x69, 0x11, 0x24, 0xc6, 0xa5, 0x31, 0xcd, 0x04, 0x2b, 0xc8, 0xe6, 0xda, 0x64, - 0xfe, 0x18, 0x20, 0xbd, 0xec, 0xe6, 0xe1, 0x10, 0x27, 0xca, 0xf0, 0x4f, 0x8e, 0x19, 0x26, 0xe5, - 0x13, 0xff, 0x34, 0x6e, 0x43, 0xcb, 0xf1, 0x3c, 0xcc, 0xa7, 0x3b, 0xa3, 0x3d, 0xec, 0x25, 0x09, - 0x24, 0x8f, 0x35, 0xff, 0x52, 0x80, 0xf6, 0x0e, 0x09, 0x2f, 0x3e, 0xc5, 0x23, 0x94, 0xc9, 0x6e, - 0x42, 0x49, 0x55, 0x3d, 0xf1, 0x6f, 0xde, 0x11, 0x9c, 0xe0, 0x11, 0x92, 0xdb, 0x5e, 0x46, 0x5d, - 0x95, 0x23, 0xc4, 0x96, 0xd7, 0x83, 0xc9, 0xd5, 0x66, 0x53, 0x0e, 0x1e, 0x12, 0x4f, 0xf4, 0x3e, - 0x1e, 0xa6, 0x76, 0x72, 0x91, 0xd9, 0xb4, 0x2a, 0x1e, 0xa6, 0x62, 0x48, 0x19, 0xb2, 0x20, 0x2e, - 0xaa, 0xb3, 0x86, 0x2c, 0x4a, 0x0c, 0x37, 0x64, 0x15, 0x16, 0xc9, 0xc9, 0x49, 0x84, 0x98, 0x58, - 0xab, 0x92, 0xa5, 0xa0, 0x24, 0x05, 0x57, 0x33, 0x29, 0x78, 0x45, 0x9c, 0x6b, 0x8f, 0x1e, 0x1d, - 0xee, 0x8e, 0x51, 0xc0, 0xf4, 0x09, 0x7c, 0x17, 0xaa, 0x1a, 0xf5, 0x43, 0xae, 0x80, 0xdf, 0x84, - 0xd6, 0x96, 0xe7, 0x0d, 0x9e, 0x38, 0xa1, 0xf6, 0x47, 0x17, 0x2a, 0xfd, 0x9d, 0x83, 0xbe, 0x74, - 0x49, 0x89, 0x1b, 0xa0, 0x40, 0x7e, 0xe2, 0xef, 0x21, 0x76, 0x88, 0x18, 0xc5, 0x6e, 0x72, 0xe2, - 0xdf, 0x82, 0x8a, 0xc2, 0xf0, 0x99, 0xbe, 0xfc, 0xd4, 0x47, 0x80, 0x02, 0xcd, 0xff, 0x03, 0xe3, - 0x47, 0xbc, 0x76, 0x45, 0xb2, 0x71, 0x51, 0x92, 0xde, 0x84, 0xce, 0x58, 0x60, 0x6d, 0x59, 0xd4, - 0x65, 0x96, 0xa1, 0x2d, 0x07, 0x44, 0x7e, 0x10, 0xb2, 0x8f, 0x60, 0x59, 0x96, 0xda, 0x92, 0xcf, - 0x15, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6, 0xc4, 0xf7, 0xbd, 0xa7, 0x86, 0x3a, 0xc6, 0xd4, - 0xcd, 0x8e, 0xb1, 0x07, 0xed, 0x89, 0x67, 0x38, 0x43, 0x5d, 0xf5, 0xcd, 0x7e, 0x9d, 0xeb, 0xad, - 0x6e, 0xc8, 0x67, 0xbd, 0x0d, 0xfd, 0xac, 0xb7, 0xb1, 0xeb, 0x87, 0xec, 0xc2, 0xd8, 0x85, 0x56, - 0xfe, 0xc1, 0xca, 0x78, 0x51, 0x57, 0xc6, 0x33, 0x9e, 0xb1, 0xe6, 0xb2, 0xd9, 0x83, 0xf6, 0xc4, - 0xdb, 0x95, 0xd6, 0x67, 0xf6, 0x93, 0xd6, 0x5c, 0x46, 0x0f, 0xa0, 0x9e, 0x79, 0xac, 0x32, 0xba, - 0x92, 0xc9, 0xf4, 0xfb, 0xd5, 0x5c, 0x06, 0x3b, 0xd0, 0xcc, 0xbd, 0x1f, 0x19, 0x3d, 0x65, 0xcf, - 0x8c, 0x47, 0xa5, 0xb9, 0x4c, 0xb6, 0xa1, 0x9e, 0x79, 0xc6, 0xd1, 0x5a, 0x4c, 0xbf, 0x15, 0xf5, - 0x6e, 0xcc, 0x18, 0x51, 0xa7, 0xe5, 0x1e, 0xb4, 0x27, 0xde, 0x76, 0xb4, 0x4b, 0x66, 0x3f, 0xf9, - 0xcc, 0x55, 0xe6, 0x73, 0xb1, 0x44, 0x99, 0xd6, 0x3d, 0xb3, 0x44, 0xd3, 0x2f, 0x39, 0xbd, 0x97, - 0x66, 0x0f, 0x2a, 0xad, 0x76, 0xa1, 0x95, 0x7f, 0xc4, 0xd1, 0xcc, 0x66, 0x3e, 0xed, 0x5c, 0xbe, - 0xde, 0xb9, 0xf7, 0x9c, 0x74, 0xbd, 0x67, 0x3d, 0xf3, 0xcc, 0x65, 0xb4, 0x05, 0xa0, 0x1a, 0x75, - 0x0f, 0x07, 0x89, 0xa3, 0xa7, 0x2e, 0x08, 0x12, 0x47, 0xcf, 0x68, 0xea, 0x1f, 0x00, 0xc8, 0xfe, - 0xda, 0x23, 0x31, 0x33, 0xae, 0x6b, 0x35, 0x26, 0x9a, 0xfa, 0x5e, 0x77, 0x7a, 0x60, 0x8a, 0x01, - 0xa2, 0xf4, 0x2a, 0x0c, 0x3e, 0x01, 0x48, 0xfb, 0x76, 0xcd, 0x60, 0xaa, 0x93, 0xbf, 0xc4, 0x07, - 0x8d, 0x6c, 0x97, 0x6e, 0x28, 0x5b, 0x67, 0x74, 0xee, 0x97, 0xb0, 0x68, 0x4f, 0x74, 0x61, 0xf9, - 0x60, 0x9b, 0x6c, 0xce, 0x7a, 0x53, 0x9d, 0x98, 0x71, 0x1f, 0x1a, 0xd9, 0xf6, 0x4b, 0x6b, 0x31, - 0xa3, 0x25, 0xeb, 0xe5, 0x5a, 0x30, 0xe3, 0x01, 0xb4, 0xf2, 0xad, 0x97, 0x0e, 0xa9, 0x99, 0x0d, - 0x59, 0x4f, 0x5d, 0x2c, 0x66, 0xc8, 0xdf, 0x05, 0x48, 0x5b, 0x34, 0xed, 0xbe, 0xa9, 0xa6, 0x6d, - 0x42, 0xea, 0x1e, 0xb4, 0x27, 0x5a, 0x2f, 0x6d, 0xf1, 0xec, 0x8e, 0xec, 0xb2, 0xbd, 0x9e, 0x69, - 0xa4, 0x74, 0x08, 0x4e, 0xb7, 0x62, 0x3a, 0x04, 0x67, 0x75, 0x5d, 0xdb, 0x50, 0x1f, 0x4c, 0xf3, - 0x18, 0xcc, 0xe5, 0x31, 0xab, 0x97, 0x7a, 0x0f, 0x20, 0x3d, 0xb6, 0xb4, 0x17, 0xa6, 0x0e, 0xb2, - 0x5e, 0x53, 0x5f, 0xfe, 0x4a, 0xba, 0x1d, 0x68, 0xe6, 0xee, 0x47, 0x74, 0xba, 0x9b, 0x75, 0x69, - 0x72, 0xd9, 0x21, 0x90, 0xbf, 0x4c, 0xd0, 0x2b, 0x38, 0xf3, 0x8a, 0xe1, 0xb2, 0x38, 0xce, 0x76, - 0x7e, 0x3a, 0x82, 0x66, 0x74, 0x83, 0xdf, 0x93, 0x57, 0xb2, 0xdd, 0x5d, 0x26, 0xaf, 0xcc, 0x68, - 0xfa, 0xe6, 0x32, 0xda, 0x87, 0xf6, 0x9e, 0x2e, 0xdc, 0x55, 0x53, 0xa1, 0xd7, 0x6f, 0xba, 0x89, - 0xea, 0xf5, 0x66, 0x0d, 0xa9, 0x75, 0xf9, 0x1c, 0x3a, 0x53, 0x0d, 0x85, 0xb1, 0x96, 0x5c, 0xc1, - 0xcf, 0xec, 0x34, 0xe6, 0xaa, 0x75, 0x00, 0x4b, 0x93, 0xfd, 0x84, 0xf1, 0x72, 0x12, 0x13, 0xb3, - 0xfa, 0x8c, 0xb9, 0xac, 0x3e, 0x84, 0xaa, 0xae, 0x11, 0x0d, 0xf5, 0xd4, 0x31, 0x51, 0x33, 0xce, - 0x9d, 0x7a, 0x5f, 0x84, 0x7c, 0x52, 0x7f, 0xa5, 0x21, 0x3f, 0x51, 0xa5, 0xf5, 0xd4, 0xcb, 0x44, - 0x42, 0x79, 0x1f, 0x2a, 0xaa, 0x0c, 0x33, 0x56, 0x92, 0xcd, 0x96, 0xa9, 0xca, 0x2e, 0x8b, 0xb0, - 0x3d, 0xc4, 0x32, 0xc5, 0x95, 0x16, 0x3a, 0x5d, 0x6f, 0xe9, 0x3d, 0x92, 0x1b, 0x51, 0x6b, 0xb1, - 0x05, 0x8d, 0x6c, 0x79, 0xa5, 0x97, 0x74, 0x46, 0xc9, 0x35, 0x4f, 0x93, 0xed, 0xf3, 0xa7, 0xdf, - 0xad, 0xbd, 0xf0, 0xb7, 0xef, 0xd6, 0x5e, 0xf8, 0xd5, 0xb3, 0xb5, 0xc2, 0xd3, 0x67, 0x6b, 0x85, - 0xbf, 0x3e, 0x5b, 0x2b, 0xfc, 0xf3, 0xd9, 0x5a, 0xe1, 0x27, 0x3f, 0xff, 0x0f, 0xff, 0x73, 0x45, - 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x8e, 0x31, 0x65, 0x99, 0xa1, 0xf0, 0x6c, 0x28, 0xff, 0x78, 0x95, - 0xf9, 0x3f, 0x16, 0xd7, 0xf2, 0x78, 0x51, 0xc0, 0xef, 0xfe, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x77, - 0xb9, 0x27, 0x67, 0xdc, 0x25, 0x00, 0x00, + // 3249 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0xcb, 0x72, 0x1c, 0x47, + 0x72, 0x9a, 0x07, 0x30, 0x33, 0x39, 0x2f, 0x4c, 0x0f, 0x08, 0x0e, 0x47, 0x14, 0x44, 0x35, 0x25, + 0x0a, 0x94, 0x4c, 0x40, 0xa2, 0x14, 0xa2, 0x1e, 0x21, 0xd3, 0x00, 0x08, 0x01, 0x90, 0x04, 0x71, + 0xdc, 0x43, 0x58, 0x0e, 0x3b, 0xec, 0x8e, 0x46, 0x77, 0x61, 0xa6, 0x84, 0xe9, 0xae, 0x56, 0x75, + 0x35, 0x08, 0xc8, 0x11, 0x0e, 0x9f, 0xec, 0x9b, 0x8f, 0xbe, 0xe9, 0x07, 0x1c, 0xfe, 0x03, 0x5f, + 0x7d, 0x50, 0xf8, 0xb4, 0xc7, 0xbd, 0xec, 0xc6, 0x8a, 0x9f, 0xb0, 0x5f, 0xb0, 0x51, 0xaf, 0x7e, + 0xcc, 0x03, 0xd2, 0x22, 0x18, 0xb1, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x56, 0x56, 0x66, + 0xd5, 0xc0, 0x60, 0x84, 0xd9, 0x38, 0x3e, 0xd9, 0x74, 0x89, 0xbf, 0x75, 0xe6, 0x30, 0xe7, 0x81, + 0x4b, 0x02, 0xe6, 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x35, 0xc1, 0x27, 0xd1, 0x56, + 0x48, 0x09, 0x23, 0x2e, 0x99, 0xa8, 0xaf, 0x68, 0xcb, 0x19, 0xa1, 0x80, 0x6d, 0x0a, 0xc0, 0x28, + 0x8f, 0x68, 0xe8, 0xf6, 0x6b, 0xc4, 0xc5, 0x12, 0xd1, 0xaf, 0xb9, 0x91, 0xfe, 0xac, 0xb3, 0xcb, + 0x10, 0x45, 0x0a, 0x78, 0x75, 0x44, 0xc8, 0x68, 0x82, 0x24, 0x8f, 0x93, 0xf8, 0x74, 0x0b, 0xf9, + 0x21, 0xbb, 0x94, 0x83, 0xe6, 0x8f, 0x45, 0x58, 0xdb, 0xa5, 0xc8, 0x61, 0x68, 0x57, 0x2b, 0x60, + 0xa1, 0xef, 0x63, 0x14, 0x31, 0xe3, 0x0d, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x15, 0xee, 0x14, + 0x36, 0x6a, 0x56, 0x3d, 0xc1, 0x1d, 0x7a, 0xc6, 0x4d, 0xa8, 0xa0, 0x0b, 0xe4, 0xf2, 0xd1, 0xa2, + 0x18, 0x5d, 0xe6, 0xe0, 0xa1, 0x67, 0xbc, 0x0f, 0xf5, 0x88, 0x51, 0x1c, 0x8c, 0xec, 0x38, 0x42, + 0xb4, 0x57, 0xba, 0x53, 0xd8, 0xa8, 0x3f, 0x5c, 0xd9, 0xe4, 0x2a, 0x6f, 0x0e, 0xc5, 0xc0, 0x71, + 0x84, 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x83, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x5e, 0xf9, + 0x4e, 0x69, 0xa3, 0xfe, 0xb0, 0x21, 0xc9, 0x9f, 0x08, 0xa4, 0xa5, 0x07, 0x8d, 0xfb, 0x50, 0x8d, + 0x18, 0xa1, 0xce, 0x08, 0x45, 0xbd, 0x25, 0x41, 0xd8, 0xd4, 0x7c, 0x05, 0xd6, 0x4a, 0x86, 0x8d, + 0xdb, 0x50, 0x7a, 0xba, 0x7b, 0xd8, 0x5b, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, + 0xb8, 0x0b, 0xcd, 0xc8, 0x09, 0xbc, 0x13, 0x72, 0x61, 0x87, 0xd8, 0x0b, 0xa2, 0x5e, 0xe5, 0x4e, + 0x61, 0xa3, 0x6a, 0x35, 0x14, 0x72, 0xc0, 0x71, 0xe6, 0xa7, 0x70, 0x63, 0xc8, 0x1c, 0xca, 0xae, + 0xe1, 0x1d, 0xf3, 0x18, 0xd6, 0x2c, 0xe4, 0x93, 0xf3, 0x6b, 0xb9, 0xb6, 0x07, 0x15, 0x86, 0x7d, + 0x44, 0x62, 0x26, 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0x4f, 0x01, 0x8c, 0xbd, 0x0b, 0xe4, 0x0e, + 0x28, 0x71, 0x51, 0x14, 0xfd, 0x85, 0x96, 0xeb, 0x6d, 0xa8, 0x84, 0x52, 0x81, 0x5e, 0x59, 0x90, + 0xab, 0x55, 0xd0, 0x5a, 0xe9, 0x51, 0xf3, 0x3b, 0x58, 0x1d, 0xe2, 0x51, 0xe0, 0x4c, 0x5e, 0xa2, + 0xbe, 0x6b, 0xb0, 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0x73, 0x00, 0xc6, 0xb7, 0x0e, + 0x66, 0x2f, 0x4f, 0x92, 0xf9, 0x00, 0xba, 0x39, 0x8e, 0x51, 0x48, 0x82, 0x08, 0x09, 0x05, 0x98, + 0xc3, 0xe2, 0x48, 0x30, 0x5b, 0xb2, 0x14, 0x64, 0x12, 0x58, 0x3b, 0x0e, 0xbd, 0x6b, 0xee, 0xa6, + 0x87, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x7b, 0xa0, 0x28, 0x9c, 0xba, 0x2a, 0x9d, 0xfa, 0x35, + 0x0e, 0xe2, 0x0b, 0x4b, 0x8f, 0x59, 0x29, 0x99, 0x8a, 0x4f, 0x16, 0x5d, 0x27, 0x3e, 0x3f, 0x85, + 0x1b, 0x03, 0x27, 0x8e, 0xae, 0xa3, 0xab, 0xf9, 0x19, 0x8f, 0xed, 0x28, 0xf6, 0xaf, 0x35, 0xf9, + 0xbf, 0x0b, 0x50, 0xdd, 0x0d, 0xe3, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0x75, 0xa8, 0x33, 0xc2, 0x9c, + 0x89, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, 0x49, 0xf0, 0x06, 0x34, 0x42, 0x44, 0xdd, + 0x30, 0x56, 0x14, 0xc5, 0x3b, 0xa5, 0x8d, 0xb2, 0x55, 0x97, 0x38, 0x49, 0xb2, 0x09, 0x5d, 0x31, + 0x66, 0xe3, 0xc0, 0x3e, 0x43, 0x34, 0x40, 0x13, 0x9f, 0x78, 0x48, 0x04, 0x47, 0xd9, 0xea, 0x88, + 0xa1, 0xc3, 0xe0, 0xab, 0x64, 0xc0, 0x78, 0x07, 0x3a, 0x09, 0x3d, 0x8f, 0x78, 0x41, 0x5d, 0x16, + 0xd4, 0x6d, 0x45, 0x7d, 0xac, 0xd0, 0xe6, 0xbf, 0x42, 0xeb, 0xd9, 0x98, 0x12, 0xc6, 0x26, 0x38, + 0x18, 0x3d, 0x71, 0x98, 0xc3, 0xb7, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, + 0x2e, 0x74, 0x98, 0xa4, 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x25, 0x19, 0x18, 0x28, 0xe2, + 0xb7, 0xa0, 0x95, 0x12, 0xf3, 0xcd, 0xad, 0xf4, 0x6d, 0x26, 0xd8, 0x67, 0xd8, 0x47, 0xe6, 0xb9, + 0xf0, 0x95, 0x58, 0x64, 0xe3, 0x5d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x08, 0x69, 0xc9, 0x08, 0xd1, + 0xee, 0xb4, 0xaa, 0x89, 0x53, 0x3e, 0x87, 0x36, 0x4b, 0x14, 0xb7, 0x3d, 0x87, 0x39, 0xf9, 0xa0, + 0xca, 0x5b, 0x65, 0xb5, 0x58, 0x0e, 0x36, 0x3f, 0x83, 0xda, 0x00, 0x7b, 0x91, 0x14, 0xdc, 0x83, + 0x8a, 0x1b, 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x0a, 0x4b, 0x13, 0xec, 0x63, 0xa6, + 0xcc, 0x94, 0x80, 0x49, 0x00, 0x8e, 0x90, 0x4f, 0xe8, 0xa5, 0x70, 0xd8, 0x2a, 0x2c, 0x65, 0x17, + 0x57, 0x02, 0xc6, 0xab, 0x50, 0xf3, 0x9d, 0x8b, 0x64, 0x51, 0xf9, 0x48, 0xd5, 0x77, 0x2e, 0xa4, + 0xf2, 0x3d, 0xa8, 0x9c, 0x3a, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x72, 0x56, + 0xe0, 0xff, 0x15, 0xa1, 0x2e, 0x25, 0x4a, 0x85, 0x57, 0x61, 0xc9, 0x75, 0xdc, 0x71, 0x22, 0x52, + 0x00, 0xc6, 0x3d, 0xad, 0x48, 0x31, 0x9b, 0xe1, 0x52, 0x4d, 0xb5, 0x6a, 0x5b, 0x00, 0xd1, 0x73, + 0x27, 0x54, 0xba, 0x95, 0x16, 0x10, 0xd7, 0x38, 0x8d, 0x54, 0xf7, 0x03, 0x68, 0xc8, 0xb8, 0x53, + 0x53, 0xca, 0x0b, 0xa6, 0xd4, 0x25, 0x95, 0x9c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x31, 0x46, + 0xd4, 0xa1, 0xee, 0xf8, 0xb2, 0xb7, 0x24, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0x3c, 0x84, + 0x25, 0x9e, 0x5b, 0xa2, 0xde, 0xb2, 0x38, 0xeb, 0x6e, 0x67, 0x59, 0x0a, 0x53, 0x37, 0xc5, 0xef, + 0x5e, 0xc0, 0xe8, 0xa5, 0x25, 0x49, 0xfb, 0x1f, 0x03, 0xa4, 0x48, 0x63, 0x05, 0x4a, 0x67, 0xe8, + 0x52, 0xed, 0x43, 0xfe, 0xc9, 0x9d, 0x73, 0xee, 0x4c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x5a, 0xfc, + 0xb8, 0x60, 0xba, 0xd0, 0xde, 0x99, 0x9c, 0x61, 0x92, 0x99, 0xbe, 0x0a, 0x4b, 0xbe, 0xf3, 0x1d, + 0xa1, 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84, + 0xc2, 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x7d, 0x19, 0x20, 0x95, + 0x62, 0x58, 0xd0, 0xc7, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x72, 0xc9, 0x50, 0x64, 0x53, + 0xe4, 0xc6, 0x34, 0xc2, 0xe7, 0x7c, 0xfd, 0xb8, 0xd9, 0x37, 0xa4, 0xd9, 0x53, 0xba, 0x59, 0x37, + 0x31, 0x19, 0xca, 0x79, 0x3b, 0x7c, 0x9a, 0xa5, 0x67, 0x19, 0x87, 0x70, 0x23, 0xe5, 0xe9, 0x65, + 0xd8, 0x15, 0xaf, 0x62, 0xd7, 0x4d, 0xd8, 0x79, 0x29, 0xab, 0x3d, 0xe8, 0x62, 0x62, 0x7f, 0x1f, + 0xa3, 0x38, 0xc7, 0xa8, 0x74, 0x15, 0xa3, 0x0e, 0x26, 0x7f, 0x2b, 0x26, 0xa4, 0x6c, 0x06, 0x70, + 0x2b, 0x63, 0x25, 0xdf, 0xee, 0x19, 0x66, 0xe5, 0xab, 0x98, 0xad, 0x25, 0x5a, 0xf1, 0x7c, 0x90, + 0x72, 0xfc, 0x12, 0xd6, 0x30, 0xb1, 0x9f, 0x3b, 0x98, 0x4d, 0xb3, 0x5b, 0xfa, 0x05, 0x23, 0xf9, + 0x89, 0x96, 0xe7, 0x25, 0x8d, 0xf4, 0x11, 0x1d, 0xe5, 0x8c, 0x5c, 0xfe, 0x05, 0x23, 0x8f, 0xc4, + 0x84, 0x94, 0xcd, 0x36, 0x74, 0x30, 0x99, 0xd6, 0xa6, 0x72, 0x15, 0x93, 0x36, 0x26, 0x79, 0x4d, + 0x76, 0xa0, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x0d, 0x82, 0xea, 0x55, 0x2c, 0x56, 0x14, 0x7d, 0xc2, + 0xc3, 0xfc, 0x47, 0x68, 0x1c, 0xc4, 0x23, 0xc4, 0x26, 0x27, 0x49, 0x32, 0x78, 0x69, 0xf9, 0xc7, + 0xfc, 0x63, 0x11, 0xea, 0xbb, 0x23, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xce, 0xc9, + 0x82, 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x21, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d, + 0x99, 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x04, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d, + 0xb5, 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x3e, 0xd4, 0x4f, 0xb8, 0x93, + 0xd4, 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x49, 0xba, 0x09, 0x0f, 0xa0, 0x39, 0x96, 0x2e, + 0x53, 0x93, 0x64, 0x0c, 0xdd, 0x55, 0x96, 0xa4, 0xf6, 0x6e, 0x66, 0x3d, 0x2b, 0x17, 0xa0, 0x31, + 0xce, 0xa0, 0xfa, 0x43, 0xe8, 0xcc, 0x90, 0xcc, 0xc9, 0x41, 0x1b, 0xd9, 0x1c, 0x54, 0x7f, 0x68, + 0x48, 0x41, 0xd9, 0x99, 0xd9, 0xbc, 0xf4, 0x9f, 0x45, 0x68, 0x7c, 0x83, 0xd8, 0x73, 0x42, 0xcf, + 0xa4, 0xbe, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa3, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e, 0xc8, + 0x04, 0xa2, 0xd6, 0xb3, 0x42, 0x2f, 0x44, 0x62, 0x30, 0x5e, 0x03, 0xa0, 0x17, 0x76, 0xe8, 0xb8, + 0x67, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xc5, 0x40, 0x22, 0x78, 0x28, 0xd0, 0x0b, 0x1b, 0x51, + 0x4a, 0x68, 0xa4, 0x72, 0x55, 0x95, 0x5e, 0xec, 0x09, 0x58, 0xcd, 0xf5, 0x28, 0x09, 0x43, 0xe4, + 0x89, 0x1c, 0x2d, 0xe6, 0x3e, 0x91, 0x08, 0x2e, 0x95, 0x69, 0xa9, 0xcb, 0x52, 0x2a, 0x4b, 0xa5, + 0xb2, 0x54, 0x6a, 0x45, 0xce, 0x64, 0x59, 0xa9, 0x2c, 0x91, 0x5a, 0x95, 0x52, 0x59, 0x46, 0x2a, + 0x4b, 0xa5, 0xd6, 0xf4, 0x5c, 0x25, 0xd5, 0xfc, 0x8f, 0x02, 0xac, 0x4d, 0x17, 0x7e, 0xaa, 0x36, + 0xfd, 0x10, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0x66, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30, + 0x7e, 0x04, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, + 0x03, 0x99, 0x1e, 0x18, 0xdf, 0x52, 0xcc, 0xd0, 0x90, 0x51, 0xe4, 0xf8, 0x2f, 0xa3, 0xba, 0x37, + 0xa0, 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0x43, 0x37, 0x27, 0x45, 0xd9, + 0xba, 0x02, 0xa5, 0x09, 0x0a, 0x04, 0xf7, 0xa6, 0xc5, 0x3f, 0x4d, 0x07, 0x3a, 0x16, 0x72, 0xbc, + 0x97, 0xa7, 0x8d, 0x12, 0x51, 0x4a, 0x45, 0x6c, 0x80, 0x91, 0x15, 0xa1, 0x54, 0xd1, 0x5a, 0x17, + 0x32, 0x5a, 0x3f, 0x85, 0xce, 0xee, 0x84, 0x44, 0x68, 0xc8, 0x3c, 0x1c, 0xbc, 0x8c, 0x76, 0xe4, + 0x5f, 0xa0, 0xfb, 0x8c, 0x5d, 0x7e, 0xcb, 0x99, 0x45, 0xf8, 0x07, 0xf4, 0x92, 0xec, 0xa3, 0xe4, + 0xb9, 0xb6, 0x8f, 0x92, 0xe7, 0xbc, 0xb9, 0x71, 0xc9, 0x24, 0xf6, 0x03, 0xb1, 0x15, 0x9a, 0x96, + 0x82, 0xcc, 0x1d, 0x68, 0xc8, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0xcd, 0xdd, 0x83, 0xeb, 0x00, + 0xa1, 0x43, 0x1d, 0x1f, 0x31, 0x44, 0x65, 0x0c, 0xd5, 0xac, 0x0c, 0xc6, 0xfc, 0xaf, 0x22, 0xac, + 0xca, 0xfb, 0x86, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x7d, 0xa8, 0x8e, 0x49, 0xc4, 0x32, 0x0c, 0x13, + 0x98, 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0xe6, 0x2e, 0x01, 0x4a, 0x57, 0x5f, 0x02, 0xcc, + 0xb4, 0xf9, 0xe5, 0xd9, 0x36, 0x9f, 0xef, 0x36, 0x4d, 0x84, 0xe5, 0x1e, 0xaf, 0x59, 0x35, 0x85, + 0x39, 0xf4, 0x8c, 0x7b, 0xd0, 0x1e, 0x71, 0x2d, 0xed, 0x31, 0x21, 0x67, 0x76, 0xe8, 0xb0, 0xb1, + 0xd8, 0xea, 0x35, 0xab, 0x29, 0xd0, 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0x3e, 0x81, 0x96, + 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, 0x1d, 0x7e, 0x6a, 0x17, 0x65, 0xbd, 0x67, 0x35, 0xcf, 0x32, + 0x50, 0x64, 0xde, 0x84, 0x1b, 0x4f, 0x50, 0xc4, 0x28, 0xb9, 0xcc, 0x3b, 0xc6, 0xfc, 0x6b, 0x80, + 0xc3, 0x80, 0x21, 0x7a, 0xea, 0xb8, 0x28, 0x32, 0xde, 0xcb, 0x42, 0xaa, 0x38, 0x5a, 0xd9, 0x94, + 0xd7, 0x3d, 0xc9, 0x80, 0x95, 0xa1, 0x31, 0x37, 0x61, 0xd9, 0x22, 0x31, 0x4f, 0x47, 0x6f, 0xea, + 0x2f, 0x35, 0xaf, 0xa1, 0xe6, 0x09, 0xa4, 0xa5, 0xc6, 0xcc, 0x03, 0xdd, 0xc2, 0xa6, 0xec, 0xd4, + 0x12, 0x6d, 0x42, 0x0d, 0x6b, 0x9c, 0xca, 0x2a, 0xb3, 0xa2, 0x53, 0x12, 0xf3, 0x33, 0xe8, 0x4a, + 0x4e, 0x92, 0xb3, 0x66, 0xf3, 0x26, 0x2c, 0x53, 0xad, 0x46, 0x21, 0xbd, 0xe7, 0x51, 0x44, 0x6a, + 0xcc, 0x3c, 0x84, 0xdb, 0x72, 0xf2, 0x5e, 0x38, 0x46, 0x3e, 0xa2, 0xce, 0xe4, 0x88, 0xc4, 0x01, + 0x4b, 0xb8, 0x64, 0x23, 0xa0, 0x70, 0x65, 0x04, 0x70, 0xd7, 0x7e, 0x8d, 0x23, 0x96, 0xfa, 0x44, + 0xbb, 0xb6, 0x0b, 0x1d, 0x3e, 0x90, 0x53, 0xcf, 0xfc, 0x02, 0x1a, 0xdb, 0xd6, 0xe0, 0x1b, 0x84, + 0x47, 0xe3, 0x13, 0x9e, 0x88, 0x3f, 0xca, 0xc3, 0x4a, 0x98, 0xa1, 0x0c, 0xcf, 0x0c, 0x59, 0x39, + 0x3a, 0xf3, 0x4b, 0x58, 0xdb, 0xf6, 0xbc, 0x2c, 0x4a, 0xab, 0xfe, 0x1e, 0xd4, 0x82, 0x0c, 0xbb, + 0xcc, 0xf1, 0x97, 0xa3, 0x4e, 0x89, 0xcc, 0x07, 0x60, 0xec, 0x23, 0x76, 0x38, 0x78, 0xe6, 0x9c, + 0x4c, 0x52, 0x47, 0xde, 0x84, 0x0a, 0x8e, 0x6c, 0x1c, 0x9e, 0x7f, 0x24, 0xb8, 0x54, 0xad, 0x65, + 0x1c, 0x1d, 0x86, 0xe7, 0x1f, 0x99, 0xf7, 0xa1, 0x9b, 0x23, 0xbf, 0x22, 0x43, 0x6d, 0x83, 0x31, + 0xfc, 0xf5, 0x9c, 0x13, 0x16, 0xc5, 0x0c, 0x8b, 0xfb, 0xd0, 0x1d, 0xfe, 0x4a, 0x69, 0xff, 0x04, + 0xdd, 0xa7, 0xc1, 0x04, 0x07, 0x68, 0x77, 0x70, 0x7c, 0x84, 0x92, 0xf4, 0x6c, 0x40, 0x99, 0x97, + 0xb1, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0x27, 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xff, 0x5a, 0x0e, + 0x4e, 0x76, 0xc3, 0x38, 0xe2, 0xe7, 0x2d, 0xaf, 0xb7, 0x48, 0x30, 0xb9, 0x14, 0x49, 0xab, 0x6a, + 0x55, 0xdc, 0x30, 0x7e, 0x1a, 0x4c, 0x2e, 0xcd, 0xbf, 0x12, 0x97, 0x12, 0x08, 0x79, 0x96, 0x13, + 0x78, 0xc4, 0x7f, 0x82, 0xce, 0x33, 0x12, 0x66, 0xf4, 0xfe, 0xa9, 0x00, 0x8d, 0xed, 0x11, 0x0a, + 0xd8, 0x13, 0xc4, 0x1c, 0x3c, 0x11, 0x4d, 0xee, 0x39, 0xa2, 0x11, 0x26, 0x81, 0xca, 0x40, 0x1a, + 0x34, 0x5e, 0x87, 0x3a, 0x0e, 0x30, 0xb3, 0x3d, 0x07, 0xf9, 0x24, 0x10, 0x5c, 0xaa, 0x16, 0x70, + 0xd4, 0x13, 0x81, 0x31, 0xde, 0x86, 0xb6, 0xbc, 0x9f, 0xb4, 0xc7, 0x4e, 0xe0, 0x4d, 0x78, 0xee, + 0x2b, 0x89, 0x6c, 0xd5, 0x92, 0xe8, 0x03, 0x85, 0x35, 0xee, 0xc3, 0x8a, 0x8a, 0xcb, 0x94, 0xb2, + 0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2, + 0x87, 0xaa, 0x43, 0x6c, 0x6b, 0xfc, 0x50, 0xa2, 0xcd, 0x11, 0x74, 0xf7, 0xb9, 0x9d, 0xca, 0x92, + 0x74, 0xa7, 0xb5, 0x7c, 0xe4, 0xdb, 0x27, 0x13, 0xe2, 0x9e, 0xd9, 0xfc, 0xbc, 0x50, 0x1e, 0xe6, + 0x35, 0xe8, 0x0e, 0x47, 0x0e, 0xf1, 0x0f, 0xe2, 0x32, 0x84, 0x53, 0x8d, 0x09, 0x0b, 0x27, 0xf1, + 0xc8, 0x0e, 0x29, 0x39, 0x41, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, + 0xbf, 0x05, 0x58, 0xcd, 0x4b, 0x52, 0xab, 0xbd, 0x05, 0xab, 0x79, 0x51, 0xaa, 0x22, 0x92, 0x15, + 0x77, 0x27, 0x2b, 0x50, 0xd6, 0x46, 0x8f, 0xa0, 0x29, 0x6e, 0xb3, 0x6d, 0x4f, 0x72, 0xca, 0xd7, + 0x81, 0xd9, 0x75, 0xb1, 0x1a, 0x4e, 0x76, 0x95, 0x3e, 0x81, 0x5b, 0xca, 0x7c, 0x7b, 0x56, 0x6d, + 0x19, 0x10, 0x6b, 0x8a, 0xe0, 0x68, 0x4a, 0xfb, 0xaf, 0xa1, 0x97, 0xa2, 0x76, 0x2e, 0x05, 0x32, + 0xdd, 0x94, 0xdd, 0x29, 0x63, 0xb7, 0x3d, 0x8f, 0x8a, 0xdd, 0x5e, 0xb6, 0xe6, 0x0d, 0x99, 0x8f, + 0xe1, 0xe6, 0x10, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xe6, 0x4c, 0x32, 0x5b, 0x81, 0xd2, 0x10, 0xb9, + 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, 0x47, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d, + 0x86, 0x50, 0xf9, 0x62, 0xb8, 0xcf, 0x4b, 0x30, 0x1e, 0xd4, 0xb2, 0x64, 0x53, 0xc7, 0x73, 0xd3, + 0xaa, 0x08, 0xf8, 0xd0, 0x33, 0xbe, 0x84, 0xae, 0x1c, 0x72, 0xc7, 0x4e, 0x30, 0x42, 0x76, 0x48, + 0x26, 0xd8, 0x95, 0xa1, 0xdf, 0x7a, 0xd8, 0x57, 0x69, 0x48, 0xf1, 0xd9, 0x15, 0x24, 0x03, 0x41, + 0x61, 0x75, 0x46, 0xd3, 0x28, 0xf3, 0x77, 0x05, 0xa8, 0xa8, 0xfc, 0xc8, 0x4f, 0x79, 0x8f, 0xe2, + 0x73, 0x44, 0x55, 0xb0, 0x2b, 0xc8, 0x78, 0x0b, 0x5a, 0xf2, 0xcb, 0x26, 0x21, 0xc3, 0x24, 0x39, + 0x77, 0x9b, 0x12, 0xfb, 0x54, 0x22, 0xc5, 0x0d, 0xa8, 0xb8, 0x83, 0x54, 0xed, 0xbe, 0x82, 0x38, + 0xfe, 0x34, 0xe2, 0x4a, 0x89, 0x73, 0xb6, 0x66, 0x29, 0x88, 0x6f, 0x2e, 0xcd, 0x6f, 0x49, 0xf0, + 0xd3, 0x20, 0xdf, 0x5c, 0x3e, 0x4f, 0xed, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0xc1, 0x0a, 0x02, 0x35, + 0xe0, 0x18, 0x63, 0x03, 0xaa, 0xa7, 0x91, 0x2d, 0xac, 0x11, 0x45, 0x74, 0x92, 0xea, 0x95, 0xd5, + 0x56, 0xe5, 0x34, 0x12, 0x1f, 0xe6, 0xbf, 0x17, 0x60, 0x59, 0xbe, 0x17, 0x18, 0x2d, 0x28, 0x26, + 0x85, 0x50, 0x11, 0x8b, 0xa2, 0x52, 0x68, 0x25, 0x8b, 0x1f, 0xf1, 0xcd, 0x73, 0xcc, 0xb9, 0x2f, + 0x8f, 0x73, 0x65, 0xc4, 0xb9, 0x2f, 0xce, 0xf1, 0xb7, 0xa0, 0x95, 0xd6, 0x53, 0x62, 0x5c, 0x1a, + 0xd3, 0x4c, 0xb0, 0x82, 0x6c, 0xa1, 0x4d, 0xe6, 0xdf, 0x03, 0xa4, 0xf7, 0xe6, 0x3c, 0x1c, 0xe2, + 0x44, 0x19, 0xfe, 0xc9, 0x31, 0xa3, 0xa4, 0x12, 0xe3, 0x9f, 0xc6, 0x3d, 0x68, 0x39, 0x9e, 0x87, + 0xf9, 0x74, 0x67, 0xb2, 0x8f, 0xbd, 0x24, 0x81, 0xe4, 0xb1, 0xe6, 0xff, 0x17, 0xa0, 0xbd, 0x4b, + 0xc2, 0xcb, 0x2f, 0xf0, 0x04, 0x65, 0xb2, 0x9b, 0x50, 0x52, 0x15, 0x62, 0xfc, 0x9b, 0x37, 0x17, + 0xa7, 0x78, 0x82, 0xe4, 0xb6, 0x97, 0x51, 0x57, 0xe5, 0x08, 0xb1, 0xe5, 0xf5, 0x60, 0x72, 0x4b, + 0xda, 0x94, 0x83, 0x47, 0xc4, 0x13, 0x6d, 0x94, 0x87, 0xa9, 0x9d, 0xdc, 0x89, 0x36, 0xad, 0x8a, + 0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x3b, 0xef, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59, + 0x83, 0x65, 0x72, 0x7a, 0x1a, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c, + 0x0a, 0x5e, 0x15, 0xe7, 0xda, 0xd3, 0xa7, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0xfa, 0x04, 0x7e, 0x00, + 0x55, 0x8d, 0xfa, 0x35, 0xb7, 0xc9, 0xef, 0x40, 0x6b, 0xdb, 0xf3, 0x86, 0xcf, 0x9d, 0x50, 0xfb, + 0xa3, 0x07, 0x95, 0xc1, 0xee, 0xe1, 0x40, 0xba, 0xa4, 0xc4, 0x0d, 0x50, 0x20, 0x3f, 0xf1, 0xf7, + 0x11, 0x3b, 0x42, 0x8c, 0x62, 0x37, 0x39, 0xf1, 0xef, 0x42, 0x45, 0x61, 0xf8, 0x4c, 0x5f, 0x7e, + 0xea, 0x23, 0x40, 0x81, 0xe6, 0xdf, 0x80, 0xf1, 0x77, 0xbc, 0x0c, 0x46, 0xb2, 0x07, 0x52, 0x92, + 0xde, 0x81, 0xce, 0xb9, 0xc0, 0xda, 0xb2, 0x3e, 0xcc, 0x2c, 0x43, 0x5b, 0x0e, 0x88, 0xfc, 0x20, + 0x64, 0x1f, 0x43, 0x57, 0x56, 0xed, 0x92, 0xcf, 0x35, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6, + 0xc4, 0xf7, 0xc3, 0x1f, 0xbb, 0xea, 0x18, 0x53, 0x97, 0x44, 0xc6, 0x3e, 0xb4, 0xa7, 0x5e, 0xf4, + 0x0c, 0x75, 0x6b, 0x38, 0xff, 0xa1, 0xaf, 0xbf, 0xb6, 0x29, 0x5f, 0x08, 0x37, 0xf5, 0x0b, 0xe1, + 0xe6, 0x9e, 0x1f, 0xb2, 0x4b, 0x63, 0x0f, 0x5a, 0xf9, 0xb7, 0x2f, 0xe3, 0x55, 0x5d, 0x62, 0xcd, + 0x79, 0x11, 0x5b, 0xc8, 0x66, 0x1f, 0xda, 0x53, 0xcf, 0x60, 0x5a, 0x9f, 0xf9, 0xaf, 0x63, 0x0b, + 0x19, 0x3d, 0x86, 0x7a, 0xe6, 0xdd, 0xcb, 0xe8, 0x49, 0x26, 0xb3, 0x4f, 0x61, 0x0b, 0x19, 0xec, + 0x42, 0x33, 0xf7, 0x14, 0x65, 0xf4, 0x95, 0x3d, 0x73, 0xde, 0xa7, 0x16, 0x32, 0xd9, 0x81, 0x7a, + 0xe6, 0x45, 0x48, 0x6b, 0x31, 0xfb, 0xec, 0xd4, 0xbf, 0x35, 0x67, 0x44, 0x9d, 0x96, 0xfb, 0xd0, + 0x9e, 0x7a, 0x26, 0xd2, 0x2e, 0x99, 0xff, 0x7a, 0xb4, 0x50, 0x99, 0x21, 0xdc, 0x98, 0x5b, 0x25, + 0x1b, 0x66, 0x96, 0xdd, 0xfc, 0x12, 0x7a, 0x21, 0xd3, 0xaf, 0xc4, 0xba, 0x67, 0xae, 0x16, 0x32, + 0xeb, 0x3e, 0xfb, 0xd2, 0xd4, 0xbf, 0x3d, 0x7f, 0x50, 0x99, 0xba, 0x07, 0xad, 0xfc, 0x23, 0x93, + 0x66, 0x36, 0xf7, 0xe9, 0xe9, 0xea, 0x20, 0xca, 0xbd, 0x37, 0xa5, 0x41, 0x34, 0xef, 0x19, 0x6a, + 0x21, 0xa3, 0x6d, 0x00, 0x75, 0x91, 0xe0, 0xe1, 0x20, 0x59, 0xbd, 0x99, 0x0b, 0x8c, 0x64, 0xf5, + 0xe6, 0x5c, 0x3a, 0x3c, 0x06, 0x90, 0xfd, 0xbf, 0x47, 0x62, 0x66, 0xdc, 0xd4, 0x6a, 0x4c, 0x5d, + 0x3a, 0xf4, 0x7b, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x39, 0x40, 0x7a, 0xaf, + 0xa0, 0x19, 0xcc, 0xdc, 0x34, 0x5c, 0xe1, 0x83, 0x46, 0xf6, 0x16, 0xc1, 0x50, 0xb6, 0xce, 0xb9, + 0x59, 0xb8, 0x82, 0x45, 0x7b, 0xaa, 0x4b, 0xcc, 0x47, 0xf0, 0x74, 0xf3, 0xd8, 0x9f, 0xe9, 0x14, + 0x8d, 0x47, 0xd0, 0xc8, 0xb6, 0x87, 0x5a, 0x8b, 0x39, 0x2d, 0x63, 0x3f, 0xd7, 0x22, 0x1a, 0x8f, + 0xa1, 0x95, 0xef, 0xe7, 0x74, 0x48, 0xcd, 0xed, 0xf2, 0xfa, 0xea, 0xe2, 0x33, 0x43, 0xfe, 0x01, + 0x40, 0xda, 0xf7, 0x69, 0xf7, 0xcd, 0x74, 0x82, 0x53, 0x52, 0xf7, 0xa1, 0x3d, 0xd5, 0xcf, 0x69, + 0x8b, 0xe7, 0xb7, 0x79, 0x57, 0x25, 0x90, 0x4c, 0x77, 0xa6, 0x43, 0x70, 0xb6, 0xbf, 0xd3, 0x21, + 0x38, 0xaf, 0x95, 0xdb, 0x81, 0xfa, 0x70, 0x96, 0xc7, 0x70, 0x21, 0x8f, 0x79, 0x0d, 0xda, 0x87, + 0x00, 0xe9, 0x59, 0xa8, 0xbd, 0x30, 0x73, 0x3a, 0xf6, 0x9b, 0xfa, 0x72, 0x5a, 0xd2, 0xed, 0x42, + 0x33, 0x77, 0x7f, 0xa3, 0x73, 0xe8, 0xbc, 0x4b, 0x9d, 0xab, 0x4e, 0x96, 0xfc, 0x65, 0x87, 0x5e, + 0xc1, 0xb9, 0x57, 0x20, 0x57, 0xc5, 0x71, 0xb6, 0x9d, 0xd4, 0x11, 0x34, 0xa7, 0xc5, 0xfc, 0x85, + 0xbc, 0x92, 0x6d, 0x19, 0x33, 0x79, 0x65, 0x4e, 0x27, 0xb9, 0x90, 0xd1, 0x01, 0xb4, 0xf7, 0x75, + 0x37, 0xa0, 0x3a, 0x15, 0xbd, 0x7e, 0xb3, 0x9d, 0x59, 0xbf, 0x3f, 0x6f, 0x48, 0xad, 0xcb, 0x57, + 0xd0, 0x99, 0xe9, 0x52, 0x8c, 0xf5, 0xe4, 0x89, 0x60, 0x6e, 0xfb, 0xb2, 0x50, 0xad, 0x43, 0x58, + 0x99, 0x6e, 0x52, 0x8c, 0xd7, 0x92, 0x98, 0x98, 0xd7, 0xbc, 0x2c, 0x64, 0xf5, 0x09, 0x54, 0x75, + 0xe1, 0x69, 0xa8, 0xa7, 0x98, 0xa9, 0x42, 0x74, 0xe1, 0xd4, 0x47, 0x22, 0xe4, 0x93, 0xa2, 0x2e, + 0x0d, 0xf9, 0xa9, 0xd2, 0xaf, 0xaf, 0x5e, 0x4e, 0x12, 0xca, 0x47, 0x50, 0x51, 0xb5, 0x9d, 0xb1, + 0x9a, 0x6c, 0xb6, 0x4c, 0xa9, 0x77, 0x55, 0x84, 0xed, 0x23, 0x96, 0xa9, 0xd8, 0xb4, 0xd0, 0xd9, + 0x22, 0x4e, 0xef, 0x91, 0xdc, 0x88, 0x5a, 0x8b, 0x6d, 0x68, 0x64, 0x6b, 0x36, 0xbd, 0xa4, 0x73, + 0xea, 0xb8, 0x45, 0x9a, 0xec, 0x5c, 0xfc, 0xf4, 0xf3, 0xfa, 0x2b, 0xbf, 0xfd, 0x79, 0xfd, 0x95, + 0x7f, 0x7b, 0xb1, 0x5e, 0xf8, 0xe9, 0xc5, 0x7a, 0xe1, 0x37, 0x2f, 0xd6, 0x0b, 0x7f, 0x78, 0xb1, + 0x5e, 0xf8, 0x87, 0x7f, 0xfe, 0x33, 0xff, 0x13, 0x46, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x9d, 0x63, + 0xca, 0x32, 0x43, 0xe1, 0xd9, 0x48, 0xfe, 0x31, 0x2c, 0xf3, 0x7f, 0x31, 0xae, 0xe5, 0xc9, 0xb2, + 0x80, 0x3f, 0xf8, 0x53, 0x00, 0x00, 0x00, 0xff, 0xff, 0xeb, 0xb0, 0x39, 0x1d, 0x7c, 0x26, 0x00, + 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -4858,6 +4901,47 @@ func (m *UpdateRoutesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *UpdateEphemeralMountsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UpdateEphemeralMountsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *UpdateEphemeralMountsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Storages) > 0 { + for iNdEx := len(m.Storages) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Storages[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintAgent(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *ListInterfacesRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6901,6 +6985,24 @@ func (m *UpdateRoutesRequest) Size() (n int) { return n } +func (m *UpdateEphemeralMountsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Storages) > 0 { + for _, e := range m.Storages { + l = e.Size() + n += 1 + l + sovAgent(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *ListInterfacesRequest) Size() (n int) { if m == nil { return 0 @@ -7664,7 +7766,7 @@ func (this *MemoryStats) String() string { return "nil" } keysForStats := make([]string, 0, len(this.Stats)) - for k := range this.Stats { + for k, _ := range this.Stats { keysForStats = append(keysForStats, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForStats) @@ -7775,7 +7877,7 @@ func (this *CgroupStats) String() string { return "nil" } keysForHugetlbStats := make([]string, 0, len(this.HugetlbStats)) - for k := range this.HugetlbStats { + for k, _ := range this.HugetlbStats { keysForHugetlbStats = append(keysForHugetlbStats, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForHugetlbStats) @@ -8008,6 +8110,22 @@ func (this *UpdateRoutesRequest) String() string { }, "") return s } +func (this *UpdateEphemeralMountsRequest) String() string { + if this == nil { + return "nil" + } + repeatedStringForStorages := "[]*Storage{" + for _, f := range this.Storages { + repeatedStringForStorages += strings.Replace(f.String(), "Storage", "Storage", 1) + "," + } + repeatedStringForStorages += "}" + s := strings.Join([]string{`&UpdateEphemeralMountsRequest{`, + `Storages:` + repeatedStringForStorages + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func (this *ListInterfacesRequest) String() string { if this == nil { return "nil" @@ -8355,6 +8473,7 @@ type AgentServiceService interface { SignalProcess(ctx context.Context, req *SignalProcessRequest) (*types.Empty, error) WaitProcess(ctx context.Context, req *WaitProcessRequest) (*WaitProcessResponse, error) UpdateContainer(ctx context.Context, req *UpdateContainerRequest) (*types.Empty, error) + UpdateEphemeralMounts(ctx context.Context, req *UpdateEphemeralMountsRequest) (*types.Empty, error) StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error) PauseContainer(ctx context.Context, req *PauseContainerRequest) (*types.Empty, error) ResumeContainer(ctx context.Context, req *ResumeContainerRequest) (*types.Empty, error) @@ -8436,6 +8555,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag } return svc.UpdateContainer(ctx, &req) }, + "UpdateEphemeralMounts": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req UpdateEphemeralMountsRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.UpdateEphemeralMounts(ctx, &req) + }, "StatsContainer": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { var req StatsContainerRequest if err := unmarshal(&req); err != nil { @@ -8701,6 +8827,14 @@ func (c *agentServiceClient) UpdateContainer(ctx context.Context, req *UpdateCon return &resp, nil } +func (c *agentServiceClient) UpdateEphemeralMounts(ctx context.Context, req *UpdateEphemeralMountsRequest) (*types.Empty, error) { + var resp types.Empty + if err := c.client.Call(ctx, "grpc.AgentService", "UpdateEphemeralMounts", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + func (c *agentServiceClient) StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error) { var resp StatsContainerResponse if err := c.client.Call(ctx, "grpc.AgentService", "StatsContainer", req, &resp); err != nil { @@ -13934,6 +14068,91 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *UpdateEphemeralMountsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UpdateEphemeralMountsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UpdateEphemeralMountsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Storages", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Storages = append(m.Storages, &Storage{}) + if err := m.Storages[len(m.Storages)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ListInterfacesRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go index f06524697..255f5ce45 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go @@ -423,7 +423,13 @@ type Hooks struct { // Poststart is a list of hooks to be run after the container process is started. Poststart []Hook `protobuf:"bytes,2,rep,name=Poststart,proto3" json:"Poststart"` // Poststop is a list of hooks to be run after the container process exits. - Poststop []Hook `protobuf:"bytes,3,rep,name=Poststop,proto3" json:"Poststop"` + Poststop []Hook `protobuf:"bytes,3,rep,name=Poststop,proto3" json:"Poststop"` + // Createruntime is a list of hooks to be run during the creation of runtime(sandbox). + CreateRuntime []Hook `protobuf:"bytes,4,rep,name=CreateRuntime,proto3" json:"CreateRuntime"` + // CreateContainer is a list of hooks to be run after VM is started, and before container is created. + CreateContainer []Hook `protobuf:"bytes,5,rep,name=CreateContainer,proto3" json:"CreateContainer"` + // StartContainer is a list of hooks to be run after container is created, but before it is started. + StartContainer []Hook `protobuf:"bytes,6,rep,name=StartContainer,proto3" json:"StartContainer"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1390,6 +1396,7 @@ type LinuxSyscall struct { Names []string `protobuf:"bytes,1,rep,name=Names,proto3" json:"Names,omitempty"` Action string `protobuf:"bytes,2,opt,name=Action,proto3" json:"Action,omitempty"` // Types that are valid to be assigned to ErrnoRet: + // // *LinuxSyscall_Errnoret ErrnoRet isLinuxSyscall_ErrnoRet `protobuf_oneof:"ErrnoRet"` Args []LinuxSeccompArg `protobuf:"bytes,4,rep,name=Args,proto3" json:"Args"` @@ -1546,140 +1553,143 @@ func init() { } var fileDescriptor_82a9ef0098ca0b24 = []byte{ - // 2121 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0xcb, 0x73, 0x5b, 0x49, - 0xd5, 0xcf, 0x95, 0x64, 0x59, 0x6a, 0xc5, 0x79, 0xf4, 0x64, 0x32, 0xf7, 0xcb, 0x97, 0xd2, 0x78, - 0x2e, 0x29, 0x30, 0x10, 0xac, 0x22, 0xe1, 0x31, 0x0c, 0x8f, 0x2a, 0xd9, 0x4e, 0x62, 0xd5, 0xc4, - 0xb1, 0x68, 0xd9, 0x13, 0x98, 0xc5, 0x54, 0xb5, 0xaf, 0xda, 0x52, 0x8f, 0xaf, 0x6e, 0xdf, 0xea, - 0xdb, 0xb2, 0x63, 0x56, 0xb0, 0x63, 0xcf, 0x82, 0x35, 0x1b, 0x1e, 0xff, 0x01, 0xc5, 0x8a, 0x1d, - 0x29, 0x56, 0x2c, 0xa9, 0xa2, 0x0a, 0x88, 0xf7, 0xec, 0x59, 0x52, 0xa7, 0xfb, 0xdc, 0xab, 0x96, - 0xe4, 0xc0, 0x04, 0x56, 0xea, 0xf3, 0x3b, 0x8f, 0x7e, 0x9c, 0xe7, 0x15, 0x79, 0x36, 0x92, 0x66, - 0x3c, 0x3d, 0xda, 0x8c, 0xd5, 0xa4, 0x73, 0xc2, 0x0d, 0xff, 0x4a, 0xac, 0x52, 0xc3, 0x65, 0x2a, - 0x74, 0xbe, 0x44, 0xe7, 0x3a, 0xee, 0x24, 0xf2, 0x28, 0xef, 0x64, 0x5a, 0x19, 0x15, 0xab, 0x04, - 0x57, 0x79, 0x47, 0xc5, 0x72, 0xd3, 0x2e, 0x69, 0x6d, 0xa4, 0xb3, 0xf8, 0x4e, 0x34, 0x52, 0x23, - 0xe5, 0x98, 0x47, 0xd3, 0xe3, 0x0e, 0x50, 0x96, 0xb0, 0x2b, 0x27, 0x19, 0xfd, 0xa1, 0x4a, 0x6a, - 0x83, 0x4c, 0xc4, 0x34, 0x24, 0xab, 0x1f, 0x09, 0x9d, 0x4b, 0x95, 0x86, 0xc1, 0x7a, 0xb0, 0xd1, - 0x64, 0x05, 0x49, 0xbf, 0x40, 0x56, 0xfb, 0x5a, 0xc5, 0x22, 0xcf, 0xc3, 0xca, 0x7a, 0xb0, 0xd1, - 0x7a, 0xb0, 0xb6, 0x09, 0xe6, 0x37, 0x11, 0x64, 0x05, 0x97, 0xb6, 0x49, 0x8d, 0x29, 0x65, 0xc2, - 0xaa, 0x95, 0x22, 0x4e, 0x0a, 0x10, 0x66, 0x71, 0x7a, 0x87, 0x34, 0x76, 0x55, 0x6e, 0x52, 0x3e, - 0x11, 0x61, 0xcd, 0xee, 0x51, 0xd2, 0xf4, 0x8b, 0xa4, 0xbe, 0xa7, 0xa6, 0xa9, 0xc9, 0xc3, 0x95, - 0xf5, 0xea, 0x46, 0xeb, 0x41, 0xcb, 0x69, 0x5b, 0x6c, 0xab, 0xf6, 0xf2, 0xaf, 0xef, 0x5e, 0x61, - 0x28, 0x40, 0xdf, 0x23, 0x2b, 0xbb, 0x4a, 0x9d, 0xe4, 0x61, 0xdd, 0xee, 0x83, 0x92, 0x16, 0x62, - 0x8e, 0x43, 0xbf, 0x4b, 0x5a, 0xdd, 0x34, 0x55, 0x86, 0x1b, 0xa9, 0xd2, 0x3c, 0x5c, 0xb5, 0x26, - 0xff, 0xdf, 0x09, 0xc2, 0x6d, 0x37, 0x3d, 0xee, 0xa3, 0xd4, 0xe8, 0x73, 0xe6, 0xcb, 0xc3, 0x0e, - 0x4f, 0x65, 0x3a, 0x7d, 0x11, 0x36, 0xfc, 0x1d, 0x2c, 0xc4, 0x1c, 0x07, 0x1e, 0x65, 0xa0, 0x12, - 0xae, 0x65, 0x1e, 0x36, 0xfd, 0x47, 0x41, 0x90, 0x15, 0x5c, 0x10, 0x7c, 0x2e, 0xd3, 0xa1, 0x3a, - 0xcb, 0x43, 0xe2, 0x0b, 0x22, 0xc8, 0x0a, 0xee, 0x9d, 0xef, 0x91, 0x1b, 0x8b, 0xa7, 0xa2, 0x37, - 0x48, 0xf5, 0x44, 0x9c, 0xa3, 0x43, 0x60, 0x49, 0x6f, 0x91, 0x95, 0x53, 0x9e, 0x4c, 0x85, 0x75, - 0x45, 0x93, 0x39, 0xe2, 0x83, 0xca, 0xfb, 0x41, 0xf4, 0xbb, 0x6a, 0xe9, 0x27, 0x78, 0xe9, 0x03, - 0xa1, 0x27, 0x32, 0xe5, 0x89, 0x55, 0x6e, 0xb0, 0x92, 0xa6, 0x5f, 0x26, 0xad, 0x6d, 0x95, 0xe6, - 0x2a, 0x11, 0x03, 0xf9, 0x23, 0x81, 0x2e, 0x6d, 0xba, 0x43, 0x6d, 0xa9, 0x17, 0xcc, 0xe7, 0xd2, - 0x7b, 0xa4, 0x76, 0x98, 0x0b, 0x3d, 0xef, 0x52, 0x40, 0xd0, 0x27, 0x96, 0x4b, 0x29, 0xa9, 0x75, - 0xf5, 0x28, 0x0f, 0x6b, 0xeb, 0xd5, 0x8d, 0x26, 0xb3, 0x6b, 0x38, 0xfa, 0xa3, 0xf4, 0xd4, 0x7a, - 0xb3, 0xc9, 0x60, 0x09, 0xc8, 0xf6, 0xd9, 0xd0, 0x7a, 0xad, 0xc9, 0x60, 0x49, 0xbf, 0x4d, 0xae, - 0x6e, 0xf3, 0x8c, 0x1f, 0xc9, 0x44, 0x1a, 0x29, 0xc0, 0x4f, 0xb0, 0xcb, 0x3b, 0xde, 0x73, 0xfb, - 0x6c, 0x36, 0x27, 0x4c, 0xbf, 0x4a, 0x56, 0x59, 0x22, 0x27, 0xd2, 0xe4, 0x61, 0xc3, 0xfa, 0xf7, - 0x26, 0x86, 0xe5, 0xfe, 0xa0, 0xf7, 0x03, 0xc7, 0xc1, 0x43, 0x16, 0x72, 0x74, 0x83, 0x5c, 0x7f, - 0xa6, 0x9e, 0x89, 0xb3, 0xbe, 0x96, 0xa7, 0x32, 0x11, 0x23, 0xe1, 0x9c, 0xd7, 0x60, 0x8b, 0x30, - 0x48, 0x76, 0xb3, 0x8c, 0xeb, 0x89, 0xd2, 0x7d, 0xad, 0x8e, 0x65, 0x22, 0xac, 0xf7, 0x9a, 0x6c, - 0x11, 0xa6, 0xeb, 0xa4, 0xb5, 0xbf, 0xbf, 0x37, 0x88, 0x95, 0x16, 0xdd, 0xe1, 0xa7, 0x61, 0x6b, - 0x3d, 0xd8, 0xa8, 0x32, 0x1f, 0xa2, 0x11, 0xb9, 0x3a, 0x10, 0x09, 0xdc, 0xe6, 0x29, 0x3f, 0x12, - 0x49, 0x78, 0xd5, 0x1a, 0x9a, 0xc3, 0xa2, 0x87, 0xa4, 0xba, 0xa5, 0x5e, 0xd0, 0xdb, 0xa4, 0xbe, - 0x2b, 0xe4, 0x68, 0x6c, 0xac, 0xd7, 0xd6, 0x18, 0x52, 0xe0, 0xf5, 0xe7, 0x72, 0x68, 0xc6, 0xd6, - 0x5b, 0x6b, 0xcc, 0x11, 0x51, 0xea, 0x9c, 0x03, 0x0f, 0x7b, 0xd8, 0xdb, 0x41, 0x15, 0x58, 0x02, - 0xf2, 0xa4, 0xb7, 0x83, 0xd2, 0xb0, 0xa4, 0x9f, 0x27, 0xd7, 0xba, 0xc3, 0xa1, 0x84, 0xd8, 0xe2, - 0xc9, 0x13, 0x39, 0xcc, 0xc3, 0xea, 0x7a, 0x75, 0x63, 0x8d, 0x2d, 0xa0, 0x10, 0x39, 0x60, 0xd3, - 0xcf, 0xd1, 0x82, 0x8e, 0x7e, 0x15, 0x90, 0x9b, 0x4b, 0x5e, 0x01, 0x8d, 0x2d, 0x35, 0x4d, 0x87, - 0x32, 0x1d, 0x85, 0x81, 0xf5, 0x76, 0x49, 0xd3, 0xbb, 0xa4, 0xf9, 0xe8, 0xf8, 0x58, 0xc4, 0x46, - 0x9e, 0x42, 0xa4, 0x01, 0x73, 0x06, 0xc0, 0xd3, 0xf5, 0xd2, 0xb1, 0xd0, 0xd2, 0xf0, 0xa3, 0x44, - 0xd8, 0x03, 0x35, 0x99, 0x0f, 0x81, 0x7e, 0x1f, 0xe2, 0xd6, 0x18, 0x31, 0xc4, 0xe8, 0x9a, 0x01, - 0x50, 0xb2, 0xba, 0x93, 0x23, 0x29, 0x52, 0x83, 0x61, 0x56, 0x90, 0x51, 0x8f, 0xb4, 0xbc, 0x30, - 0x80, 0xf8, 0x3c, 0x38, 0xcf, 0x04, 0xe6, 0x91, 0x5d, 0x03, 0xb6, 0xcb, 0xf5, 0xd0, 0xbe, 0x51, - 0x8d, 0xd9, 0x35, 0x60, 0x03, 0x75, 0xec, 0x0a, 0x58, 0x8d, 0xd9, 0x75, 0xa4, 0xc8, 0x8a, 0xad, - 0x3b, 0x70, 0xda, 0xa1, 0xc8, 0x8d, 0x4c, 0x6d, 0x82, 0xa2, 0x2d, 0x1f, 0x02, 0xef, 0xe5, 0x6a, - 0xaa, 0xe3, 0x22, 0x39, 0x91, 0x02, 0xb3, 0x06, 0xb6, 0xaf, 0xba, 0xed, 0x61, 0x0d, 0x67, 0x57, - 0x99, 0xab, 0x4e, 0xee, 0x5e, 0x05, 0x19, 0x7d, 0xc3, 0x55, 0x51, 0xd0, 0xea, 0x73, 0x33, 0x2e, - 0x0e, 0x0d, 0x6b, 0x78, 0x6b, 0x26, 0xf8, 0x50, 0xa5, 0xc9, 0xb9, 0xdd, 0xa3, 0xc1, 0x4a, 0x3a, - 0xfa, 0x59, 0x80, 0x75, 0x91, 0xde, 0x27, 0x8d, 0xbe, 0x16, 0xb9, 0xe1, 0xda, 0x58, 0x8f, 0x94, - 0x89, 0x0b, 0x6c, 0xcc, 0x89, 0x52, 0x82, 0x6e, 0x92, 0x66, 0x5f, 0xe5, 0xc6, 0x89, 0x57, 0x5e, - 0x23, 0x3e, 0x13, 0xb1, 0xd6, 0x2d, 0xa1, 0x32, 0xeb, 0xb2, 0xcb, 0xad, 0xa3, 0x44, 0xf4, 0x31, - 0xa9, 0x01, 0x7e, 0xe9, 0x6d, 0x8a, 0xb2, 0x51, 0x59, 0x2e, 0x1b, 0xd5, 0x59, 0xd9, 0x08, 0xc9, - 0xea, 0x81, 0x9c, 0x08, 0x35, 0x35, 0x36, 0x20, 0xab, 0xac, 0x20, 0xa3, 0xdf, 0xac, 0x60, 0x9d, - 0xa6, 0xdf, 0x21, 0xad, 0xc3, 0xde, 0xce, 0x1e, 0xcf, 0x32, 0x99, 0x8e, 0x72, 0xbc, 0xf4, 0x2d, - 0xaf, 0x8e, 0x94, 0x4c, 0x3c, 0xa0, 0x2f, 0x0e, 0xda, 0x4f, 0x3c, 0xed, 0xca, 0x7f, 0xd6, 0xf6, - 0xc4, 0x69, 0x87, 0xd4, 0x07, 0xe7, 0x79, 0x6c, 0x12, 0x7c, 0x0d, 0xbf, 0x7c, 0x6d, 0x3a, 0x8e, - 0x6b, 0x31, 0x28, 0x46, 0x1f, 0x90, 0x26, 0x13, 0x2e, 0x34, 0x72, 0x7b, 0xa5, 0xf9, 0xcd, 0x4a, - 0x1e, 0x9b, 0x89, 0x41, 0xf0, 0x6d, 0x8f, 0xb4, 0x9a, 0x66, 0xb9, 0x7d, 0xc5, 0x15, 0x17, 0x7c, - 0x1e, 0x44, 0x3f, 0x20, 0xe4, 0x19, 0x9f, 0x88, 0x3c, 0xe3, 0x60, 0xb6, 0xbe, 0x74, 0x87, 0x92, - 0x89, 0x77, 0xf0, 0xa4, 0xa1, 0x94, 0xee, 0x88, 0x53, 0x19, 0x8b, 0xa2, 0x55, 0xde, 0xf4, 0x14, - 0x1d, 0xa7, 0x28, 0xa5, 0x28, 0x47, 0xef, 0x93, 0xd5, 0x81, 0x88, 0x63, 0x35, 0xc9, 0xb0, 0x49, - 0x52, 0x4f, 0x05, 0x39, 0xac, 0x10, 0xa1, 0xf7, 0xc9, 0x4d, 0x88, 0xe9, 0xe3, 0xbc, 0xaf, 0x55, - 0xc6, 0x47, 0x2e, 0x83, 0x9a, 0xf6, 0x12, 0xcb, 0x0c, 0xb8, 0xec, 0x1e, 0xcf, 0x4f, 0xc4, 0x10, - 0x2e, 0x06, 0x6d, 0xd3, 0xd6, 0x05, 0x0f, 0xa2, 0xf7, 0xc8, 0x5a, 0x11, 0xf7, 0x4e, 0xa6, 0x65, - 0x65, 0xe6, 0x41, 0xda, 0x26, 0xc4, 0xa6, 0xae, 0x5f, 0x76, 0x3d, 0x84, 0x76, 0x48, 0xa3, 0x97, - 0x1a, 0x91, 0xb0, 0xa1, 0x09, 0xd7, 0xec, 0x25, 0xde, 0xf2, 0x9d, 0x8e, 0x2c, 0x56, 0x0a, 0xdd, - 0xf9, 0x16, 0x69, 0x79, 0x0e, 0x7d, 0xa3, 0xee, 0xfc, 0x6e, 0x39, 0x06, 0x80, 0xd0, 0x70, 0x3a, - 0x99, 0x14, 0x8a, 0x8e, 0x00, 0x81, 0x62, 0x64, 0xb8, 0x5c, 0xe0, 0x13, 0x72, 0x6d, 0x3e, 0x18, - 0x6d, 0xb7, 0x50, 0xb9, 0x29, 0x4b, 0x3f, 0x52, 0x36, 0x58, 0x8a, 0x79, 0xb1, 0xec, 0x02, 0x3e, - 0x64, 0x0b, 0x1d, 0x34, 0xff, 0xaa, 0x65, 0xd9, 0x75, 0xf4, 0x3e, 0xda, 0x2f, 0xe3, 0xe2, 0x75, - 0x65, 0xd3, 0x46, 0x60, 0x65, 0x96, 0xc7, 0xd1, 0x2f, 0x02, 0xd2, 0xf2, 0x42, 0xe5, 0x75, 0xb9, - 0x6e, 0x6d, 0x55, 0x3c, 0x5b, 0xb7, 0xc8, 0xca, 0x1e, 0xff, 0x54, 0xb9, 0xe9, 0xa2, 0xca, 0x1c, - 0x61, 0x51, 0x99, 0x2a, 0x8d, 0xd9, 0xee, 0x08, 0xa8, 0x7c, 0x8f, 0x65, 0x22, 0xf6, 0xd4, 0x50, - 0xd8, 0xe8, 0x5f, 0x63, 0x25, 0x5d, 0xf4, 0xbf, 0xfa, 0x52, 0xff, 0x5b, 0x2d, 0xfb, 0x5f, 0xf4, - 0xb7, 0x0a, 0x5e, 0x6f, 0x96, 0x53, 0xdf, 0x9c, 0x45, 0x7d, 0xb0, 0x94, 0xb9, 0x8e, 0xe3, 0x12, - 0x6c, 0x31, 0xf6, 0x61, 0x56, 0x15, 0x13, 0xa5, 0xcf, 0x71, 0x78, 0xf2, 0xb3, 0xc5, 0x31, 0x18, - 0x0a, 0xd0, 0x75, 0x52, 0xdd, 0xee, 0x1f, 0xe2, 0xf8, 0x74, 0xcd, 0x1f, 0x6c, 0xfa, 0x87, 0x0c, - 0x58, 0xf4, 0x73, 0xa4, 0xd6, 0x87, 0x76, 0xec, 0x0a, 0xc1, 0x75, 0x4f, 0x04, 0x60, 0x66, 0x99, - 0x90, 0x6d, 0x5b, 0x89, 0x8a, 0x4f, 0x7a, 0xfb, 0xf6, 0xf2, 0xf3, 0xd9, 0x86, 0x1c, 0x56, 0x88, - 0xd0, 0xc7, 0xe4, 0xda, 0xee, 0x74, 0x24, 0x32, 0x3e, 0x12, 0x4f, 0xdd, 0x80, 0xe4, 0xca, 0x41, - 0xe8, 0x29, 0xcd, 0x09, 0xe0, 0x05, 0x17, 0xb4, 0x60, 0xd7, 0x67, 0xc2, 0x9c, 0x29, 0x7d, 0x82, - 0x93, 0x99, 0xbf, 0x2b, 0x72, 0x58, 0x21, 0x12, 0xfd, 0xa5, 0x88, 0x02, 0xbc, 0xfa, 0x2d, 0x28, - 0xce, 0x13, 0xe9, 0x46, 0x99, 0x2a, 0x73, 0x04, 0xc4, 0x26, 0x13, 0xb9, 0xd0, 0xa7, 0xae, 0x06, - 0x54, 0xdc, 0xb8, 0xe4, 0x41, 0x36, 0x36, 0xcf, 0x78, 0x86, 0x41, 0x61, 0xd7, 0x10, 0xe9, 0x1f, - 0x0a, 0x9d, 0x8a, 0x04, 0x83, 0x02, 0x29, 0x98, 0x0f, 0xdc, 0xea, 0x60, 0xbb, 0x6f, 0x5f, 0xa6, - 0xca, 0x66, 0x00, 0xe4, 0x3f, 0x68, 0x67, 0x32, 0x85, 0x6f, 0x97, 0xba, 0x6d, 0xea, 0x1e, 0x42, - 0xbf, 0x44, 0x6e, 0xec, 0xc8, 0x1c, 0x06, 0x8d, 0xfd, 0xfd, 0xbd, 0x0f, 0x65, 0x92, 0x08, 0x6d, - 0x2f, 0xda, 0x60, 0x4b, 0x78, 0xf4, 0xc7, 0x80, 0x34, 0x0a, 0xc7, 0xc1, 0x71, 0x06, 0x63, 0xae, - 0x6d, 0xe0, 0x80, 0x51, 0xa4, 0xe0, 0xca, 0xdf, 0x9f, 0x2a, 0xc3, 0xf1, 0x5a, 0x8e, 0x00, 0xe9, - 0xbe, 0xd0, 0x52, 0x0d, 0x71, 0xae, 0x40, 0x0a, 0x66, 0x4c, 0x26, 0x78, 0x62, 0xe4, 0x44, 0xb0, - 0x69, 0x0a, 0x3f, 0x78, 0xbb, 0x45, 0x18, 0x86, 0xb7, 0x02, 0x42, 0x4b, 0x2b, 0xd6, 0xd2, 0x02, - 0x0a, 0x4f, 0xb7, 0x9d, 0x4d, 0x73, 0x1c, 0xb1, 0xed, 0x1a, 0xb0, 0x3d, 0x31, 0x71, 0xb3, 0x75, - 0x93, 0xd9, 0x75, 0x74, 0x86, 0x73, 0xdc, 0x73, 0x3b, 0x5d, 0x62, 0xd6, 0x96, 0xd9, 0x18, 0x5c, - 0x9a, 0x8d, 0x15, 0x3f, 0x1b, 0x6f, 0x93, 0xba, 0xd3, 0xc5, 0x0a, 0x82, 0x14, 0xbc, 0xf8, 0x53, - 0xc1, 0x8f, 0x91, 0x57, 0xb3, 0x3c, 0x0f, 0x89, 0x0e, 0xc9, 0x5b, 0x76, 0xe3, 0x83, 0xb1, 0x56, - 0xc6, 0x24, 0xe2, 0xbf, 0xd8, 0x9a, 0x92, 0x1a, 0xe3, 0x46, 0x14, 0x33, 0x1a, 0xac, 0xa3, 0x7f, - 0x54, 0xc9, 0x55, 0x3f, 0x15, 0xbc, 0xf3, 0x05, 0xff, 0xe6, 0x7c, 0x95, 0xc5, 0xf3, 0xd1, 0x2e, - 0xb9, 0xea, 0xbf, 0xc9, 0x25, 0x1d, 0xdd, 0x67, 0x63, 0xda, 0xcc, 0xa9, 0xd0, 0x43, 0xf2, 0x76, - 0x71, 0x3b, 0xe8, 0x46, 0x5b, 0x59, 0x8e, 0xb6, 0x6a, 0xd6, 0xd6, 0xff, 0x79, 0xb6, 0xe6, 0x5f, - 0x01, 0xad, 0x5d, 0xae, 0x4d, 0x9f, 0x93, 0xdb, 0x05, 0xe3, 0xb9, 0x96, 0x46, 0xcc, 0xec, 0xae, - 0x7c, 0x36, 0xbb, 0xaf, 0x51, 0xf7, 0x0d, 0xc3, 0x8e, 0xbd, 0xfd, 0xfe, 0x00, 0x0d, 0xd7, 0xdf, - 0xd0, 0xf0, 0xbc, 0x3a, 0xfd, 0x21, 0x79, 0x67, 0x6e, 0x4b, 0xcf, 0xf2, 0xea, 0x67, 0xb3, 0xfc, - 0x3a, 0xfd, 0xe8, 0x3d, 0xd2, 0x2c, 0x2b, 0xe4, 0xe5, 0x75, 0x26, 0xfa, 0x49, 0xf1, 0xad, 0xe2, - 0x17, 0x72, 0x90, 0xed, 0x26, 0x89, 0x3a, 0xc3, 0x8f, 0x62, 0x47, 0xfc, 0xcf, 0xbd, 0xe9, 0x36, - 0xa9, 0x77, 0x63, 0xfb, 0xff, 0x88, 0x9b, 0xcb, 0x90, 0x8a, 0x12, 0x8c, 0x4a, 0xac, 0x90, 0x30, - 0xc9, 0x6e, 0x27, 0x3c, 0xcf, 0xcb, 0x86, 0x5d, 0x90, 0x74, 0x8b, 0x90, 0xbe, 0x96, 0x4a, 0xbb, - 0xcf, 0x60, 0x37, 0x80, 0xde, 0x5d, 0x98, 0x45, 0xf4, 0x31, 0x8f, 0x05, 0x4a, 0x9d, 0x17, 0x43, - 0xdc, 0x4c, 0x2b, 0x7a, 0x4c, 0xe8, 0x72, 0x65, 0x87, 0xbe, 0xd9, 0xe7, 0x23, 0x91, 0x43, 0xb7, - 0x77, 0xfd, 0xb8, 0xa4, 0x67, 0x2f, 0xe7, 0xbe, 0x81, 0xf0, 0xe5, 0x76, 0xc9, 0xed, 0xcb, 0xf7, - 0x84, 0x77, 0x82, 0xe1, 0xa0, 0xe8, 0xeb, 0xb0, 0xb6, 0xf6, 0x91, 0x8f, 0xf9, 0x54, 0xd2, 0xd1, - 0x2f, 0x03, 0x7c, 0x80, 0x62, 0x0c, 0xbc, 0x47, 0xd6, 0x76, 0xc4, 0x31, 0x9f, 0x26, 0xa6, 0x1b, - 0x7b, 0x1f, 0x51, 0xf3, 0x20, 0x48, 0x75, 0x75, 0x3c, 0x96, 0x46, 0xc4, 0x66, 0xaa, 0x45, 0xf1, - 0x7d, 0x30, 0x0f, 0xc2, 0xe1, 0x1f, 0x27, 0x7c, 0x94, 0xe3, 0xa7, 0x82, 0x23, 0xe8, 0xd7, 0x48, - 0x03, 0x26, 0x34, 0x9e, 0x24, 0x39, 0x26, 0xdc, 0xdc, 0x5c, 0xea, 0x58, 0xc5, 0x47, 0x4a, 0x21, - 0x19, 0x49, 0x72, 0xdd, 0x3f, 0x67, 0x57, 0x8f, 0xc0, 0x7c, 0x2f, 0x1d, 0x8a, 0x17, 0x58, 0xe1, - 0x1d, 0x01, 0xe8, 0x47, 0xe5, 0x7c, 0x57, 0x63, 0x8e, 0x80, 0x37, 0xb0, 0x8b, 0x83, 0x33, 0x85, - 0x65, 0xa9, 0xa4, 0xe9, 0x35, 0x52, 0xd9, 0xcf, 0xf0, 0x4b, 0xba, 0xb2, 0x9f, 0x45, 0x3f, 0x2f, - 0xdf, 0xc4, 0x6d, 0x0e, 0x26, 0xed, 0xc4, 0x85, 0xdf, 0xce, 0x8e, 0x70, 0x21, 0x55, 0x76, 0x48, - 0x1b, 0x52, 0xf6, 0x6d, 0xee, 0x92, 0x86, 0xd0, 0x3a, 0x55, 0x5a, 0x60, 0xe9, 0xdd, 0xbd, 0xc2, - 0x4a, 0x84, 0x76, 0xbc, 0xff, 0x61, 0x5a, 0x0f, 0xde, 0x5e, 0x9e, 0xc8, 0xbb, 0xba, 0xf8, 0x84, - 0xb1, 0x82, 0x5b, 0x84, 0x34, 0x1e, 0x81, 0x32, 0x13, 0x26, 0xfa, 0x3a, 0x59, 0x9b, 0x9b, 0x7b, - 0xc1, 0x0f, 0x4f, 0x1f, 0x6e, 0xf3, 0x78, 0x2c, 0x06, 0xf1, 0x58, 0x4c, 0x78, 0xe1, 0xad, 0x39, - 0x70, 0xeb, 0xa7, 0xc1, 0xcb, 0x57, 0xed, 0x2b, 0x7f, 0x7e, 0xd5, 0xbe, 0xf2, 0xcf, 0x57, 0xed, - 0xe0, 0xc7, 0x17, 0xed, 0xe0, 0xd7, 0x17, 0xed, 0xe0, 0xb7, 0x17, 0xed, 0xe0, 0xf7, 0x17, 0xed, - 0xe0, 0xe5, 0x45, 0x3b, 0xf8, 0xd3, 0x45, 0x3b, 0xf8, 0xfb, 0x45, 0x3b, 0xf8, 0xf8, 0x93, 0x37, - 0xfc, 0x97, 0x53, 0xbb, 0xf6, 0xd7, 0x39, 0x95, 0xda, 0x78, 0xac, 0xec, 0x64, 0xd4, 0xe1, 0x23, - 0x91, 0x1a, 0xef, 0x1f, 0x50, 0xb8, 0xe9, 0x51, 0xdd, 0xd2, 0x0f, 0xff, 0x15, 0x00, 0x00, 0xff, - 0xff, 0x10, 0x2c, 0x26, 0x0c, 0x4e, 0x15, 0x00, 0x00, + // 2163 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0xcd, 0x73, 0x1b, 0x49, + 0x15, 0xcf, 0x48, 0xb2, 0x2c, 0xb5, 0x22, 0x67, 0xd3, 0x9b, 0xcd, 0x0e, 0x21, 0xa5, 0xf5, 0x0e, + 0x29, 0x30, 0x10, 0xac, 0x22, 0x81, 0x25, 0x84, 0x8f, 0x2a, 0x59, 0x4e, 0x62, 0xd5, 0xc6, 0xb1, + 0x68, 0xd9, 0x6b, 0xd8, 0xc3, 0x56, 0xb5, 0x47, 0x6d, 0xa9, 0xd7, 0xa3, 0xe9, 0xa9, 0x9e, 0x96, + 0x1d, 0x73, 0x82, 0x1b, 0xff, 0x01, 0x67, 0x2e, 0x7c, 0xfc, 0x07, 0x14, 0x27, 0x38, 0x91, 0xe2, + 0xc4, 0x91, 0x2a, 0xaa, 0x80, 0xf8, 0xce, 0x9d, 0x23, 0xf5, 0xfa, 0x63, 0xd4, 0x92, 0x6c, 0xd8, + 0xc0, 0xad, 0xdf, 0x7b, 0xbf, 0xf7, 0xfa, 0xe3, 0x7d, 0xce, 0xa0, 0x17, 0x23, 0xae, 0xc6, 0xd3, + 0xa3, 0xcd, 0x58, 0x4c, 0xda, 0x27, 0x54, 0xd1, 0xaf, 0xc5, 0x22, 0x55, 0x94, 0xa7, 0x4c, 0xe6, + 0x4b, 0x74, 0x2e, 0xe3, 0x76, 0xc2, 0x8f, 0xf2, 0x76, 0x26, 0x85, 0x12, 0xb1, 0x48, 0xec, 0x2a, + 0x6f, 0x8b, 0x98, 0x6f, 0xea, 0x25, 0xae, 0x8c, 0x64, 0x16, 0xdf, 0x89, 0x46, 0x62, 0x24, 0x8c, + 0xf0, 0x68, 0x7a, 0xdc, 0x06, 0x4a, 0x13, 0x7a, 0x65, 0x90, 0xd1, 0x1f, 0xcb, 0xa8, 0x32, 0xc8, + 0x58, 0x8c, 0x43, 0xb4, 0xfa, 0x11, 0x93, 0x39, 0x17, 0x69, 0x18, 0xac, 0x07, 0x1b, 0x75, 0xe2, + 0x48, 0xfc, 0x25, 0xb4, 0xda, 0x97, 0x22, 0x66, 0x79, 0x1e, 0x96, 0xd6, 0x83, 0x8d, 0xc6, 0x83, + 0xe6, 0x26, 0x98, 0xdf, 0xb4, 0x4c, 0xe2, 0xa4, 0xb8, 0x85, 0x2a, 0x44, 0x08, 0x15, 0x96, 0x35, + 0x0a, 0x19, 0x14, 0x70, 0x88, 0xe6, 0xe3, 0x3b, 0xa8, 0xb6, 0x23, 0x72, 0x95, 0xd2, 0x09, 0x0b, + 0x2b, 0x7a, 0x8f, 0x82, 0xc6, 0x5f, 0x46, 0xd5, 0x5d, 0x31, 0x4d, 0x55, 0x1e, 0xae, 0xac, 0x97, + 0x37, 0x1a, 0x0f, 0x1a, 0x46, 0x5b, 0xf3, 0xb6, 0x2a, 0xaf, 0xfe, 0xf6, 0xde, 0x35, 0x62, 0x01, + 0xf8, 0x7d, 0xb4, 0xb2, 0x23, 0xc4, 0x49, 0x1e, 0x56, 0xf5, 0x3e, 0x16, 0xa9, 0x59, 0xc4, 0x48, + 0xf0, 0xf7, 0x50, 0xa3, 0x93, 0xa6, 0x42, 0x51, 0xc5, 0x45, 0x9a, 0x87, 0xab, 0xda, 0xe4, 0xe7, + 0x0d, 0x10, 0x6e, 0xbb, 0xe9, 0x49, 0x9f, 0xa4, 0x4a, 0x9e, 0x13, 0x1f, 0x0f, 0x3b, 0x3c, 0xe7, + 0xe9, 0xf4, 0x65, 0x58, 0xf3, 0x77, 0xd0, 0x2c, 0x62, 0x24, 0xf0, 0x28, 0x03, 0x91, 0x50, 0xc9, + 0xf3, 0xb0, 0xee, 0x3f, 0x8a, 0x65, 0x12, 0x27, 0x05, 0xe0, 0x21, 0x4f, 0x87, 0xe2, 0x2c, 0x0f, + 0x91, 0x0f, 0xb4, 0x4c, 0xe2, 0xa4, 0x77, 0xbe, 0x8f, 0xde, 0x5a, 0x3c, 0x15, 0x7e, 0x0b, 0x95, + 0x4f, 0xd8, 0xb9, 0x75, 0x08, 0x2c, 0xf1, 0x2d, 0xb4, 0x72, 0x4a, 0x93, 0x29, 0xd3, 0xae, 0xa8, + 0x13, 0x43, 0x3c, 0x2e, 0x3d, 0x0a, 0xa2, 0xdf, 0x95, 0x0b, 0x3f, 0xc1, 0x4b, 0xef, 0x33, 0x39, + 0xe1, 0x29, 0x4d, 0xb4, 0x72, 0x8d, 0x14, 0x34, 0xfe, 0x2a, 0x6a, 0x74, 0x45, 0x9a, 0x8b, 0x84, + 0x0d, 0xf8, 0x8f, 0x99, 0x75, 0x69, 0xdd, 0x1c, 0x6a, 0x4b, 0xbc, 0x24, 0xbe, 0x14, 0xdf, 0x43, + 0x95, 0x83, 0x9c, 0xc9, 0x79, 0x97, 0x02, 0xc7, 0xfa, 0x44, 0x4b, 0x31, 0x46, 0x95, 0x8e, 0x1c, + 0xe5, 0x61, 0x65, 0xbd, 0xbc, 0x51, 0x27, 0x7a, 0x0d, 0x47, 0x7f, 0x92, 0x9e, 0x6a, 0x6f, 0xd6, + 0x09, 0x2c, 0x81, 0xd3, 0x3d, 0x1b, 0x6a, 0xaf, 0xd5, 0x09, 0x2c, 0xf1, 0x77, 0xd0, 0xf5, 0x2e, + 0xcd, 0xe8, 0x11, 0x4f, 0xb8, 0xe2, 0x0c, 0xfc, 0x04, 0xbb, 0xbc, 0xeb, 0x3d, 0xb7, 0x2f, 0x26, + 0x73, 0x60, 0xfc, 0x75, 0xb4, 0x4a, 0x12, 0x3e, 0xe1, 0x2a, 0x0f, 0x6b, 0xda, 0xbf, 0x37, 0x6d, + 0x58, 0xee, 0x0d, 0x7a, 0x3f, 0x34, 0x12, 0x7b, 0x48, 0x87, 0xc3, 0x1b, 0xe8, 0xc6, 0x0b, 0xf1, + 0x82, 0x9d, 0xf5, 0x25, 0x3f, 0xe5, 0x09, 0x1b, 0x31, 0xe3, 0xbc, 0x1a, 0x59, 0x64, 0x03, 0xb2, + 0x93, 0x65, 0x54, 0x4e, 0x84, 0xec, 0x4b, 0x71, 0xcc, 0x13, 0xa6, 0xbd, 0x57, 0x27, 0x8b, 0x6c, + 0xbc, 0x8e, 0x1a, 0x7b, 0x7b, 0xbb, 0x83, 0x58, 0x48, 0xd6, 0x19, 0x7e, 0x1a, 0x36, 0xd6, 0x83, + 0x8d, 0x32, 0xf1, 0x59, 0x38, 0x42, 0xd7, 0x07, 0x2c, 0x81, 0xdb, 0x3c, 0xa7, 0x47, 0x2c, 0x09, + 0xaf, 0x6b, 0x43, 0x73, 0xbc, 0xe8, 0x21, 0x2a, 0x6f, 0x89, 0x97, 0xf8, 0x36, 0xaa, 0xee, 0x30, + 0x3e, 0x1a, 0x2b, 0xed, 0xb5, 0x26, 0xb1, 0x14, 0x78, 0xfd, 0x90, 0x0f, 0xd5, 0x58, 0x7b, 0xab, + 0x49, 0x0c, 0x11, 0xa5, 0xc6, 0x39, 0xf0, 0xb0, 0x07, 0xbd, 0x6d, 0xab, 0x02, 0x4b, 0xe0, 0x3c, + 0xeb, 0x6d, 0x5b, 0x34, 0x2c, 0xf1, 0x17, 0xd1, 0x5a, 0x67, 0x38, 0xe4, 0x10, 0x5b, 0x34, 0x79, + 0xc6, 0x87, 0x79, 0x58, 0x5e, 0x2f, 0x6f, 0x34, 0xc9, 0x02, 0x17, 0x22, 0x07, 0x6c, 0xfa, 0x39, + 0xea, 0xe8, 0xe8, 0x57, 0x01, 0xba, 0xb9, 0xe4, 0x15, 0xd0, 0xd8, 0x12, 0xd3, 0x74, 0xc8, 0xd3, + 0x51, 0x18, 0x68, 0x6f, 0x17, 0x34, 0xbe, 0x8b, 0xea, 0x4f, 0x8e, 0x8f, 0x59, 0xac, 0xf8, 0x29, + 0x44, 0x1a, 0x08, 0x67, 0x0c, 0x78, 0xba, 0x5e, 0x3a, 0x66, 0x92, 0x2b, 0x7a, 0x94, 0x30, 0x7d, + 0xa0, 0x3a, 0xf1, 0x59, 0xa0, 0xdf, 0x87, 0xb8, 0x55, 0x8a, 0x0d, 0x6d, 0x74, 0xcd, 0x18, 0x50, + 0xb2, 0x3a, 0x93, 0x23, 0xce, 0x52, 0x65, 0xc3, 0xcc, 0x91, 0x51, 0x0f, 0x35, 0xbc, 0x30, 0x80, + 0xf8, 0xdc, 0x3f, 0xcf, 0x98, 0xcd, 0x23, 0xbd, 0x06, 0xde, 0x0e, 0x95, 0x43, 0xfd, 0x46, 0x15, + 0xa2, 0xd7, 0xc0, 0x1b, 0x88, 0x63, 0x53, 0xc0, 0x2a, 0x44, 0xaf, 0x23, 0x81, 0x56, 0x74, 0xdd, + 0x81, 0xd3, 0x0e, 0x59, 0xae, 0x78, 0xaa, 0x13, 0xd4, 0xda, 0xf2, 0x59, 0xe0, 0xbd, 0x5c, 0x4c, + 0x65, 0xec, 0x92, 0xd3, 0x52, 0x60, 0x56, 0xc1, 0xf6, 0x65, 0xb3, 0x3d, 0xac, 0xe1, 0xec, 0x22, + 0x33, 0xd5, 0xc9, 0xdc, 0xcb, 0x91, 0xd1, 0x07, 0xa6, 0x8a, 0x82, 0x56, 0x9f, 0xaa, 0xb1, 0x3b, + 0x34, 0xac, 0xe1, 0xad, 0x09, 0xa3, 0x43, 0x91, 0x26, 0xe7, 0x7a, 0x8f, 0x1a, 0x29, 0xe8, 0xe8, + 0x0f, 0x25, 0x5b, 0x17, 0xf1, 0x7d, 0x54, 0xeb, 0x4b, 0x96, 0x2b, 0x2a, 0x95, 0xf6, 0x48, 0x91, + 0xb8, 0x20, 0xb6, 0x39, 0x51, 0x20, 0xf0, 0x26, 0xaa, 0xf7, 0x45, 0xae, 0x0c, 0xbc, 0x74, 0x05, + 0x7c, 0x06, 0xd1, 0xd6, 0x35, 0x21, 0x32, 0xed, 0xb2, 0xcb, 0xad, 0x5b, 0x04, 0xfe, 0x00, 0x35, + 0xbb, 0x92, 0x51, 0xc5, 0xc8, 0x34, 0x55, 0x5c, 0x07, 0xd5, 0xe5, 0x2a, 0xf3, 0x30, 0xfc, 0x18, + 0xdd, 0x30, 0x8c, 0xae, 0xeb, 0x7c, 0xb6, 0x31, 0x2c, 0x6b, 0x2e, 0x02, 0xf1, 0x23, 0xb4, 0x36, + 0x80, 0xa3, 0xce, 0x54, 0xab, 0x57, 0xa8, 0x2e, 0xe0, 0xa2, 0x8f, 0x51, 0x05, 0xa4, 0x97, 0xbe, + 0xbd, 0x2b, 0x72, 0xa5, 0xe5, 0x22, 0x57, 0x9e, 0x15, 0xb9, 0x10, 0xad, 0xee, 0xf3, 0x09, 0x13, + 0x53, 0xa5, 0xd3, 0xa7, 0x4c, 0x1c, 0x19, 0xfd, 0x66, 0xc5, 0x76, 0x15, 0xfc, 0x5d, 0xd4, 0x38, + 0xe8, 0x6d, 0xef, 0xd2, 0x2c, 0xe3, 0xe9, 0x28, 0xb7, 0x2e, 0xba, 0xe5, 0x55, 0xbd, 0x42, 0x68, + 0x8f, 0xe9, 0xc3, 0x41, 0xfb, 0x99, 0xa7, 0x5d, 0xfa, 0xef, 0xda, 0x1e, 0x1c, 0xb7, 0x51, 0x75, + 0x70, 0x9e, 0xc7, 0x2a, 0xb1, 0xbe, 0xf3, 0x8b, 0xed, 0xa6, 0x91, 0x98, 0x86, 0x68, 0x61, 0xf8, + 0x01, 0xaa, 0x13, 0x66, 0x02, 0x39, 0xd7, 0x57, 0x9a, 0xdf, 0xac, 0x90, 0x91, 0x19, 0x0c, 0x52, + 0xa5, 0x3b, 0x92, 0x62, 0x9a, 0xe5, 0xfa, 0x15, 0x57, 0x4c, 0xaa, 0x78, 0x2c, 0xfc, 0x18, 0xa1, + 0x17, 0x74, 0xc2, 0xf2, 0x8c, 0x82, 0xd9, 0xea, 0xd2, 0x1d, 0x0a, 0xa1, 0xbd, 0x83, 0x87, 0x86, + 0xc2, 0xbf, 0xcd, 0x4e, 0x79, 0xcc, 0x5c, 0x63, 0xbf, 0xe9, 0x29, 0x1a, 0x89, 0x2b, 0xfc, 0x16, + 0x87, 0xef, 0xa3, 0xd5, 0x01, 0x8b, 0x63, 0x31, 0xc9, 0x6c, 0x4b, 0xc7, 0x9e, 0x8a, 0x95, 0x10, + 0x07, 0xc1, 0xf7, 0xd1, 0x4d, 0xc8, 0xc0, 0xe3, 0xbc, 0x2f, 0x45, 0x46, 0x47, 0x26, 0xdf, 0xeb, + 0xfa, 0x12, 0xcb, 0x02, 0xb8, 0xec, 0x2e, 0xcd, 0x4f, 0xd8, 0x10, 0x2e, 0x06, 0x4d, 0x5e, 0x57, + 0x31, 0x8f, 0x85, 0xef, 0xa1, 0xa6, 0xcb, 0x52, 0x83, 0x69, 0x68, 0xcc, 0x3c, 0x13, 0xb7, 0x10, + 0xd2, 0x85, 0xc6, 0x6f, 0x12, 0x1e, 0x07, 0xb7, 0x51, 0xad, 0x97, 0x2a, 0x96, 0x90, 0xa1, 0x0a, + 0x9b, 0xfa, 0x12, 0x6f, 0xfb, 0x4e, 0xb7, 0x22, 0x52, 0x80, 0xee, 0x7c, 0x1b, 0x35, 0x3c, 0x87, + 0xbe, 0xd1, 0x2c, 0xf1, 0x5e, 0x31, 0xb4, 0x00, 0x68, 0x38, 0x9d, 0x4c, 0x9c, 0xa2, 0x21, 0x00, + 0xe0, 0x06, 0x9c, 0xcb, 0x01, 0x9f, 0xa0, 0xb5, 0xf9, 0x60, 0xd4, 0xbd, 0x4d, 0xe4, 0xaa, 0x68, + 0x54, 0x96, 0xd2, 0xc1, 0xe2, 0x12, 0xb0, 0xe8, 0x59, 0x3e, 0x4b, 0x97, 0x65, 0x18, 0x55, 0xca, + 0x5a, 0xa4, 0xd7, 0xd1, 0x23, 0x6b, 0xbf, 0x88, 0x8b, 0xab, 0x8a, 0xbc, 0x8e, 0xc0, 0xd2, 0x2c, + 0x8f, 0xa3, 0x5f, 0x04, 0xa8, 0xe1, 0x85, 0xca, 0x55, 0xb9, 0xae, 0x6d, 0x95, 0x3c, 0x5b, 0xb7, + 0xd0, 0xca, 0x2e, 0xfd, 0x54, 0x98, 0x59, 0xa8, 0x4c, 0x0c, 0xa1, 0xb9, 0x3c, 0x15, 0xd2, 0x66, + 0xbb, 0x21, 0xa0, 0x4e, 0x3f, 0xe5, 0x09, 0xdb, 0x15, 0x43, 0xa6, 0xa3, 0xbf, 0x49, 0x0a, 0xda, + 0x75, 0xeb, 0xea, 0x52, 0xb7, 0x5e, 0x2d, 0xba, 0x75, 0xf4, 0xf7, 0x92, 0xbd, 0xde, 0x2c, 0xa7, + 0xbe, 0x35, 0x8b, 0xfa, 0x60, 0x29, 0x73, 0x8d, 0xc4, 0x24, 0xd8, 0x62, 0xec, 0xc3, 0x64, 0xcd, + 0x26, 0x42, 0x9e, 0xdb, 0x51, 0xcf, 0xcf, 0x16, 0x23, 0x20, 0x16, 0x80, 0xd7, 0x51, 0xb9, 0xdb, + 0x3f, 0xb0, 0xc3, 0xde, 0x9a, 0x3f, 0x86, 0xf5, 0x0f, 0x08, 0x88, 0xf0, 0x17, 0x50, 0xa5, 0x0f, + 0xc3, 0x83, 0x29, 0x04, 0x37, 0x3c, 0x08, 0xb0, 0x89, 0x16, 0x42, 0xb6, 0x6d, 0x25, 0x22, 0x3e, + 0xe9, 0xed, 0xe9, 0xcb, 0xcf, 0x67, 0x9b, 0x95, 0x10, 0x07, 0xc1, 0x4f, 0xd1, 0xda, 0xce, 0x74, + 0xc4, 0x32, 0x3a, 0x62, 0xcf, 0xcd, 0x38, 0x67, 0xca, 0x41, 0xe8, 0x29, 0xcd, 0x01, 0x5c, 0xed, + 0x9e, 0xd7, 0x82, 0x5d, 0x5f, 0x30, 0x75, 0x26, 0xe4, 0x89, 0x9d, 0x23, 0xfd, 0x5d, 0xad, 0x84, + 0x38, 0x48, 0xf4, 0x57, 0x17, 0x05, 0xf6, 0xea, 0xb7, 0xa0, 0x38, 0x4f, 0xb8, 0x19, 0xbc, 0xca, + 0xc4, 0x10, 0x10, 0x9b, 0x84, 0xe5, 0x4c, 0x9e, 0x9a, 0x1a, 0x50, 0x32, 0xc3, 0x9d, 0xc7, 0xd2, + 0xb1, 0x79, 0x46, 0x33, 0x1b, 0x14, 0x7a, 0x0d, 0x91, 0xfe, 0x21, 0x93, 0x29, 0x4b, 0x6c, 0x50, + 0x58, 0x0a, 0xa6, 0x19, 0xb3, 0xda, 0xef, 0xf6, 0xf5, 0xcb, 0x94, 0xc9, 0x8c, 0x01, 0xf9, 0x0f, + 0xda, 0x19, 0x4f, 0xe1, 0x4b, 0xab, 0xaa, 0x47, 0x10, 0x8f, 0x83, 0xbf, 0x82, 0xde, 0xda, 0xe6, + 0x39, 0x8c, 0x45, 0x7b, 0x7b, 0xbb, 0x1f, 0xf2, 0x24, 0x61, 0x52, 0x5f, 0xb4, 0x46, 0x96, 0xf8, + 0xd1, 0x9f, 0x02, 0x54, 0x73, 0x8e, 0x83, 0xe3, 0x0c, 0xc6, 0x54, 0xea, 0xc0, 0x01, 0xa3, 0x96, + 0x82, 0x2b, 0xff, 0x60, 0x2a, 0x14, 0xb5, 0xd7, 0x32, 0x04, 0xa0, 0xfb, 0x4c, 0x72, 0x31, 0xb4, + 0x53, 0x90, 0xa5, 0x60, 0x22, 0x26, 0x8c, 0x26, 0xd0, 0x9c, 0x67, 0xad, 0x1c, 0xf4, 0x16, 0xd9, + 0x30, 0x6a, 0x3a, 0x96, 0xb5, 0xb4, 0xa2, 0x2d, 0x2d, 0x70, 0xe1, 0xe9, 0xba, 0xd9, 0x34, 0xb7, + 0x1f, 0x04, 0x7a, 0x0d, 0xbc, 0x5d, 0x36, 0x31, 0x5f, 0x02, 0x75, 0xa2, 0xd7, 0xd1, 0x99, 0x9d, + 0x3a, 0x0f, 0xf5, 0x2c, 0x6c, 0xb3, 0xb6, 0xc8, 0xc6, 0xe0, 0xd2, 0x6c, 0x2c, 0xf9, 0xd9, 0x78, + 0x1b, 0x55, 0x8d, 0xae, 0xad, 0x20, 0x96, 0x82, 0x17, 0x7f, 0xce, 0xe8, 0xb1, 0x95, 0x55, 0xb4, + 0xcc, 0xe3, 0x44, 0x07, 0xe8, 0x6d, 0xbd, 0xf1, 0xfe, 0x58, 0x0a, 0xa5, 0x12, 0xf6, 0x3f, 0x6c, + 0x8d, 0x51, 0x85, 0x50, 0xc5, 0xdc, 0x44, 0x09, 0xeb, 0xe8, 0x9f, 0x65, 0x74, 0xdd, 0x4f, 0x05, + 0xef, 0x7c, 0xc1, 0x7f, 0x38, 0x5f, 0x69, 0xf1, 0x7c, 0xb8, 0x83, 0xae, 0xfb, 0x6f, 0x72, 0x49, + 0x47, 0xf7, 0xc5, 0x36, 0x6d, 0xe6, 0x54, 0xf0, 0x01, 0x7a, 0xc7, 0xdd, 0x0e, 0xba, 0xd1, 0x56, + 0x96, 0x5b, 0x5b, 0x66, 0x4c, 0xfb, 0x9c, 0x67, 0x6b, 0xfe, 0x15, 0xac, 0xb5, 0xcb, 0xb5, 0xf1, + 0x21, 0xba, 0xed, 0x04, 0x87, 0x92, 0x2b, 0x36, 0xb3, 0xbb, 0xf2, 0xd9, 0xec, 0x5e, 0xa1, 0xee, + 0x1b, 0x86, 0x1d, 0x7b, 0x7b, 0xfd, 0x81, 0x35, 0x5c, 0x7d, 0x43, 0xc3, 0xf3, 0xea, 0xf8, 0x47, + 0xe8, 0xdd, 0xb9, 0x2d, 0x3d, 0xcb, 0xab, 0x9f, 0xcd, 0xf2, 0x55, 0xfa, 0xd1, 0xfb, 0xa8, 0x5e, + 0x54, 0xc8, 0xcb, 0xeb, 0x4c, 0xf4, 0x53, 0xf7, 0x65, 0xe5, 0x17, 0x72, 0xc0, 0x76, 0x92, 0x44, + 0x9c, 0xd9, 0x4f, 0x78, 0x43, 0xfc, 0xdf, 0xbd, 0xe9, 0x36, 0xaa, 0x76, 0x62, 0xfd, 0x37, 0xc7, + 0xcc, 0x65, 0x96, 0x8a, 0x12, 0x1b, 0x95, 0xb6, 0x42, 0xc2, 0x24, 0xdb, 0x4d, 0x68, 0x9e, 0x17, + 0x0d, 0xdb, 0x91, 0x78, 0x0b, 0xa1, 0xbe, 0xe4, 0x42, 0x9a, 0x8f, 0x76, 0x33, 0x80, 0xde, 0x5d, + 0x98, 0x45, 0xe4, 0x31, 0x8d, 0x99, 0x45, 0x9d, 0xbb, 0x21, 0x6e, 0xa6, 0x15, 0x3d, 0x45, 0x78, + 0xb9, 0xb2, 0x43, 0xdf, 0xec, 0xd3, 0x11, 0xcb, 0xa1, 0xdb, 0x9b, 0x7e, 0x5c, 0xd0, 0xb3, 0x97, + 0x33, 0x5f, 0x6c, 0xf6, 0xe5, 0x76, 0xd0, 0xed, 0xcb, 0xf7, 0x84, 0x77, 0x82, 0xe1, 0xc0, 0xf5, + 0x75, 0x58, 0x6b, 0xfb, 0x56, 0x6e, 0xf3, 0xa9, 0xa0, 0xa3, 0x5f, 0x06, 0xf6, 0x01, 0xdc, 0x18, + 0x78, 0x0f, 0x35, 0xb7, 0xd9, 0x31, 0x9d, 0x26, 0xaa, 0x13, 0x7b, 0x9f, 0x7c, 0xf3, 0x4c, 0x40, + 0x75, 0x64, 0x3c, 0xe6, 0x8a, 0xc5, 0x6a, 0x2a, 0x99, 0xfb, 0x3e, 0x98, 0x67, 0xc2, 0xe1, 0x9f, + 0x26, 0x74, 0x94, 0xdb, 0x4f, 0x05, 0x43, 0xe0, 0x6f, 0xa0, 0x1a, 0x4c, 0x68, 0x34, 0x49, 0x72, + 0x9b, 0x70, 0x73, 0x73, 0xa9, 0x11, 0xb9, 0x4f, 0x2a, 0x87, 0x8c, 0x38, 0xba, 0xe1, 0x9f, 0xb3, + 0x23, 0x47, 0x60, 0xbe, 0x97, 0x0e, 0xd9, 0x4b, 0x5b, 0xe1, 0x0d, 0x01, 0xdc, 0x8f, 0x8a, 0xf9, + 0xae, 0x42, 0x0c, 0x01, 0x6f, 0xa0, 0x17, 0xfb, 0x67, 0xc2, 0x96, 0xa5, 0x82, 0xc6, 0x6b, 0xa8, + 0xb4, 0x97, 0xd9, 0xef, 0xfe, 0xd2, 0x5e, 0x16, 0xfd, 0xbc, 0x78, 0x13, 0xb3, 0x39, 0x98, 0xd4, + 0x13, 0x97, 0xfd, 0xd2, 0x37, 0x84, 0x09, 0xa9, 0xa2, 0x43, 0xea, 0x90, 0xd2, 0x6f, 0x73, 0x17, + 0xd5, 0x98, 0x94, 0xa9, 0x90, 0xcc, 0x96, 0xde, 0x9d, 0x6b, 0xa4, 0xe0, 0xe0, 0xb6, 0xf7, 0xd7, + 0xa8, 0xf1, 0xe0, 0x9d, 0xe5, 0x89, 0xbc, 0x23, 0xdd, 0x27, 0x8c, 0x06, 0x6e, 0x21, 0x54, 0x7b, + 0x02, 0xca, 0x84, 0xa9, 0xe8, 0x9b, 0xa8, 0x39, 0x37, 0xf7, 0x82, 0x1f, 0x9e, 0x3f, 0xec, 0xd2, + 0x78, 0xcc, 0x06, 0xf1, 0x98, 0x4d, 0xa8, 0xf3, 0xd6, 0x1c, 0x73, 0xeb, 0x67, 0xc1, 0xab, 0xd7, + 0xad, 0x6b, 0x7f, 0x79, 0xdd, 0xba, 0xf6, 0xaf, 0xd7, 0xad, 0xe0, 0x27, 0x17, 0xad, 0xe0, 0xd7, + 0x17, 0xad, 0xe0, 0xb7, 0x17, 0xad, 0xe0, 0xf7, 0x17, 0xad, 0xe0, 0xd5, 0x45, 0x2b, 0xf8, 0xf3, + 0x45, 0x2b, 0xf8, 0xc7, 0x45, 0x2b, 0xf8, 0xf8, 0x93, 0x37, 0xfc, 0x27, 0x2b, 0x4d, 0xfb, 0x6b, + 0x9f, 0x72, 0xa9, 0x3c, 0x51, 0x76, 0x32, 0x6a, 0xd3, 0x11, 0x4b, 0x95, 0xf7, 0xbf, 0x16, 0x6e, + 0x7a, 0x54, 0xd5, 0xf4, 0xc3, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xce, 0xda, 0xe2, 0x21, 0xfc, + 0x15, 0x00, 0x00, } func (this *Spec) Equal(that interface{}) bool { @@ -2103,6 +2113,30 @@ func (this *Hooks) Equal(that interface{}) bool { return false } } + if len(this.CreateRuntime) != len(that1.CreateRuntime) { + return false + } + for i := range this.CreateRuntime { + if !this.CreateRuntime[i].Equal(&that1.CreateRuntime[i]) { + return false + } + } + if len(this.CreateContainer) != len(that1.CreateContainer) { + return false + } + for i := range this.CreateContainer { + if !this.CreateContainer[i].Equal(&that1.CreateContainer[i]) { + return false + } + } + if len(this.StartContainer) != len(that1.StartContainer) { + return false + } + for i := range this.StartContainer { + if !this.StartContainer[i].Equal(&that1.StartContainer[i]) { + return false + } + } if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { return false } @@ -3674,6 +3708,48 @@ func (m *Hooks) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.StartContainer) > 0 { + for iNdEx := len(m.StartContainer) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.StartContainer[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOci(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } + if len(m.CreateContainer) > 0 { + for iNdEx := len(m.CreateContainer) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.CreateContainer[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOci(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + } + if len(m.CreateRuntime) > 0 { + for iNdEx := len(m.CreateRuntime) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.CreateRuntime[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOci(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } if len(m.Poststop) > 0 { for iNdEx := len(m.Poststop) - 1; iNdEx >= 0; iNdEx-- { { @@ -5277,8 +5353,32 @@ func NewPopulatedHooks(r randyOci, easy bool) *Hooks { this.Poststop[i] = *v21 } } + if r.Intn(5) != 0 { + v22 := r.Intn(5) + this.CreateRuntime = make([]Hook, v22) + for i := 0; i < v22; i++ { + v23 := NewPopulatedHook(r, easy) + this.CreateRuntime[i] = *v23 + } + } + if r.Intn(5) != 0 { + v24 := r.Intn(5) + this.CreateContainer = make([]Hook, v24) + for i := 0; i < v24; i++ { + v25 := NewPopulatedHook(r, easy) + this.CreateContainer[i] = *v25 + } + } + if r.Intn(5) != 0 { + v26 := r.Intn(5) + this.StartContainer = make([]Hook, v26) + for i := 0; i < v26; i++ { + v27 := NewPopulatedHook(r, easy) + this.StartContainer[i] = *v27 + } + } if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedOci(r, 4) + this.XXX_unrecognized = randUnrecognizedOci(r, 7) } return this } @@ -5286,14 +5386,14 @@ func NewPopulatedHooks(r randyOci, easy bool) *Hooks { func NewPopulatedHook(r randyOci, easy bool) *Hook { this := &Hook{} this.Path = string(randStringOci(r)) - v22 := r.Intn(10) - this.Args = make([]string, v22) - for i := 0; i < v22; i++ { + v28 := r.Intn(10) + this.Args = make([]string, v28) + for i := 0; i < v28; i++ { this.Args[i] = string(randStringOci(r)) } - v23 := r.Intn(10) - this.Env = make([]string, v23) - for i := 0; i < v23; i++ { + v29 := r.Intn(10) + this.Env = make([]string, v29) + for i := 0; i < v29; i++ { this.Env[i] = string(randStringOci(r)) } this.Timeout = int64(r.Int63()) @@ -5309,25 +5409,25 @@ func NewPopulatedHook(r randyOci, easy bool) *Hook { func NewPopulatedLinux(r randyOci, easy bool) *Linux { this := &Linux{} if r.Intn(5) != 0 { - v24 := r.Intn(5) - this.UIDMappings = make([]LinuxIDMapping, v24) - for i := 0; i < v24; i++ { - v25 := NewPopulatedLinuxIDMapping(r, easy) - this.UIDMappings[i] = *v25 + v30 := r.Intn(5) + this.UIDMappings = make([]LinuxIDMapping, v30) + for i := 0; i < v30; i++ { + v31 := NewPopulatedLinuxIDMapping(r, easy) + this.UIDMappings[i] = *v31 } } if r.Intn(5) != 0 { - v26 := r.Intn(5) - this.GIDMappings = make([]LinuxIDMapping, v26) - for i := 0; i < v26; i++ { - v27 := NewPopulatedLinuxIDMapping(r, easy) - this.GIDMappings[i] = *v27 + v32 := r.Intn(5) + this.GIDMappings = make([]LinuxIDMapping, v32) + for i := 0; i < v32; i++ { + v33 := NewPopulatedLinuxIDMapping(r, easy) + this.GIDMappings[i] = *v33 } } if r.Intn(5) != 0 { - v28 := r.Intn(10) + v34 := r.Intn(10) this.Sysctl = make(map[string]string) - for i := 0; i < v28; i++ { + for i := 0; i < v34; i++ { this.Sysctl[randStringOci(r)] = randStringOci(r) } } @@ -5336,33 +5436,33 @@ func NewPopulatedLinux(r randyOci, easy bool) *Linux { } this.CgroupsPath = string(randStringOci(r)) if r.Intn(5) != 0 { - v29 := r.Intn(5) - this.Namespaces = make([]LinuxNamespace, v29) - for i := 0; i < v29; i++ { - v30 := NewPopulatedLinuxNamespace(r, easy) - this.Namespaces[i] = *v30 + v35 := r.Intn(5) + this.Namespaces = make([]LinuxNamespace, v35) + for i := 0; i < v35; i++ { + v36 := NewPopulatedLinuxNamespace(r, easy) + this.Namespaces[i] = *v36 } } if r.Intn(5) != 0 { - v31 := r.Intn(5) - this.Devices = make([]LinuxDevice, v31) - for i := 0; i < v31; i++ { - v32 := NewPopulatedLinuxDevice(r, easy) - this.Devices[i] = *v32 + v37 := r.Intn(5) + this.Devices = make([]LinuxDevice, v37) + for i := 0; i < v37; i++ { + v38 := NewPopulatedLinuxDevice(r, easy) + this.Devices[i] = *v38 } } if r.Intn(5) != 0 { this.Seccomp = NewPopulatedLinuxSeccomp(r, easy) } this.RootfsPropagation = string(randStringOci(r)) - v33 := r.Intn(10) - this.MaskedPaths = make([]string, v33) - for i := 0; i < v33; i++ { + v39 := r.Intn(10) + this.MaskedPaths = make([]string, v39) + for i := 0; i < v39; i++ { this.MaskedPaths[i] = string(randStringOci(r)) } - v34 := r.Intn(10) - this.ReadonlyPaths = make([]string, v34) - for i := 0; i < v34; i++ { + v40 := r.Intn(10) + this.ReadonlyPaths = make([]string, v40) + for i := 0; i < v40; i++ { this.ReadonlyPaths[i] = string(randStringOci(r)) } this.MountLabel = string(randStringOci(r)) @@ -5438,11 +5538,11 @@ func NewPopulatedLinuxDevice(r randyOci, easy bool) *LinuxDevice { func NewPopulatedLinuxResources(r randyOci, easy bool) *LinuxResources { this := &LinuxResources{} if r.Intn(5) != 0 { - v35 := r.Intn(5) - this.Devices = make([]LinuxDeviceCgroup, v35) - for i := 0; i < v35; i++ { - v36 := NewPopulatedLinuxDeviceCgroup(r, easy) - this.Devices[i] = *v36 + v41 := r.Intn(5) + this.Devices = make([]LinuxDeviceCgroup, v41) + for i := 0; i < v41; i++ { + v42 := NewPopulatedLinuxDeviceCgroup(r, easy) + this.Devices[i] = *v42 } } if r.Intn(5) != 0 { @@ -5458,11 +5558,11 @@ func NewPopulatedLinuxResources(r randyOci, easy bool) *LinuxResources { this.BlockIO = NewPopulatedLinuxBlockIO(r, easy) } if r.Intn(5) != 0 { - v37 := r.Intn(5) - this.HugepageLimits = make([]LinuxHugepageLimit, v37) - for i := 0; i < v37; i++ { - v38 := NewPopulatedLinuxHugepageLimit(r, easy) - this.HugepageLimits[i] = *v38 + v43 := r.Intn(5) + this.HugepageLimits = make([]LinuxHugepageLimit, v43) + for i := 0; i < v43; i++ { + v44 := NewPopulatedLinuxHugepageLimit(r, easy) + this.HugepageLimits[i] = *v44 } } if r.Intn(5) != 0 { @@ -5564,44 +5664,44 @@ func NewPopulatedLinuxBlockIO(r randyOci, easy bool) *LinuxBlockIO { this := &LinuxBlockIO{} this.Weight = uint32(r.Uint32()) this.LeafWeight = uint32(r.Uint32()) - if r.Intn(5) != 0 { - v39 := r.Intn(5) - this.WeightDevice = make([]LinuxWeightDevice, v39) - for i := 0; i < v39; i++ { - v40 := NewPopulatedLinuxWeightDevice(r, easy) - this.WeightDevice[i] = *v40 - } - } - if r.Intn(5) != 0 { - v41 := r.Intn(5) - this.ThrottleReadBpsDevice = make([]LinuxThrottleDevice, v41) - for i := 0; i < v41; i++ { - v42 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleReadBpsDevice[i] = *v42 - } - } - if r.Intn(5) != 0 { - v43 := r.Intn(5) - this.ThrottleWriteBpsDevice = make([]LinuxThrottleDevice, v43) - for i := 0; i < v43; i++ { - v44 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleWriteBpsDevice[i] = *v44 - } - } if r.Intn(5) != 0 { v45 := r.Intn(5) - this.ThrottleReadIOPSDevice = make([]LinuxThrottleDevice, v45) + this.WeightDevice = make([]LinuxWeightDevice, v45) for i := 0; i < v45; i++ { - v46 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleReadIOPSDevice[i] = *v46 + v46 := NewPopulatedLinuxWeightDevice(r, easy) + this.WeightDevice[i] = *v46 } } if r.Intn(5) != 0 { v47 := r.Intn(5) - this.ThrottleWriteIOPSDevice = make([]LinuxThrottleDevice, v47) + this.ThrottleReadBpsDevice = make([]LinuxThrottleDevice, v47) for i := 0; i < v47; i++ { v48 := NewPopulatedLinuxThrottleDevice(r, easy) - this.ThrottleWriteIOPSDevice[i] = *v48 + this.ThrottleReadBpsDevice[i] = *v48 + } + } + if r.Intn(5) != 0 { + v49 := r.Intn(5) + this.ThrottleWriteBpsDevice = make([]LinuxThrottleDevice, v49) + for i := 0; i < v49; i++ { + v50 := NewPopulatedLinuxThrottleDevice(r, easy) + this.ThrottleWriteBpsDevice[i] = *v50 + } + } + if r.Intn(5) != 0 { + v51 := r.Intn(5) + this.ThrottleReadIOPSDevice = make([]LinuxThrottleDevice, v51) + for i := 0; i < v51; i++ { + v52 := NewPopulatedLinuxThrottleDevice(r, easy) + this.ThrottleReadIOPSDevice[i] = *v52 + } + } + if r.Intn(5) != 0 { + v53 := r.Intn(5) + this.ThrottleWriteIOPSDevice = make([]LinuxThrottleDevice, v53) + for i := 0; i < v53; i++ { + v54 := NewPopulatedLinuxThrottleDevice(r, easy) + this.ThrottleWriteIOPSDevice[i] = *v54 } } if !easy && r.Intn(10) != 0 { @@ -5645,11 +5745,11 @@ func NewPopulatedLinuxNetwork(r randyOci, easy bool) *LinuxNetwork { this := &LinuxNetwork{} this.ClassID = uint32(r.Uint32()) if r.Intn(5) != 0 { - v49 := r.Intn(5) - this.Priorities = make([]LinuxInterfacePriority, v49) - for i := 0; i < v49; i++ { - v50 := NewPopulatedLinuxInterfacePriority(r, easy) - this.Priorities[i] = *v50 + v55 := r.Intn(5) + this.Priorities = make([]LinuxInterfacePriority, v55) + for i := 0; i < v55; i++ { + v56 := NewPopulatedLinuxInterfacePriority(r, easy) + this.Priorities[i] = *v56 } } if !easy && r.Intn(10) != 0 { @@ -5681,22 +5781,22 @@ func NewPopulatedLinuxInterfacePriority(r randyOci, easy bool) *LinuxInterfacePr func NewPopulatedLinuxSeccomp(r randyOci, easy bool) *LinuxSeccomp { this := &LinuxSeccomp{} this.DefaultAction = string(randStringOci(r)) - v51 := r.Intn(10) - this.Architectures = make([]string, v51) - for i := 0; i < v51; i++ { + v57 := r.Intn(10) + this.Architectures = make([]string, v57) + for i := 0; i < v57; i++ { this.Architectures[i] = string(randStringOci(r)) } - v52 := r.Intn(10) - this.Flags = make([]string, v52) - for i := 0; i < v52; i++ { + v58 := r.Intn(10) + this.Flags = make([]string, v58) + for i := 0; i < v58; i++ { this.Flags[i] = string(randStringOci(r)) } if r.Intn(5) != 0 { - v53 := r.Intn(5) - this.Syscalls = make([]LinuxSyscall, v53) - for i := 0; i < v53; i++ { - v54 := NewPopulatedLinuxSyscall(r, easy) - this.Syscalls[i] = *v54 + v59 := r.Intn(5) + this.Syscalls = make([]LinuxSyscall, v59) + for i := 0; i < v59; i++ { + v60 := NewPopulatedLinuxSyscall(r, easy) + this.Syscalls[i] = *v60 } } if !easy && r.Intn(10) != 0 { @@ -5719,9 +5819,9 @@ func NewPopulatedLinuxSeccompArg(r randyOci, easy bool) *LinuxSeccompArg { func NewPopulatedLinuxSyscall(r randyOci, easy bool) *LinuxSyscall { this := &LinuxSyscall{} - v55 := r.Intn(10) - this.Names = make([]string, v55) - for i := 0; i < v55; i++ { + v61 := r.Intn(10) + this.Names = make([]string, v61) + for i := 0; i < v61; i++ { this.Names[i] = string(randStringOci(r)) } this.Action = string(randStringOci(r)) @@ -5731,11 +5831,11 @@ func NewPopulatedLinuxSyscall(r randyOci, easy bool) *LinuxSyscall { this.ErrnoRet = NewPopulatedLinuxSyscall_Errnoret(r, easy) } if r.Intn(5) != 0 { - v56 := r.Intn(5) - this.Args = make([]LinuxSeccompArg, v56) - for i := 0; i < v56; i++ { - v57 := NewPopulatedLinuxSeccompArg(r, easy) - this.Args[i] = *v57 + v62 := r.Intn(5) + this.Args = make([]LinuxSeccompArg, v62) + for i := 0; i < v62; i++ { + v63 := NewPopulatedLinuxSeccompArg(r, easy) + this.Args[i] = *v63 } } if !easy && r.Intn(10) != 0 { @@ -5777,9 +5877,9 @@ func randUTF8RuneOci(r randyOci) rune { return rune(ru + 61) } func randStringOci(r randyOci) string { - v58 := r.Intn(100) - tmps := make([]rune, v58) - for i := 0; i < v58; i++ { + v64 := r.Intn(100) + tmps := make([]rune, v64) + for i := 0; i < v64; i++ { tmps[i] = randUTF8RuneOci(r) } return string(tmps) @@ -5801,11 +5901,11 @@ func randFieldOci(dAtA []byte, r randyOci, fieldNumber int, wire int) []byte { switch wire { case 0: dAtA = encodeVarintPopulateOci(dAtA, uint64(key)) - v59 := r.Int63() + v65 := r.Int63() if r.Intn(2) == 0 { - v59 *= -1 + v65 *= -1 } - dAtA = encodeVarintPopulateOci(dAtA, uint64(v59)) + dAtA = encodeVarintPopulateOci(dAtA, uint64(v65)) case 1: dAtA = encodeVarintPopulateOci(dAtA, uint64(key)) dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) @@ -6133,6 +6233,24 @@ func (m *Hooks) Size() (n int) { n += 1 + l + sovOci(uint64(l)) } } + if len(m.CreateRuntime) > 0 { + for _, e := range m.CreateRuntime { + l = e.Size() + n += 1 + l + sovOci(uint64(l)) + } + } + if len(m.CreateContainer) > 0 { + for _, e := range m.CreateContainer { + l = e.Size() + n += 1 + l + sovOci(uint64(l)) + } + } + if len(m.StartContainer) > 0 { + for _, e := range m.StartContainer { + l = e.Size() + n += 1 + l + sovOci(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6797,7 +6915,7 @@ func (this *Spec) String() string { } repeatedStringForMounts += "}" keysForAnnotations := make([]string, 0, len(this.Annotations)) - for k := range this.Annotations { + for k, _ := range this.Annotations { keysForAnnotations = append(keysForAnnotations, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForAnnotations) @@ -6948,10 +7066,28 @@ func (this *Hooks) String() string { repeatedStringForPoststop += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," } repeatedStringForPoststop += "}" + repeatedStringForCreateRuntime := "[]Hook{" + for _, f := range this.CreateRuntime { + repeatedStringForCreateRuntime += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," + } + repeatedStringForCreateRuntime += "}" + repeatedStringForCreateContainer := "[]Hook{" + for _, f := range this.CreateContainer { + repeatedStringForCreateContainer += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," + } + repeatedStringForCreateContainer += "}" + repeatedStringForStartContainer := "[]Hook{" + for _, f := range this.StartContainer { + repeatedStringForStartContainer += strings.Replace(strings.Replace(f.String(), "Hook", "Hook", 1), `&`, ``, 1) + "," + } + repeatedStringForStartContainer += "}" s := strings.Join([]string{`&Hooks{`, `Prestart:` + repeatedStringForPrestart + `,`, `Poststart:` + repeatedStringForPoststart + `,`, `Poststop:` + repeatedStringForPoststop + `,`, + `CreateRuntime:` + repeatedStringForCreateRuntime + `,`, + `CreateContainer:` + repeatedStringForCreateContainer + `,`, + `StartContainer:` + repeatedStringForStartContainer + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -6996,7 +7132,7 @@ func (this *Linux) String() string { } repeatedStringForDevices += "}" keysForSysctl := make([]string, 0, len(this.Sysctl)) - for k := range this.Sysctl { + for k, _ := range this.Sysctl { keysForSysctl = append(keysForSysctl, k) } github_com_gogo_protobuf_sortkeys.Strings(keysForSysctl) @@ -9298,6 +9434,108 @@ func (m *Hooks) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateRuntime", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOci + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOci + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOci + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CreateRuntime = append(m.CreateRuntime, Hook{}) + if err := m.CreateRuntime[len(m.CreateRuntime)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateContainer", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOci + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOci + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOci + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CreateContainer = append(m.CreateContainer, Hook{}) + if err := m.CreateContainer[len(m.CreateContainer)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StartContainer", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOci + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOci + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOci + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StartContainer = append(m.StartContainer, Hook{}) + if err := m.StartContainer[len(m.StartContainer)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipOci(dAtA[iNdEx:]) diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go index 634dc7fd8..6109d29ac 100644 --- a/src/runtime/virtcontainers/pkg/mock/mock.go +++ b/src/runtime/virtcontainers/pkg/mock/mock.go @@ -125,6 +125,10 @@ func (p *HybridVSockTTRPCMockImp) UpdateContainer(ctx context.Context, req *pb.U return emptyResp, nil } +func (p *HybridVSockTTRPCMockImp) UpdateEphemeralMounts(ctx context.Context, req *pb.UpdateEphemeralMountsRequest) (*gpb.Empty, error) { + return emptyResp, nil +} + func (p *HybridVSockTTRPCMockImp) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*gpb.Empty, error) { return emptyResp, nil } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 523c07259..dcc93048f 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -17,6 +17,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "sync" "syscall" @@ -44,6 +45,9 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" ) // sandboxTracingTags defines tags for the trace span @@ -2109,11 +2113,69 @@ func (s *Sandbox) updateResources(ctx context.Context) error { if newMemoryMB == finalMemoryMB { break } + } + tmpfsMounts, err := s.prepareEphemeralMounts(finalMemoryMB) + if err != nil { + return err + } + if err := s.agent.updateEphemeralMounts(ctx, tmpfsMounts); err != nil { + // upgrade path: if runtime is newer version, but agent is old + // then ignore errUnimplemented + if grpcStatus.Convert(err).Code() == codes.Unimplemented { + s.Logger().Warnf("agent does not support updateMounts") + return nil + } + return err } return nil +} +func (s *Sandbox) prepareEphemeralMounts(memoryMB uint32) ([]*grpc.Storage, error) { + tmpfsMounts := []*grpc.Storage{} + for _, c := range s.containers { + for _, mount := range c.mounts { + // if a tmpfs ephemeral mount is present + // update its size to occupy the entire sandbox's memory + if mount.Type == KataEphemeralDevType { + sizeLimited := false + for _, opt := range mount.Options { + if strings.HasPrefix(opt, "size") { + sizeLimited = true + } + } + if sizeLimited { // do not resize sizeLimited emptyDirs + continue + } + + mountOptions := []string{"remount", fmt.Sprintf("size=%dM", memoryMB)} + + origin_src := mount.Source + stat := syscall.Stat_t{} + err := syscall.Stat(origin_src, &stat) + if err != nil { + return nil, err + } + + // if volume's gid isn't root group(default group), this means there's + // an specific fsGroup is set on this local volume, then it should pass + // to guest. + if stat.Gid != 0 { + mountOptions = append(mountOptions, fmt.Sprintf("%s=%d", fsGid, stat.Gid)) + } + + tmpfsMounts = append(tmpfsMounts, &grpc.Storage{ + Driver: KataEphemeralDevType, + MountPoint: filepath.Join(ephemeralPath(), filepath.Base(mount.Source)), + Source: "tmpfs", + Fstype: "tmpfs", + Options: mountOptions, + }) + } + } + } + return tmpfsMounts, nil } func (s *Sandbox) updateMemory(ctx context.Context, newMemoryMB uint32) error { diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 8b86fa995..de3b1885c 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -179,6 +179,65 @@ func TestCalculateSandboxMem(t *testing.T) { } } +func TestPrepareEphemeralMounts(t *testing.T) { + sandbox := &Sandbox{} + sandbox.containers = map[string]*Container{ + "container1": { + mounts: []Mount{ + { + // happy path + Type: KataEphemeralDevType, + Options: []string{ + "rw", + "relatime", + }, + Source: "/tmp", + }, + { + // should be ignored because it is not KataEphemeralDevType + Type: KataLocalDevType, + Options: []string{ + "rw", + "relatime", + }, + Source: "/tmp", + }, + { + // should be ignored because it is sizeLimited + Type: KataLocalDevType, + Options: []string{ + "rw", + "relatime", + "size=1M", + }, + Source: "/tmp", + }, + }, + }, + } + + storages, err := sandbox.prepareEphemeralMounts(1024) + assert.NoError(t, err) + assert.Equal(t, len(storages), 1) + for _, s := range storages { + assert.Equal(t, s.Driver, KataEphemeralDevType) + assert.Equal(t, s.Source, "tmpfs") + assert.Equal(t, s.Fstype, "tmpfs") + assert.Equal(t, s.MountPoint, filepath.Join(ephemeralPath(), "tmp")) + assert.Equal(t, len(s.Options), 2) // remount, size=1024M + + validSet := map[string]struct{}{ + "remount": {}, + "size=1024M": {}, + } + for _, opt := range s.Options { + if _, ok := validSet[opt]; !ok { + t.Error("invalid mount opt: " + opt) + } + } + } +} + func TestCalculateSandboxMemHandlesNegativeLimits(t *testing.T) { sandbox := &Sandbox{} sandbox.config = &SandboxConfig{} From 40f4eef5355fcac66a8de519393f69338e447de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 15 Feb 2023 16:38:26 +0100 Subject: [PATCH 10/40] build: Use the correct kernel name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When calling `MAKE_KERNEL_NAME` we're considering the default kernel name will be `vmlinux.container` or `vmlinuz.container`, which is not the case as the runtime-rs, when used with dragonball, relies on the `vmlinu[zx]-dragonball-experimental.container` kernel. Other hypervisors will have to introduce a similar `MAKE_KERNEL_NAME_${HYPERVISOR}` to adapt this to the kernel they want to use, similarly to what's already done for the go runtime. By doing this we also ensure that no changes in the configuration file will be required to run runtime-rs, with dragonball, as part of our CI or as part of kata-deploy. Fixes: #6290 Signed-off-by: Fabiano Fidêncio --- src/runtime-rs/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 4c17fe36c..599949bb0 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -203,7 +203,7 @@ ifneq (,$(DBCMD)) DEFNETWORKMODEL_DB := tcfilter KERNELPARAMS = console=ttyS1 agent.log_vport=1025 KERNELTYPE_DB = uncompressed - KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_DB)) + KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME_DB,$(KERNELTYPE_DB)) KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB) DEFSANDBOXCGROUPONLY = true RUNTIMENAME := virt_container @@ -371,8 +371,8 @@ endef # Returns the name of the kernel file to use based on the provided KERNELTYPE. # $1 : KERNELTYPE (compressed or uncompressed) -define MAKE_KERNEL_NAME -$(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container) +define MAKE_KERNEL_NAME_DB +$(if $(findstring uncompressed,$1),vmlinux-dragonball-experimental.container,vmlinuz-dragonball-experimental.container) endef .DEFAULT_GOAL := default From 974a5c22f0061bd27728e9a6de752899523147f7 Mon Sep 17 00:00:00 2001 From: Henry Beberman Date: Fri, 10 Jun 2022 23:21:18 +0000 Subject: [PATCH 11/40] runtime: add support for Hyper-V MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds /dev/mshv to the list of sandbox devices so that VMMs can create Hyper-V VMs. In our testing, this also doesn't error out in case /dev/mshv isn't present. Fixes #6454. Signed-off-by: Aurélien Bombo --- src/runtime/pkg/resourcecontrol/cgroups.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/pkg/resourcecontrol/cgroups.go b/src/runtime/pkg/resourcecontrol/cgroups.go index da88820ef..be8e9dc97 100644 --- a/src/runtime/pkg/resourcecontrol/cgroups.go +++ b/src/runtime/pkg/resourcecontrol/cgroups.go @@ -67,7 +67,8 @@ func sandboxDevices() []specs.LinuxDeviceCgroup { // In order to run Virtual Machines and create virtqueues, hypervisors // need access to certain character devices in the host, like kvm and vhost-net. hypervisorDevices := []string{ - "/dev/kvm", // To run virtual machines + "/dev/kvm", // To run virtual machines with KVM + "/dev/mshv", // To run virtual machines with Hyper-V "/dev/vhost-net", // To create virtqueues "/dev/vfio/vfio", // To access VFIO devices "/dev/vhost-vsock", // To interact with vsock if From 97cdba97ea9872d4be2efbbb58abafe09bc1d67f Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Tue, 14 Mar 2023 16:30:18 +0800 Subject: [PATCH 12/40] runtime-rs: update load_config comment Since shimv2 create task option is already implemented, we need to update the corresponding comments. Also, the ordering is also updated to fit with the code. fixes: #3961 Signed-off-by: Chao Wu --- src/runtime-rs/crates/runtimes/src/manager.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 84c1fa4a9..d8aad3a0a 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -353,9 +353,10 @@ impl RuntimeHandlerManager { /// Config override ordering(high to low): /// 1. podsandbox annotation -/// 2. shimv2 create task option -/// TODO: https://github.com/kata-containers/kata-containers/issues/3961 -/// 3. environment +/// 2. environment variable +/// 3. shimv2 create task option +/// 4. If above three are not set, then get default path from DEFAULT_RUNTIME_CONFIGURATIONS +/// in kata-containers/src/libs/kata-types/src/config/default.rs, in array order. fn load_config(spec: &oci::Spec, option: &Option>) -> Result { const KATA_CONF_FILE: &str = "KATA_CONF_FILE"; let annotation = Annotation::new(spec.annotations.clone()); From a8b55bf8746d48e1793e5a95ea59ebe9ca3224d8 Mon Sep 17 00:00:00 2001 From: "Eduardo Lima (Etrunko)" Date: Wed, 15 Mar 2023 12:21:12 -0300 Subject: [PATCH 13/40] dependency: update cgroups-rs Huge pages failure with cgroups v2. https://github.com/kata-containers/cgroups-rs/issues/112 Fixes: #6470 Signed-off-by: Eduardo Lima (Etrunko) --- src/agent/Cargo.lock | 4 ++-- src/agent/Cargo.toml | 2 +- src/agent/rustjail/Cargo.toml | 2 +- src/libs/Cargo.lock | 4 ++-- src/libs/kata-sys-util/Cargo.toml | 2 +- src/runtime-rs/Cargo.lock | 4 ++-- src/runtime-rs/crates/resource/Cargo.toml | 2 +- src/tools/agent-ctl/Cargo.lock | 4 ++-- src/tools/runk/Cargo.lock | 4 ++-- src/tools/runk/libcontainer/Cargo.toml | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 0de459116..1e4f8425b 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -276,9 +276,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 8851798fd..c72d2db30 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -51,7 +51,7 @@ log = "0.4.11" prometheus = { version = "0.13.0", features = ["process"] } procfs = "0.12.0" anyhow = "1.0.32" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } # Tracing tracing = "0.1.26" diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index d4ba4e6f3..8c9c2230d 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -25,7 +25,7 @@ scan_fmt = "0.2.6" regex = "1.5.6" path-absolutize = "1.2.0" anyhow = "1.0.32" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } rlimit = "0.5.3" cfg-if = "0.1.0" diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index e0c1296a2..16bdfda6f 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -110,9 +110,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index efcde317e..20497bfa6 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -12,7 +12,7 @@ edition = "2018" [dependencies] byteorder = "1.4.3" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } chrono = "0.4.0" common-path = "=1.0.0" fail = "0.5.0" diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 3e2d68fd1..caff85873 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -402,9 +402,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 7ae1f799f..73b577c5a 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -14,7 +14,7 @@ anyhow = "^1.0" async-trait = "0.1.48" bitflags = "1.2.1" byte-unit = "4.0.14" -cgroups-rs = "0.3.1" +cgroups-rs = "0.3.2" futures = "0.3.11" lazy_static = "1.4.0" libc = ">=0.2.39" diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index deb3385f6..234830c3a 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -250,9 +250,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock index 9a8e92748..f9ebce67b 100644 --- a/src/tools/runk/Cargo.lock +++ b/src/tools/runk/Cargo.lock @@ -233,9 +233,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cgroups-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1" +checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965" dependencies = [ "libc", "log", diff --git a/src/tools/runk/libcontainer/Cargo.toml b/src/tools/runk/libcontainer/Cargo.toml index 82e56b06e..8df62d3b2 100644 --- a/src/tools/runk/libcontainer/Cargo.toml +++ b/src/tools/runk/libcontainer/Cargo.toml @@ -20,7 +20,7 @@ chrono = { version = "0.4.19", features = ["serde"] } serde = { version = "1.0.133", features = ["derive"] } serde_json = "1.0.74" scopeguard = "1.1.0" -cgroups = { package = "cgroups-rs", version = "0.3.1" } +cgroups = { package = "cgroups-rs", version = "0.3.2" } procfs = "0.14.0" [dev-dependencies] From 68a586e52c88914f97519f6f664e16525dcfc4fd Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 16 Feb 2022 15:18:07 +0000 Subject: [PATCH 14/40] agent: Use a constant for CCW root bus path used a function like PCI does, but this is not necessary Signed-off-by: Jakob Naucke --- src/agent/src/device.rs | 4 ++-- src/agent/src/linux_abi.rs | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index ec277f78e..2d01b63ab 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -280,7 +280,7 @@ pub async fn get_virtio_blk_ccw_device_name( sandbox: &Arc>, device: &ccw::Device, ) -> Result { - let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device); + let matcher = VirtioBlkCCWMatcher::new(CCW_ROOT_BUS_PATH, device); let uev = wait_for_uevent(sandbox, matcher).await?; let devname = uev.devname; return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() { @@ -1378,7 +1378,7 @@ mod tests { #[cfg(target_arch = "s390x")] #[tokio::test] async fn test_virtio_blk_ccw_matcher() { - let root_bus = create_ccw_root_bus_path(); + let root_bus = CCW_ROOT_BUS_PATH; let subsystem = "block"; let devname = "vda"; let relpath = "0.0.0002"; diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index c4d304d28..fbfb9b377 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -65,9 +65,8 @@ pub fn create_pci_root_bus_path() -> String { } #[cfg(target_arch = "s390x")] -pub fn create_ccw_root_bus_path() -> String { - String::from("/devices/css0") -} +pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0"; + // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources From db89c88f4fcb3d59456595932adf8fd82a32fab5 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 15 Feb 2022 17:15:21 +0100 Subject: [PATCH 15/40] agent: Use cfg-if for s390x CCW Uses fewer lines in upcoming VFIO-AP support. Signed-off-by: Jakob Naucke --- src/agent/Cargo.lock | 1 + src/agent/Cargo.toml | 1 + src/agent/src/device.rs | 9 +++++++-- src/agent/src/linux_abi.rs | 9 +++++++-- src/agent/src/main.rs | 9 +++++++-- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 0de459116..119183cb4 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -801,6 +801,7 @@ dependencies = [ "async-recursion", "async-trait", "capctl", + "cfg-if 1.0.0", "cgroups-rs", "clap", "futures", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 8851798fd..b5a82f257 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -48,6 +48,7 @@ slog-scope = "4.1.2" slog-stdlog = "4.0.0" log = "0.4.11" +cfg-if = "1.0.0" prometheus = { version = "0.13.0", features = ["process"] } procfs = "0.12.0" anyhow = "1.0.32" diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 2d01b63ab..620163ee0 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -16,13 +16,12 @@ use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; -#[cfg(target_arch = "s390x")] -use crate::ccw; use crate::linux_abi::*; use crate::pci; use crate::sandbox::Sandbox; use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher}; use anyhow::{anyhow, Context, Result}; +use cfg_if::cfg_if; use oci::{LinuxDeviceCgroup, LinuxResources, Spec}; use protocols::agent::Device; use tracing::instrument; @@ -54,6 +53,12 @@ pub const DRIVER_VFIO_TYPE: &str = "vfio"; pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs"; pub const FS_TYPE_HUGETLB: &str = "hugetlbfs"; +cfg_if! { + if #[cfg(target_arch = "s390x")] { + use crate::ccw; + } +} + #[instrument] pub fn online_device(path: &str) -> Result<()> { fs::write(path, "1")?; diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index fbfb9b377..9df398d61 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -3,6 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 // +use cfg_if::cfg_if; + /// Linux ABI related constants. #[cfg(target_arch = "aarch64")] @@ -64,8 +66,11 @@ pub fn create_pci_root_bus_path() -> String { ret } -#[cfg(target_arch = "s390x")] -pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0"; +cfg_if! { + if #[cfg(target_arch = "s390x")] { + pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0"; + } +} // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index d8e9fc828..4ca89a9a1 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -20,6 +20,7 @@ extern crate scopeguard; extern crate slog; use anyhow::{anyhow, Context, Result}; +use cfg_if::cfg_if; use clap::{AppSettings, Parser}; use nix::fcntl::OFlag; use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr}; @@ -34,8 +35,6 @@ use std::process::exit; use std::sync::Arc; use tracing::{instrument, span}; -#[cfg(target_arch = "s390x")] -mod ccw; mod config; mod console; mod device; @@ -74,6 +73,12 @@ use tokio::{ mod rpc; mod tracer; +cfg_if! { + if #[cfg(target_arch = "s390x")] { + mod ccw; + } +} + const NAME: &str = "kata-agent"; lazy_static! { From 4c527d00c7b747b8ffc66b8126d57f3fdaa61206 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 8 Mar 2022 18:50:58 +0100 Subject: [PATCH 16/40] agent: Rename VFIO handling to VFIO PCI handling e.g., split_vfio_option is PCI-specific and should instead be named split_vfio_pci_option. This mutually affects the runtime, most notably how the labels are named for the agent. Signed-off-by: Jakob Naucke --- src/agent/src/device.rs | 32 ++++++++++++--------- src/runtime/pkg/device/config/config.go | 8 +++--- src/runtime/pkg/device/drivers/utils.go | 4 +-- src/runtime/pkg/device/drivers/vfio.go | 4 +-- src/runtime/pkg/device/drivers/vfio_test.go | 4 +-- src/runtime/virtcontainers/kata_agent.go | 14 ++++----- src/runtime/virtcontainers/qemu.go | 8 +++--- 7 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 620163ee0..23bc154a7 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -45,11 +45,11 @@ pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm"; pub const DRIVER_EPHEMERAL_TYPE: &str = "ephemeral"; pub const DRIVER_LOCAL_TYPE: &str = "local"; pub const DRIVER_WATCHABLE_BIND_TYPE: &str = "watchable-bind"; -// VFIO device to be bound to a guest kernel driver -pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk"; -// VFIO device to be bound to vfio-pci and made available inside the +// VFIO PCI device to be bound to a guest kernel driver +pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk"; +// VFIO PCI device to be bound to vfio-pci and made available inside the // container as a VFIO device node -pub const DRIVER_VFIO_TYPE: &str = "vfio"; +pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci"; pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs"; pub const FS_TYPE_HUGETLB: &str = "hugetlbfs"; @@ -704,7 +704,7 @@ async fn virtio_nvdimm_device_handler( Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into()) } -fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { +fn split_vfio_pci_option(opt: &str) -> Option<(&str, &str)> { let mut tokens = opt.split('='); let hostbdf = tokens.next()?; let path = tokens.next()?; @@ -719,14 +719,18 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> { // Each option should have the form "DDDD:BB:DD.F=" // DDDD:BB:DD.F is the device's PCI address in the host // is a PCI path to the device in the guest (see pci.rs) -async fn vfio_device_handler(device: &Device, sandbox: &Arc>) -> Result { - let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE; +#[instrument] +async fn vfio_pci_device_handler( + device: &Device, + sandbox: &Arc>, +) -> Result { + let vfio_in_guest = device.field_type != DRIVER_VFIO_PCI_GK_TYPE; let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new(); let mut group = None; for opt in device.options.iter() { let (host, pcipath) = - split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?; + split_vfio_pci_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?; let host = pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?; let pcipath = pci::Path::from_str(pcipath)?; @@ -833,7 +837,9 @@ async fn add_device(device: &Device, sandbox: &Arc>) -> Result virtiommio_blk_device_handler(device, sandbox).await, DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await, DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await, - DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await, + DRIVER_VFIO_PCI_GK_TYPE | DRIVER_VFIO_PCI_TYPE => { + vfio_pci_device_handler(device, sandbox).await + } _ => Err(anyhow!("Unknown device type {}", device.field_type)), } } @@ -1492,13 +1498,13 @@ mod tests { } #[test] - fn test_split_vfio_option() { + fn test_split_vfio_pci_option() { assert_eq!( - split_vfio_option("0000:01:00.0=02/01"), + split_vfio_pci_option("0000:01:00.0=02/01"), Some(("0000:01:00.0", "02/01")) ); - assert_eq!(split_vfio_option("0000:01:00.0=02/01=rubbish"), None); - assert_eq!(split_vfio_option("0000:01:00.0"), None); + assert_eq!(split_vfio_pci_option("0000:01:00.0=02/01=rubbish"), None); + assert_eq!(split_vfio_pci_option("0000:01:00.0"), None); } #[test] diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index 35af6afec..a44723dac 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -258,11 +258,11 @@ const ( // VFIODeviceErrorType is the error type of VFIO device VFIODeviceErrorType VFIODeviceType = iota - // VFIODeviceNormalType is a normal VFIO device type - VFIODeviceNormalType + // VFIOPCIDeviceNormalType is a normal VFIO PCI device type + VFIOPCIDeviceNormalType - // VFIODeviceMediatedType is a VFIO mediated device type - VFIODeviceMediatedType + // VFIOPCIDeviceMediatedType is a VFIO PCI mediated device type + VFIOPCIDeviceMediatedType ) // VFIODev represents a VFIO drive used for hotplugging diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index 25f021eda..abdf5d816 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -94,12 +94,12 @@ func GetVFIODeviceType(deviceFileName string) config.VFIODeviceType { tokens := strings.Split(deviceFileName, ":") vfioDeviceType := config.VFIODeviceErrorType if len(tokens) == 3 { - vfioDeviceType = config.VFIODeviceNormalType + vfioDeviceType = config.VFIOPCIDeviceNormalType } else { //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 tokens = strings.Split(deviceFileName, "-") if len(tokens) == 5 { - vfioDeviceType = config.VFIODeviceMediatedType + vfioDeviceType = config.VFIOPCIDeviceMediatedType } } return vfioDeviceType diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 58658b0b8..86033aa73 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -207,12 +207,12 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS vfioDeviceType = GetVFIODeviceType(deviceFileName) switch vfioDeviceType { - case config.VFIODeviceNormalType: + case config.VFIOPCIDeviceNormalType: // Get bdf of device eg. 0000:00:1c.0 deviceBDF = getBDF(deviceFileName) // Get sysfs path used by cloud-hypervisor deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName) - case config.VFIODeviceMediatedType: + case config.VFIOPCIDeviceMediatedType: // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) deviceSysfsDev, err = getSysfsDev(sysfsDevStr) diff --git a/src/runtime/pkg/device/drivers/vfio_test.go b/src/runtime/pkg/device/drivers/vfio_test.go index 3c25a64c3..8ee008e3b 100644 --- a/src/runtime/pkg/device/drivers/vfio_test.go +++ b/src/runtime/pkg/device/drivers/vfio_test.go @@ -32,9 +32,9 @@ func TestGetVFIODetails(t *testing.T) { deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "") switch vfioDeviceType { - case config.VFIODeviceNormalType: + case config.VFIOPCIDeviceNormalType: assert.Equal(t, d.expectedStr, deviceBDF) - case config.VFIODeviceMediatedType: + case config.VFIOPCIDeviceMediatedType: assert.Equal(t, d.expectedStr, deviceSysfsDev) default: assert.NotNil(t, err) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 66784404f..98411aa45 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1117,10 +1117,10 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * groupNum := filepath.Base(dev.ContainerPath) - // Each /dev/vfio/NN device represents a VFIO group, which - // could include several PCI devices. So we give group - // information in the main structure, then list each - // individual PCI device in the Options array. + // For VFIO-PCI, each /dev/vfio/NN device represents a VFIO group, + // which could include several PCI devices. So we give group + // information in the main structure, then list each individual PCI + // device in the Options array. // // Each option is formatted as "DDDD:BB:DD.F=" // DDDD:BB:DD.F is the device's PCI address on the @@ -1128,9 +1128,9 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * // (see qomGetPciPath() for details). kataDevice := &grpc.Device{ ContainerPath: dev.ContainerPath, - Type: kataVfioDevType, + Type: kataVfioPciDevType, Id: groupNum, - Options: make([]string, len(devList)), + Options: nil, } // We always pass the device information to the agent, since @@ -1138,7 +1138,7 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * // on the vfio_mode, we need to use a different device type so // the agent can handle it properly if c.sandbox.config.VfioMode == config.VFIOModeGuestKernel { - kataDevice.Type = kataVfioGuestKernelDevType + kataDevice.Type = kataVfioPciGuestKernelDevType } for i, pciDev := range devList { diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 0d4fa3e85..17b3edbe7 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1743,9 +1743,9 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op } switch device.Type { - case config.VFIODeviceNormalType: + case config.VFIOPCIDeviceNormalType: err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, device.Bus, romFile) - case config.VFIODeviceMediatedType: + case config.VFIOPCIDeviceMediatedType: if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) } else { @@ -1767,9 +1767,9 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op }() switch device.Type { - case config.VFIODeviceNormalType: + case config.VFIOPCIDeviceNormalType: err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile) - case config.VFIODeviceMediatedType: + case config.VFIOPCIDeviceMediatedType: if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) } else { From b546eca26f0ef8e35288afe6cab0021fdc75ed46 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 8 Mar 2022 18:51:08 +0100 Subject: [PATCH 17/40] runtime: Generalize VFIO devices Generalize VFIO devices to allow for adding AP in the next patch. The logic for VFIOPciDeviceMediatedType() has been changed and IsAPVFIOMediatedDevice() has been removed. The rationale for the revomal is: - VFIODeviceMediatedType is divided into 2 subtypes for AP and PCI - Logic of checking a subtype of mediated device is included in GetVFIODeviceType() - VFIOPciDeviceMediatedType() can simply fulfill the device addition based on a type categorized by GetVFIODeviceType() Signed-off-by: Jakob Naucke --- src/runtime/pkg/device/config/config.go | 22 +++++- src/runtime/pkg/device/drivers/vfio.go | 50 ++++++++---- src/runtime/virtcontainers/clh.go | 11 ++- src/runtime/virtcontainers/clh_test.go | 2 +- src/runtime/virtcontainers/kata_agent.go | 6 +- src/runtime/virtcontainers/qemu.go | 79 +++++++++++-------- src/runtime/virtcontainers/qemu_arch_base.go | 11 +-- .../virtcontainers/qemu_arch_base_test.go | 4 +- src/runtime/virtcontainers/sandbox.go | 16 +++- .../virtcontainers/utils/utils_linux.go | 3 +- .../virtcontainers/utils/utils_linux_test.go | 16 ---- 11 files changed, 135 insertions(+), 85 deletions(-) diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index a44723dac..6bcbed32c 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -265,8 +265,14 @@ const ( VFIOPCIDeviceMediatedType ) -// VFIODev represents a VFIO drive used for hotplugging -type VFIODev struct { +type VFIODev interface { + GetID() *string + GetType() VFIODeviceType + GetSysfsDev() *string +} + +// VFIOPCIDev represents a VFIO PCI device used for hotplugging +type VFIOPCIDev struct { // ID is used to identify this drive in the hypervisor options. ID string @@ -298,6 +304,18 @@ type VFIODev struct { IsPCIe bool } +func (d VFIOPCIDev) GetID() *string { + return &d.ID +} + +func (d VFIOPCIDev) GetType() VFIODeviceType { + return d.Type +} + +func (d VFIOPCIDev) GetSysfsDev() *string { + return &d.SysfsDev +} + // RNGDev represents a random number generator device type RNGDev struct { // ID is used to identify the device in the hypervisor options. diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 86033aa73..a07a7df73 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -85,19 +85,29 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece if err != nil { return err } - vfio := &config.VFIODev{ - ID: utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize), - Type: vfioDeviceType, - BDF: deviceBDF, - SysfsDev: deviceSysfsDev, - IsPCIe: isPCIeDevice(deviceBDF), - Class: getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass), - } - device.VfioDevs = append(device.VfioDevs, vfio) - if vfio.IsPCIe { - vfio.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) - AllPCIeDevs[vfio.BDF] = true + id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize) + + var vfio config.VFIODev + + switch vfioDeviceType { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + isPCIe := isPCIeDevice(deviceBDF) + // Do not directly assign to `vfio` -- need to access field still + vfioPCI := config.VFIOPCIDev{ + ID: id, + Type: vfioDeviceType, + BDF: deviceBDF, + SysfsDev: deviceSysfsDev, + IsPCIe: isPCIe, + Class: getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass), + } + if isPCIe { + vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) + AllPCIeDevs[deviceBDF] = true + } + vfio = vfioPCI } + device.VfioDevs = append(device.VfioDevs, &vfio) } coldPlug := device.DeviceInfo.ColdPlug @@ -180,7 +190,16 @@ func (device *VFIODevice) Save() config.DeviceState { devs := device.VfioDevs for _, dev := range devs { if dev != nil { - ds.VFIODevs = append(ds.VFIODevs, dev) + bdf := "" + if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDev.BDF + } + ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{ + ID: *(*dev).GetID(), + Type: uint32((*dev).GetType()), + BDF: bdf, + SysfsDev: *(*dev).GetSysfsDev(), + }) } } return ds @@ -192,12 +211,13 @@ func (device *VFIODevice) Load(ds config.DeviceState) { device.GenericDevice.Load(ds) for _, dev := range ds.VFIODevs { - device.VfioDevs = append(device.VfioDevs, &config.VFIODev{ + var vfioDev config.VFIODev = config.VFIOPCIDev{ ID: dev.ID, Type: config.VFIODeviceType(dev.Type), BDF: dev.BDF, SysfsDev: dev.SysfsDev, - }) + } + device.VfioDevs = append(device.VfioDevs, &vfioDev) } } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 71bd931dc..d9be15082 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -856,6 +856,11 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device) + } + // Create the clh device config via the constructor to ensure default values are properly assigned clhDevice := *chclient.NewDeviceConfig(device.SysfsDev) pciInfo, _, err := cl.VmAddDevicePut(ctx, clhDevice) @@ -879,7 +884,9 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { return fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } - device.GuestPciPath, err = types.PciPathFromString(tokens[0]) + guestPciPath, err := vcTypes.PciPathFromString(tokens[0]) + pciDevice.GuestPciPath = guestPciPath + *device = pciDevice return err } @@ -923,7 +930,7 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int case BlockDev: deviceID = clhDriveIndexToID(devInfo.(*config.BlockDrive).Index) case VfioDev: - deviceID = devInfo.(*config.VFIODev).ID + deviceID = *devInfo.(config.VFIODev).GetID() default: clh.Logger().WithFields(log.Fields{"devInfo": devInfo, "deviceType": devType}).Error("HotplugRemoveDevice: unsupported device") diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 20e6935d8..a1c3c4b64 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -624,7 +624,7 @@ func TestCloudHypervisorHotplugRemoveDevice(t *testing.T) { _, err = clh.HotplugRemoveDevice(context.Background(), &config.BlockDrive{}, BlockDev) assert.NoError(err, "Hotplug remove block device expected no error") - _, err = clh.HotplugRemoveDevice(context.Background(), &config.VFIODev{}, VfioDev) + _, err = clh.HotplugRemoveDevice(context.Background(), &config.VFIOPCIDev{}, VfioDev) assert.NoError(err, "Hotplug remove vfio block device expected no error") _, err = clh.HotplugRemoveDevice(context.Background(), nil, NetDev) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 98411aa45..60010a14a 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1141,8 +1141,10 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * kataDevice.Type = kataVfioPciGuestKernelDevType } - for i, pciDev := range devList { - kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDev.BDF, pciDev.GuestPciPath) + kataDevice.Options = make([]string, len(devList)) + for i, device := range devList { + pciDevice := (*device).(config.VFIOPCIDev) + kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath) } return kataDevice diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 17b3edbe7..6354b9d8a 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1713,7 +1713,7 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op return err } - devID := device.ID + devID := *(*device).GetID() machineType := q.HypervisorConfig().HypervisorMachineType if op == AddDevice { @@ -1730,29 +1730,29 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. if q.state.HotplugVFIOOnRootBus { - - // In case MachineType is q35, a PCIe device is hotplugged on a PCIe Root Port. - switch machineType { - case QemuQ35: - if device.IsPCIe && q.state.PCIeRootPort <= 0 { - q.Logger().WithField("dev-id", device.ID).Warn("VFIO device is a PCIe device. It's recommended to add the PCIe Root Port by setting the pcie_root_port parameter in the configuration for q35") - device.Bus = "" + switch (*device).GetType() { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + // In case MachineType is q35, a PCIe device is hotplugged on a PCIe Root Port. + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device) } - default: - device.Bus = "" - } + switch machineType { + case QemuQ35: + if pciDevice.IsPCIe && q.state.PCIeRootPort <= 0 { + q.Logger().WithField("dev-id", (*device).GetID()).Warn("VFIO device is a PCIe device. It's recommended to add the PCIe Root Port by setting the pcie_root_port parameter in the configuration for q35") + pciDevice.Bus = "" + } + default: + pciDevice.Bus = "" + } + *device = pciDevice - switch device.Type { - case config.VFIOPCIDeviceNormalType: - err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, device.Bus, romFile) - case config.VFIOPCIDeviceMediatedType: - if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { - err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) + if pciDevice.Type == config.VFIOPCIDeviceNormalType { + err = q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, pciDevice.Bus, romFile) } else { - err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", device.Bus, romFile) + err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), "", pciDevice.Bus, romFile) } - default: - return fmt.Errorf("Incorrect VFIO device type found") } } else { addr, bridge, err := q.arch.addDeviceToBridge(ctx, devID, types.PCI) @@ -1766,15 +1766,15 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op } }() - switch device.Type { + switch (*device).GetType() { case config.VFIOPCIDeviceNormalType: - err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile) - case config.VFIOPCIDeviceMediatedType: - if utils.IsAPVFIOMediatedDevice(device.SysfsDev) { - err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, device.SysfsDev) - } else { - err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile) + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device) } + err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, addr, bridge.ID, romFile) + case config.VFIOPCIDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), addr, bridge.ID, romFile) default: return fmt.Errorf("Incorrect VFIO device type found") } @@ -1782,13 +1782,24 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op if err != nil { return err } - // XXX: Depending on whether we're doing root port or - // bridge hotplug, and how the bridge is set up in - // other parts of the code, we may or may not already - // have information about the slot number of the - // bridge and or the device. For simplicity, just - // query both of them back from qemu - device.GuestPciPath, err = q.qomGetPciPath(devID) + + switch (*device).GetType() { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, but its Type said otherwise", device) + } + // XXX: Depending on whether we're doing root port or + // bridge hotplug, and how the bridge is set up in + // other parts of the code, we may or may not already + // have information about the slot number of the + // bridge and or the device. For simplicity, just + // query both of them back from qemu + guestPciPath, err := q.qomGetPciPath(devID) + pciDevice.GuestPciPath = guestPciPath + *device = pciDevice + return err + } return err } else { q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device") diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 5a03b659f..9aff1e76c 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -675,16 +675,17 @@ func (q *qemuArchBase) appendVhostUserDevice(ctx context.Context, devices []govm } func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDev config.VFIODev) []govmmQemu.Device { - if vfioDev.BDF == "" { + pciDevice := vfioDev.(config.VFIOPCIDev) + if pciDevice.BDF == "" { return devices } devices = append(devices, govmmQemu.VFIODevice{ - BDF: vfioDev.BDF, - VendorID: vfioDev.VendorID, - DeviceID: vfioDev.DeviceID, - Bus: vfioDev.Bus, + BDF: pciDevice.BDF, + VendorID: pciDevice.VendorID, + DeviceID: pciDevice.DeviceID, + Bus: pciDevice.Bus, }, ) diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 37611bb5b..51c11bd91 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -463,7 +463,7 @@ func TestQemuArchBaseAppendVFIODevice(t *testing.T) { }, } - vfDevice := config.VFIODev{ + vfDevice := config.VFIOPCIDev{ BDF: bdf, } @@ -483,7 +483,7 @@ func TestQemuArchBaseAppendVFIODeviceWithVendorDeviceID(t *testing.T) { }, } - vfDevice := config.VFIODev{ + vfDevice := config.VFIOPCIDev{ BDF: bdf, VendorID: vendorID, DeviceID: deviceID, diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index dcc93048f..e7c52fcc2 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -1856,11 +1856,15 @@ func (s *Sandbox) HotplugAddDevice(ctx context.Context, device api.Device, devTy // adding a group of VFIO devices for _, dev := range vfioDevices { if _, err := s.hypervisor.HotplugAddDevice(ctx, dev, VfioDev); err != nil { + bdf := "" + if pciDevice, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDevice.BDF + } s.Logger(). WithFields(logrus.Fields{ "sandbox": s.id, - "vfio-device-ID": dev.ID, - "vfio-device-BDF": dev.BDF, + "vfio-device-ID": (*dev).GetID(), + "vfio-device-BDF": bdf, }).WithError(err).Error("failed to hotplug VFIO device") return err } @@ -1909,11 +1913,15 @@ func (s *Sandbox) HotplugRemoveDevice(ctx context.Context, device api.Device, de // remove a group of VFIO devices for _, dev := range vfioDevices { if _, err := s.hypervisor.HotplugRemoveDevice(ctx, dev, VfioDev); err != nil { + bdf := "" + if pciDevice, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDevice.BDF + } s.Logger().WithError(err). WithFields(logrus.Fields{ "sandbox": s.id, - "vfio-device-ID": dev.ID, - "vfio-device-BDF": dev.BDF, + "vfio-device-ID": (*dev).GetID(), + "vfio-device-BDF": bdf, }).Error("failed to hot unplug VFIO device") return err } diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go index 40c5c360e..0b46d2572 100644 --- a/src/runtime/virtcontainers/utils/utils_linux.go +++ b/src/runtime/virtcontainers/utils/utils_linux.go @@ -89,8 +89,7 @@ func FindContextID() (*os.File, uint64, error) { const ( procMountsFile = "/proc/mounts" - fieldsPerLine = 6 - vfioAPSysfsDir = "vfio_ap" + fieldsPerLine = 6 ) const ( diff --git a/src/runtime/virtcontainers/utils/utils_linux_test.go b/src/runtime/virtcontainers/utils/utils_linux_test.go index dbf9fde38..c8e213c2c 100644 --- a/src/runtime/virtcontainers/utils/utils_linux_test.go +++ b/src/runtime/virtcontainers/utils/utils_linux_test.go @@ -63,19 +63,3 @@ func TestGetDevicePathAndFsTypeOptionsSuccessful(t *testing.T) { assert.Equal(fstype, fstypeOut) assert.Equal(fsOptions, optsOut) } - -func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) { - assert := assert.New(t) - - // Should be false for a PCI device - isAPMdev := IsAPVFIOMediatedDevice("/sys/bus/pci/devices/0000:00:02.0/a297db4a-f4c2-11e6-90f6-d3b88d6c9525") - assert.False(isAPMdev) -} - -func TestIsAPVFIOMediatedDeviceTrue(t *testing.T) { - assert := assert.New(t) - - // Typical AP sysfsdev - isAPMdev := IsAPVFIOMediatedDevice("/sys/devices/vfio_ap/matrix/a297db4a-f4c2-11e6-90f6-d3b88d6c9525") - assert.True(isAPMdev) -} From f666f8e2df6bfee78c34d1bf449865bd9c1f4b8f Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 8 Mar 2022 18:51:08 +0100 Subject: [PATCH 18/40] agent: Add VFIO-AP device handling Initial VFIO-AP support (#578) was simple, but somewhat hacky; a different code path would be chosen for performing the hotplug, and agent-side device handling was bound to knowing the assigned queue numbers (APQNs) through some other means; plus the code for awaiting them was written for the Go agent and never released. This code also artificially increased the hotplug timeout to wait for the (relatively expensive, thus limited to 5 seconds at the quickest) AP rescan, which is impractical for e.g. common k8s timeouts. Since then, the general handling logic was improved (#1190), but it assumed PCI in several places. In the runtime, introduce and parse AP devices. Annotate them as such when passing to the agent, and include information about the associated APQNs. The agent awaits the passed APQNs through uevents and triggers a rescan directly. Fixes: #3678 Signed-off-by: Jakob Naucke --- src/agent/src/ap.rs | 79 ++++++++++++++++++ src/agent/src/device.rs | 89 +++++++++++++++++++++ src/agent/src/linux_abi.rs | 2 + src/agent/src/main.rs | 1 + src/runtime/pkg/device/config/config.go | 29 +++++++ src/runtime/pkg/device/drivers/utils.go | 49 +++++++++--- src/runtime/pkg/device/drivers/vfio.go | 67 ++++++++++------ src/runtime/pkg/device/drivers/vfio_test.go | 2 +- src/runtime/virtcontainers/clh.go | 4 +- src/runtime/virtcontainers/kata_agent.go | 83 ++++++++++--------- src/runtime/virtcontainers/qemu.go | 4 + 11 files changed, 335 insertions(+), 74 deletions(-) create mode 100644 src/agent/src/ap.rs diff --git a/src/agent/src/ap.rs b/src/agent/src/ap.rs new file mode 100644 index 000000000..454ada9ff --- /dev/null +++ b/src/agent/src/ap.rs @@ -0,0 +1,79 @@ +// Copyright (c) IBM Corp. 2022 +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fmt; +use std::str::FromStr; + +use anyhow::{anyhow, Context}; + +// IBM Adjunct Processor (AP) is the bus used by IBM Crypto Express hardware security modules on +// IBM Z & LinuxONE (s390x) +// AP bus ID follow the format . [1, p. 476], where +// - is the adapter ID, i.e. the card and +// - is the adapter domain. +// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf + +#[derive(Debug)] +pub struct Address { + pub adapter_id: u8, + pub adapter_domain: u16, +} + +impl Address { + pub fn new(adapter_id: u8, adapter_domain: u16) -> Address { + Address { + adapter_id, + adapter_domain, + } + } +} + +impl FromStr for Address { + type Err = anyhow::Error; + + fn from_str(s: &str) -> anyhow::Result { + let split: Vec<&str> = s.split('.').collect(); + if split.len() != 2 { + return Err(anyhow!( + "Wrong AP bus format. It needs to be in the form ., got {:?}", + s + )); + } + + let adapter_id = u8::from_str_radix(split[0], 16).context(format!( + "Wrong AP bus format. AP ID needs to be in the form , got {:?}", + split[0] + ))?; + let adapter_domain = u16::from_str_radix(split[1], 16).context(format!( + "Wrong AP bus format. AP domain needs to be in the form , got {:?}", + split[1] + ))?; + + Ok(Address::new(adapter_id, adapter_domain)) + } +} + +impl fmt::Display for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_str() { + let device = Address::from_str("a.1").unwrap(); + assert_eq!(format!("{}", device), "0a.0001"); + + assert!(Address::from_str("").is_err()); + assert!(Address::from_str(".").is_err()); + assert!(Address::from_str("0.0.0").is_err()); + assert!(Address::from_str("0g.0000").is_err()); + assert!(Address::from_str("0a.10000").is_err()); + } +} diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 23bc154a7..b95e304fb 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -50,11 +50,13 @@ pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk"; // VFIO PCI device to be bound to vfio-pci and made available inside the // container as a VFIO device node pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci"; +pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap"; pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs"; pub const FS_TYPE_HUGETLB: &str = "hugetlbfs"; cfg_if! { if #[cfg(target_arch = "s390x")] { + use crate::ap; use crate::ccw; } } @@ -406,6 +408,39 @@ async fn get_vfio_device_name(sandbox: &Arc>, grp: IommuGroup) -> Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname)) } +#[cfg(target_arch = "s390x")] +#[derive(Debug)] +struct ApMatcher { + syspath: String, +} + +#[cfg(target_arch = "s390x")] +impl ApMatcher { + fn new(address: ap::Address) -> ApMatcher { + ApMatcher { + syspath: format!( + "{}/card{:02x}/{}", + AP_ROOT_BUS_PATH, address.adapter_id, address + ), + } + } +} + +#[cfg(target_arch = "s390x")] +impl UeventMatcher for ApMatcher { + fn is_match(&self, uev: &Uevent) -> bool { + uev.action == "add" && uev.devpath == self.syspath + } +} + +#[cfg(target_arch = "s390x")] +#[instrument] +async fn wait_for_ap_device(sandbox: &Arc>, address: ap::Address) -> Result<()> { + let matcher = ApMatcher::new(address); + wait_for_uevent(sandbox, matcher).await?; + Ok(()) +} + /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN) #[instrument] fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { @@ -772,6 +807,28 @@ async fn vfio_pci_device_handler( }) } +// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options +// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that +// amoutn to the hotplug timeout. +#[cfg(target_arch = "s390x")] +#[instrument] +async fn vfio_ap_device_handler( + device: &Device, + sandbox: &Arc>, +) -> Result { + // Force AP bus rescan + fs::write(AP_SCANS_PATH, "1")?; + for apqn in device.options.iter() { + wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?; + } + Ok(Default::default()) +} + +#[cfg(not(target_arch = "s390x"))] +async fn vfio_ap_device_handler(_: &Device, _: &Arc>) -> Result { + Err(anyhow!("AP is only supported on s390x")) +} + #[instrument] pub async fn add_devices( devices: &[Device], @@ -840,6 +897,7 @@ async fn add_device(device: &Device, sandbox: &Arc>) -> Result { vfio_pci_device_handler(device, sandbox).await } + DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await, _ => Err(anyhow!("Unknown device type {}", device.field_type)), } } @@ -1583,4 +1641,35 @@ mod tests { // Test dev2 assert!(pci_iommu_group(&syspci, dev2).is_err()); } + + #[cfg(target_arch = "s390x")] + #[tokio::test] + async fn test_vfio_ap_matcher() { + let subsystem = "ap"; + let card = "0a"; + let relpath = format!("{}.0001", card); + + let mut uev = Uevent::default(); + uev.action = U_EVENT_ACTION_ADD.to_string(); + uev.subsystem = subsystem.to_string(); + uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath); + + let ap_address = ap::Address::from_str(&relpath).unwrap(); + let matcher = ApMatcher::new(ap_address); + + assert!(matcher.is_match(&uev)); + + let mut uev_remove = uev.clone(); + uev_remove.action = U_EVENT_ACTION_REMOVE.to_string(); + assert!(!matcher.is_match(&uev_remove)); + + let mut uev_other_device = uev.clone(); + uev_other_device.devpath = format!( + "{}/card{}/{}", + AP_ROOT_BUS_PATH, + card, + format!("{}.0002", card) + ); + assert!(!matcher.is_match(&uev_other_device)); + } } diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index 9df398d61..042acd0ae 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -69,6 +69,8 @@ pub fn create_pci_root_bus_path() -> String { cfg_if! { if #[cfg(target_arch = "s390x")] { pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0"; + pub const AP_ROOT_BUS_PATH: &str = "/devices/ap"; + pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans"; } } diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 4ca89a9a1..085ea396d 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -75,6 +75,7 @@ mod tracer; cfg_if! { if #[cfg(target_arch = "s390x")] { + mod ap; mod ccw; } } diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index 6bcbed32c..dee6291ed 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -263,6 +263,9 @@ const ( // VFIOPCIDeviceMediatedType is a VFIO PCI mediated device type VFIOPCIDeviceMediatedType + + // VFIOAPDeviceMediatedType is a VFIO AP mediated device type + VFIOAPDeviceMediatedType ) type VFIODev interface { @@ -316,6 +319,32 @@ func (d VFIOPCIDev) GetSysfsDev() *string { return &d.SysfsDev } +type VFIOAPDev struct { + // ID is used to identify this drive in the hypervisor options. + ID string + + // sysfsdev of VFIO mediated device + SysfsDev string + + // APDevices are the Adjunct Processor devices assigned to the mdev + APDevices []string + + // Type of VFIO device + Type VFIODeviceType +} + +func (d VFIOAPDev) GetID() *string { + return &d.ID +} + +func (d VFIOAPDev) GetType() VFIODeviceType { + return d.Type +} + +func (d VFIOAPDev) GetSysfsDev() *string { + return &d.SysfsDev +} + // RNGDev represents a random number generator device type RNGDev struct { // ID is used to identify the device in the hypervisor options. diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index abdf5d816..79e00adbd 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -89,18 +89,47 @@ func readPCIProperty(propertyPath string) (string, error) { return strings.Split(string(buf), "\n")[0], nil } -func GetVFIODeviceType(deviceFileName string) config.VFIODeviceType { +func GetVFIODeviceType(deviceFilePath string) (config.VFIODeviceType, error) { + deviceFileName := filepath.Base(deviceFilePath) + //For example, 0000:04:00.0 tokens := strings.Split(deviceFileName, ":") - vfioDeviceType := config.VFIODeviceErrorType if len(tokens) == 3 { - vfioDeviceType = config.VFIOPCIDeviceNormalType - } else { - //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 - tokens = strings.Split(deviceFileName, "-") - if len(tokens) == 5 { - vfioDeviceType = config.VFIOPCIDeviceMediatedType - } + return config.VFIOPCIDeviceNormalType, nil } - return vfioDeviceType + + //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 + tokens = strings.Split(deviceFileName, "-") + if len(tokens) != 5 { + return config.VFIODeviceErrorType, fmt.Errorf("Incorrect tokens found while parsing VFIO details: %s", deviceFileName) + } + + deviceSysfsDev, err := GetSysfsDev(deviceFilePath) + if err != nil { + return config.VFIODeviceErrorType, err + } + + if strings.HasPrefix(deviceSysfsDev, vfioAPSysfsDir) { + return config.VFIOAPDeviceMediatedType, nil + } + + return config.VFIOPCIDeviceMediatedType, nil +} + +// GetSysfsDev returns the sysfsdev of mediated device +// Expected input string format is absolute path to the sysfs dev node +// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 +func GetSysfsDev(sysfsDevStr string) (string, error) { + return filepath.EvalSymlinks(sysfsDevStr) +} + +// GetAPVFIODevices retrieves all APQNs associated with a mediated VFIO-AP +// device +func GetAPVFIODevices(sysfsdev string) ([]string, error) { + data, err := os.ReadFile(filepath.Join(sysfsdev, "matrix")) + if err != nil { + return []string{}, err + } + // Split by newlines, omitting final newline + return strings.Split(string(data[:len(data)-1]), "\n"), nil } diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index a07a7df73..3450e61d9 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -30,6 +30,7 @@ const ( iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group" vfioDevPath = "/dev/vfio/%s" pcieRootPortPrefix = "rp" + vfioAPSysfsDir = "/sys/devices/vfio_ap" ) var ( @@ -106,6 +107,17 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece AllPCIeDevs[deviceBDF] = true } vfio = vfioPCI + case config.VFIOAPDeviceMediatedType: + devices, err := GetAPVFIODevices(deviceSysfsDev) + if err != nil { + return err + } + vfio = config.VFIOAPDev{ + ID: id, + SysfsDev: deviceSysfsDev, + Type: config.VFIOAPDeviceMediatedType, + APDevices: devices, + } } device.VfioDevs = append(device.VfioDevs, &vfio) } @@ -190,16 +202,7 @@ func (device *VFIODevice) Save() config.DeviceState { devs := device.VfioDevs for _, dev := range devs { if dev != nil { - bdf := "" - if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { - bdf = pciDev.BDF - } - ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{ - ID: *(*dev).GetID(), - Type: uint32((*dev).GetType()), - BDF: bdf, - SysfsDev: *(*dev).GetSysfsDev(), - }) + ds.VFIODevs = append(ds.VFIODevs, dev) } } return ds @@ -211,20 +214,38 @@ func (device *VFIODevice) Load(ds config.DeviceState) { device.GenericDevice.Load(ds) for _, dev := range ds.VFIODevs { - var vfioDev config.VFIODev = config.VFIOPCIDev{ - ID: dev.ID, - Type: config.VFIODeviceType(dev.Type), - BDF: dev.BDF, - SysfsDev: dev.SysfsDev, + var vfio config.VFIODev + + if (*device.VfioDevs[0]).GetType() == config.VFIOAPDeviceMediatedType { + vfio = config.VFIOAPDev{ + ID: *(*dev).GetID(), + SysfsDev: *(*dev).GetSysfsDev(), + } + } else { + bdf := "" + if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDev.BDF + } + vfio = config.VFIOPCIDev{ + ID: *(*dev).GetID(), + Type: config.VFIODeviceType((*dev).GetType()), + BDF: bdf, + SysfsDev: *(*dev).GetSysfsDev(), + } } - device.VfioDevs = append(device.VfioDevs, &vfioDev) + + device.VfioDevs = append(device.VfioDevs, &vfio) } } // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) { - vfioDeviceType = GetVFIODeviceType(deviceFileName) + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + vfioDeviceType, err = GetVFIODeviceType(sysfsDevStr) + if err != nil { + return deviceBDF, deviceSysfsDev, vfioDeviceType, err + } switch vfioDeviceType { case config.VFIOPCIDeviceNormalType: @@ -235,8 +256,11 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS case config.VFIOPCIDeviceMediatedType: // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) - deviceSysfsDev, err = getSysfsDev(sysfsDevStr) + deviceSysfsDev, err = GetSysfsDev(sysfsDevStr) deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev)) + case config.VFIOAPDeviceMediatedType: + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + deviceSysfsDev, err = GetSysfsDev(sysfsDevStr) default: err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName) } @@ -264,13 +288,6 @@ func getBDF(deviceSysStr string) string { return tokens[1] } -// getSysfsDev returns the sysfsdev of mediated device -// Expected input string format is absolute path to the sysfs dev node -// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 -func getSysfsDev(sysfsDevStr string) (string, error) { - return filepath.EvalSymlinks(sysfsDevStr) -} - // BindDevicetoVFIO binds the device to vfio driver after unbinding from host. // Will be called by a network interface or a generic pcie device. func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) { diff --git a/src/runtime/pkg/device/drivers/vfio_test.go b/src/runtime/pkg/device/drivers/vfio_test.go index 8ee008e3b..6a1ab61eb 100644 --- a/src/runtime/pkg/device/drivers/vfio_test.go +++ b/src/runtime/pkg/device/drivers/vfio_test.go @@ -34,7 +34,7 @@ func TestGetVFIODetails(t *testing.T) { switch vfioDeviceType { case config.VFIOPCIDeviceNormalType: assert.Equal(t, d.expectedStr, deviceBDF) - case config.VFIOPCIDeviceMediatedType: + case config.VFIOPCIDeviceMediatedType, config.VFIOAPDeviceMediatedType: assert.Equal(t, d.expectedStr, deviceSysfsDev) default: assert.NotNil(t, err) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index d9be15082..8799d6a93 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -867,7 +867,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { if err != nil { return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } - clh.devicesIds[device.ID] = pciInfo.GetId() + clh.devicesIds[*(*device).GetID()] = pciInfo.GetId() // clh doesn't use bridges, so the PCI path is simply the slot // number of the device. This will break if clh starts using @@ -884,7 +884,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { return fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } - guestPciPath, err := vcTypes.PciPathFromString(tokens[0]) + guestPciPath, err := types.PciPathFromString(tokens[0]) pciDevice.GuestPciPath = guestPciPath *device = pciDevice diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 60010a14a..9e5c8b34f 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -77,38 +77,39 @@ const ( ) var ( - checkRequestTimeout = 30 * time.Second - defaultRequestTimeout = 60 * time.Second - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" - defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" - defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" - mountGuestTag = "kataShared" - defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" - type9pFs = "9p" - typeVirtioFS = "virtiofs" - typeOverlayFS = "overlay" - kata9pDevType = "9p" - kataMmioBlkDevType = "mmioblk" - kataBlkDevType = "blk" - kataBlkCCWDevType = "blk-ccw" - kataSCSIDevType = "scsi" - kataNvdimmDevType = "nvdimm" - kataVirtioFSDevType = "virtio-fs" - kataOverlayDevType = "overlayfs" - kataWatchableBindDevType = "watchable-bind" - kataVfioDevType = "vfio" // VFIO device to used as VFIO in the container - kataVfioGuestKernelDevType = "vfio-gk" // VFIO device for consumption by the guest kernel - sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} - sharedDirVirtioFSOptions = []string{} - sharedDirVirtioFSDaxOptions = "dax" - shmDir = "shm" - kataEphemeralDevType = "ephemeral" - defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) - grpcMaxDataSize = int64(1024 * 1024) - localDirOptions = []string{"mode=0777"} - maxHostnameLen = 64 - GuestDNSFile = "/etc/resolv.conf" + checkRequestTimeout = 30 * time.Second + defaultRequestTimeout = 60 * time.Second + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" + defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" + defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" + mountGuestTag = "kataShared" + defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" + type9pFs = "9p" + typeVirtioFS = "virtiofs" + typeOverlayFS = "overlay" + kata9pDevType = "9p" + kataMmioBlkDevType = "mmioblk" + kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" + kataSCSIDevType = "scsi" + kataNvdimmDevType = "nvdimm" + kataVirtioFSDevType = "virtio-fs" + kataOverlayDevType = "overlayfs" + kataWatchableBindDevType = "watchable-bind" + kataVfioPciDevType = "vfio-pci" // VFIO PCI device to used as VFIO in the container + kataVfioPciGuestKernelDevType = "vfio-pci-gk" // VFIO PCI device for consumption by the guest kernel + kataVfioApDevType = "vfio-ap" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} + sharedDirVirtioFSOptions = []string{} + sharedDirVirtioFSDaxOptions = "dax" + shmDir = "shm" + kataEphemeralDevType = "ephemeral" + defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) + grpcMaxDataSize = int64(1024 * 1024) + localDirOptions = []string{"mode=0777"} + maxHostnameLen = 64 + GuestDNSFile = "/etc/resolv.conf" ) const ( @@ -1126,6 +1127,11 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * // DDDD:BB:DD.F is the device's PCI address on the // *host*. is the device's PCI path in the guest // (see qomGetPciPath() for details). + // + // For VFIO-AP, one VFIO group could include several queue devices. They are + // identified by APQNs (Adjunct Processor Queue Numbers), which do not differ + // between host and guest. They are passed as options so they can be awaited + // by the agent. kataDevice := &grpc.Device{ ContainerPath: dev.ContainerPath, Type: kataVfioPciDevType, @@ -1141,10 +1147,15 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * kataDevice.Type = kataVfioPciGuestKernelDevType } - kataDevice.Options = make([]string, len(devList)) - for i, device := range devList { - pciDevice := (*device).(config.VFIOPCIDev) - kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath) + if (*devList[0]).GetType() == config.VFIOAPDeviceMediatedType { + kataDevice.Type = kataVfioApDevType + kataDevice.Options = (*devList[0]).(config.VFIOAPDev).APDevices + } else { + kataDevice.Options = make([]string, len(devList)) + for i, device := range devList { + pciDevice := (*device).(config.VFIOPCIDev) + kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath) + } } return kataDevice diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 6354b9d8a..16872c126 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1753,6 +1753,8 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op } else { err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), "", pciDevice.Bus, romFile) } + case config.VFIOAPDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev()) } } else { addr, bridge, err := q.arch.addDeviceToBridge(ctx, devID, types.PCI) @@ -1775,6 +1777,8 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, addr, bridge.ID, romFile) case config.VFIOPCIDeviceMediatedType: err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), addr, bridge.ID, romFile) + case config.VFIOAPDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev()) default: return fmt.Errorf("Incorrect VFIO device type found") } From 96baa838952594c506805c7cc809d7645cf84f18 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Mon, 4 Jul 2022 17:41:00 +0200 Subject: [PATCH 19/40] agent: Bring in VFIO-AP device handling again This PR is a continuing work for (kata-containers#3679). This generalizes the previous VFIO device handling which only focuses on PCI to include AP (IBM Z specific). Fixes: kata-containers#3678 Signed-off-by: Hyounggyu Choi --- src/agent/src/ap.rs | 20 +++++++++--------- src/agent/src/device.rs | 6 +++--- src/runtime/pkg/device/drivers/vfio.go | 21 +++++++++++++------ src/runtime/virtcontainers/clh.go | 12 +++++------ .../virtcontainers/utils/utils_linux.go | 12 ----------- 5 files changed, 34 insertions(+), 37 deletions(-) diff --git a/src/agent/src/ap.rs b/src/agent/src/ap.rs index 454ada9ff..202d33064 100644 --- a/src/agent/src/ap.rs +++ b/src/agent/src/ap.rs @@ -1,18 +1,18 @@ -// Copyright (c) IBM Corp. 2022 +// Copyright (c) IBM Corp. 2023 // // SPDX-License-Identifier: Apache-2.0 // - use std::fmt; use std::str::FromStr; use anyhow::{anyhow, Context}; -// IBM Adjunct Processor (AP) is the bus used by IBM Crypto Express hardware security modules on -// IBM Z & LinuxONE (s390x) -// AP bus ID follow the format . [1, p. 476], where -// - is the adapter ID, i.e. the card and -// - is the adapter domain. +// IBM Adjunct Processor (AP) is used for cryptographic operations +// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x). +// In Linux, virtual cryptographic devices are called AP queues. +// The name of an AP queue respects a format . in hexadecimal notation [1, p.467]: +// - is an adapter ID +// - is an adapter domain ID // [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf #[derive(Debug)] @@ -37,17 +37,17 @@ impl FromStr for Address { let split: Vec<&str> = s.split('.').collect(); if split.len() != 2 { return Err(anyhow!( - "Wrong AP bus format. It needs to be in the form ., got {:?}", + "Wrong AP bus format. It needs to be in the form . (e.g. 0a.003f), got {:?}", s )); } let adapter_id = u8::from_str_radix(split[0], 16).context(format!( - "Wrong AP bus format. AP ID needs to be in the form , got {:?}", + "Wrong AP bus format. AP ID needs to be in the form (e.g. 0a), got {:?}", split[0] ))?; let adapter_domain = u16::from_str_radix(split[1], 16).context(format!( - "Wrong AP bus format. AP domain needs to be in the form , got {:?}", + "Wrong AP bus format. AP domain needs to be in the form (e.g. 003f), got {:?}", split[1] ))?; diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index b95e304fb..535a729f6 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -764,8 +764,8 @@ async fn vfio_pci_device_handler( let mut group = None; for opt in device.options.iter() { - let (host, pcipath) = - split_vfio_pci_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?; + let (host, pcipath) = split_vfio_pci_option(opt) + .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?; let host = pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?; let pcipath = pci::Path::from_str(pcipath)?; @@ -809,7 +809,7 @@ async fn vfio_pci_device_handler( // The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options // and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that -// amoutn to the hotplug timeout. +// amount to the hotplug timeout. #[cfg(target_arch = "s390x")] #[instrument] async fn vfio_ap_device_handler( diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 3450e61d9..94139aaa2 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -118,6 +118,8 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece Type: config.VFIOAPDeviceMediatedType, APDevices: devices, } + default: + return fmt.Errorf("Failed to append device: VFIO device type unrecognized") } device.VfioDevs = append(device.VfioDevs, &vfio) } @@ -216,12 +218,9 @@ func (device *VFIODevice) Load(ds config.DeviceState) { for _, dev := range ds.VFIODevs { var vfio config.VFIODev - if (*device.VfioDevs[0]).GetType() == config.VFIOAPDeviceMediatedType { - vfio = config.VFIOAPDev{ - ID: *(*dev).GetID(), - SysfsDev: *(*dev).GetSysfsDev(), - } - } else { + vfioDeviceType := (*device.VfioDevs[0]).GetType() + switch vfioDeviceType { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: bdf := "" if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { bdf = pciDev.BDF @@ -232,6 +231,16 @@ func (device *VFIODevice) Load(ds config.DeviceState) { BDF: bdf, SysfsDev: *(*dev).GetSysfsDev(), } + case config.VFIOAPDeviceMediatedType: + vfio = config.VFIOAPDev{ + ID: *(*dev).GetID(), + SysfsDev: *(*dev).GetSysfsDev(), + } + default: + deviceLogger().WithError( + fmt.Errorf("VFIO device type unrecognized"), + ).Error("Failed to append device") + return } device.VfioDevs = append(device.VfioDevs, &vfio) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 8799d6a93..1f493772d 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -856,13 +856,8 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - pciDevice, ok := (*device).(config.VFIOPCIDev) - if !ok { - return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device) - } - // Create the clh device config via the constructor to ensure default values are properly assigned - clhDevice := *chclient.NewDeviceConfig(device.SysfsDev) + clhDevice := *chclient.NewDeviceConfig(*(*device).GetSysfsDev()) pciInfo, _, err := cl.VmAddDevicePut(ctx, clhDevice) if err != nil { return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) @@ -885,6 +880,11 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { } guestPciPath, err := types.PciPathFromString(tokens[0]) + + pciDevice, ok := (*device).(config.VFIOPCIDev) + if !ok { + return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device) + } pciDevice.GuestPciPath = guestPciPath *device = pciDevice diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go index 0b46d2572..a31b8d351 100644 --- a/src/runtime/virtcontainers/utils/utils_linux.go +++ b/src/runtime/virtcontainers/utils/utils_linux.go @@ -141,18 +141,6 @@ func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string } } -// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device -// by checking for the existence of "vfio_ap" in the path -func IsAPVFIOMediatedDevice(sysfsdev string) bool { - split := strings.Split(sysfsdev, string(os.PathSeparator)) - for _, el := range split { - if el == vfioAPSysfsDir { - return true - } - } - return false -} - func waitProcessUsingPidfd(pid int, timeoutSecs uint, logger *logrus.Entry) (bool, error) { pidfd, err := unix.PidfdOpen(pid, 0) From f4938c0d90a1d87ec3692772baf078d36fddd071 Mon Sep 17 00:00:00 2001 From: Yushuo Date: Thu, 9 Mar 2023 23:17:43 +0800 Subject: [PATCH 20/40] bugfix: set hostname Setting hostname according to the spec. Fixes: #6247 Signed-off-by: Yushuo --- src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 129bdd7fa..f996c5747 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -216,7 +216,7 @@ impl Sandbox for VirtSandbox { let agent_config = self.agent.agent_config().await; let kernel_modules = KernelModule::set_kernel_modules(agent_config.kernel_modules)?; let req = agent::CreateSandboxRequest { - hostname: "".to_string(), + hostname: spec.hostname.clone(), dns, storages: self .resource_manager From 99505c0f4f3a67fd657566b65aefd9a7e12b704d Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 16 Mar 2023 16:12:26 +0000 Subject: [PATCH 21/40] versions: Update firecracker version This PR updates the firecracker version being used in kata containers versions.yaml The changes in version 1.3.1 are Added Introduced T2CL (Intel) and T2A (AMD) CPU templates to provide instruction set feature parity between Intel and AMD CPUs when using these templates. Added Graviton3 support (c7g instance type). Changed Improved error message when invalid network backend provided. Improved TCP throughput by between 5% and 15% (depending on CPU) by using scatter-gather I/O in the net device's TX path. Upgraded Rust toolchain from 1.64.0 to 1.66.0. Made seccompiler output bit-reproducible. Fixed Fixed feature flags in T2 CPU template on Intel Ice Lake. Fixes #6482 Signed-off-by: Gabriela Cervantes --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 174162b99..b68bf8fbd 100644 --- a/versions.yaml +++ b/versions.yaml @@ -83,7 +83,7 @@ assets: uscan-url: >- https://github.com/firecracker-microvm/firecracker/tags .*/v?(\d\S+)\.tar\.gz - version: "v1.1.0" + version: "v1.3.1" qemu: description: "VMM that uses KVM" From 08fe49f708e54a2a0e0cc5b90616bfaeb99a1156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 12:09:34 +0100 Subject: [PATCH 22/40] versions: Adjust kernel names to match kata-deploy build targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's adjust the kernel names in versions.yaml so those can match the names used as part of the kata-deploy local build scripts. Right now this doesn't bring any benefit nor drawback, but it'll make our life easier later on in this same series. Depends-on: github.com/kata-containers/tests#5534 Signed-off-by: Fabiano Fidêncio --- .../packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 2 +- tools/packaging/kernel/build-kernel.sh | 4 ++-- versions.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index ded375cb6..79cf5a141 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -108,7 +108,7 @@ install_kernel() { #Install dragonball experimental kernel asset install_dragonball_experimental_kernel() { info "build dragonball experimental kernel" - export kernel_version="$(yq r $versions_yaml assets.dragonball-kernel-experimental.version)" + export kernel_version="$(yq r $versions_yaml assets.kernel-dragonball-experimental.version)" info "kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -e -t dragonball -v ${kernel_version} } diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 014e08359..af7e36e6b 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -556,7 +556,7 @@ main() { case "${arch_target}" in "aarch64") build_type="arm-experimental" - kernel_version=$(get_from_kata_deps "assets.arm-kernel-experimental.version") + kernel_version=$(get_from_kata_deps "assets.kernel-arm-experimental.version") ;; *) info "No arch-specific experimental kernel supported, using experimental one instead" @@ -564,7 +564,7 @@ main() { ;; esac elif [[ ${build_type} == "dragonball-experimental" ]]; then - kernel_version=$(get_from_kata_deps "assets.dragonball-kernel-experimental.version") + kernel_version=$(get_from_kata_deps "assets.kernel-dragonball-experimental.version") elif [[ "${conf_guest}" != "" ]]; then #If specifying a tag for kernel_version, must be formatted version-like to avoid unintended parsing issues kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) diff --git a/versions.yaml b/versions.yaml index b68bf8fbd..817ff1202 100644 --- a/versions.yaml +++ b/versions.yaml @@ -177,12 +177,12 @@ assets: url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" tag: "v5.13.10" - arm-kernel-experimental: + kernel-arm-experimental: description: "Linux kernel with cpu/mem hotplug support on arm64" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" version: "v5.15.7" - dragonball-kernel-experimental: + kernel-dragonball-experimental: description: "Linux kernel with Dragonball VMM optimizations like upcall" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" version: "v5.10.25" From 6b1b424fc733fd34496c322de2bb1d6acb0ce820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 12:36:21 +0100 Subject: [PATCH 23/40] tools: Add support for caching Cloud Hypervisor artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching Cloud Hypervisor artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../static-build/cache_components_main.sh | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 tools/packaging/static-build/cache_components_main.sh diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh new file mode 100755 index 000000000..b6780e935 --- /dev/null +++ b/tools/packaging/static-build/cache_components_main.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${script_dir}/../scripts/lib.sh" + +cache_clh_artifacts() { + local clh_tarball_name="kata-static-cloud-hypervisor.tar.xz" + local current_clh_version="$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" + create_cache_asset "${clh_tarball_name}" "${current_clh_version}" "" +} + +create_cache_asset() { + local component_name="${1}" + local component_version="${2}" + local component_image="${3}" + + sudo cp "${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/${component_name}" . + sudo chown -R "${USER}:${USER}" . + sha256sum "${component_name}" > "sha256sum-${component_name}" + cat "sha256sum-${component_name}" + echo "${component_version}" > "latest" + cat "latest" + echo "${component_image}" > "latest_image" + cat "latest_image" +} + +help() { +echo "$(cat << EOF +Usage: $0 "[options]" + Description: + Builds the cache of several kata components. + Options: + -c Cloud hypervisor cache + -h Shows help +EOF +)" +} + +main() { + local cloud_hypervisor_component="${cloud_hypervisor_component:-}" + local OPTIND + while getopts ":ch:" opt + do + case "$opt" in + c) + cloud_hypervisor_component="1" + ;; + h) + help + exit 0; + ;; + :) + echo "Missing argument for -$OPTARG"; + help + exit 1; + ;; + esac + done + shift $((OPTIND-1)) + + [[ -z "${cloud_hypervisor_component}" ]] && \ + help && die "Must choose at least one option" + + mkdir -p "${WORKSPACE}/artifacts" + pushd "${WORKSPACE}/artifacts" + echo "Artifacts:" + + [ "${cloud_hypervisor_component}" == "1" ] && cache_clh_artifacts + + ls -la "${WORKSPACE}/artifacts/" + popd + sync +} + +main "$@" From 762f9f4c3edf44d18e8f979afc2e46cf72b29b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 12:39:38 +0100 Subject: [PATCH 24/40] tools: Add support for caching Firecracker artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching Firecracker artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../static-build/cache_components_main.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index b6780e935..4ba3d16e0 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -18,6 +18,12 @@ cache_clh_artifacts() { create_cache_asset "${clh_tarball_name}" "${current_clh_version}" "" } +cache_firecracker_artifacts() { + local fc_tarball_name="kata-static-firecracker.tar.xz" + local current_fc_version="$(get_from_kata_deps "assets.hypervisor.firecracker.version")" + create_cache_asset "${fc_tarball_name}" "${current_fc_version}" "" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -40,6 +46,7 @@ Usage: $0 "[options]" Builds the cache of several kata components. Options: -c Cloud hypervisor cache + -F Firecracker cache -h Shows help EOF )" @@ -47,13 +54,17 @@ EOF main() { local cloud_hypervisor_component="${cloud_hypervisor_component:-}" + local firecracker_component="${firecracker_component:-}" local OPTIND - while getopts ":ch:" opt + while getopts ":cFh:" opt do case "$opt" in c) cloud_hypervisor_component="1" ;; + F) + firecracker_component="1" + ;; h) help exit 0; @@ -68,6 +79,7 @@ main() { shift $((OPTIND-1)) [[ -z "${cloud_hypervisor_component}" ]] && \ + [[ -z "${firecracker_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -75,6 +87,7 @@ main() { echo "Artifacts:" [ "${cloud_hypervisor_component}" == "1" ] && cache_clh_artifacts + [ "${firecracker_component}" == "1" ] && cache_firecracker_artifacts ls -la "${WORKSPACE}/artifacts/" popd From cb4cbe29580f9e44b7f70ca9f5c7ceadb448dc4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 12:43:21 +0100 Subject: [PATCH 25/40] tools: Add support for caching Kernel artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching Kernel artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- tools/packaging/kernel/kata_config_version | 2 +- tools/packaging/scripts/lib.sh | 5 +++++ .../static-build/cache_components_main.sh | 21 ++++++++++++++++++- tools/packaging/static-build/kernel/build.sh | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index 398050c62..257e56326 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -101 +102 diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 10c535172..deece4ede 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -129,3 +129,8 @@ push_to_registry() { fi fi } + +get_kernel_image_name() { + kernel_script_dir="${repo_root_dir}/tools/packaging/static-build/kernel" + echo "${BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)" +} diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index 4ba3d16e0..3f620887c 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -12,6 +12,8 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../scripts/lib.sh" +KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-experimental | kernel-arm-experimetnal | kernel-dragonball-experimental + cache_clh_artifacts() { local clh_tarball_name="kata-static-cloud-hypervisor.tar.xz" local current_clh_version="$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" @@ -24,6 +26,14 @@ cache_firecracker_artifacts() { create_cache_asset "${fc_tarball_name}" "${current_fc_version}" "" } +cache_kernel_artifacts() { + local kernel_tarball_name="kata-static-${KERNEL_FLAVOUR}.tar.xz" + local current_kernel_image="$(get_kernel_image_name)" + local current_kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")-${current_kernel_kata_config_version}" + create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -47,6 +57,9 @@ Usage: $0 "[options]" Options: -c Cloud hypervisor cache -F Firecracker cache + -k Kernel cache + * Export KERNEL_FLAVOUR="kernel|kernek-experimental|kernel-arm-experimental|kernel-dragonball-experimental" for a specific build + The default KERNEL_FLAVOUR value is "kernel" -h Shows help EOF )" @@ -55,8 +68,9 @@ EOF main() { local cloud_hypervisor_component="${cloud_hypervisor_component:-}" local firecracker_component="${firecracker_component:-}" + local kernel_component="${kernel_component:-}" local OPTIND - while getopts ":cFh:" opt + while getopts ":cFkh:" opt do case "$opt" in c) @@ -65,6 +79,9 @@ main() { F) firecracker_component="1" ;; + k) + kernel_component="1" + ;; h) help exit 0; @@ -80,6 +97,7 @@ main() { [[ -z "${cloud_hypervisor_component}" ]] && \ [[ -z "${firecracker_component}" ]] && \ + [[ -z "${kernel_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -88,6 +106,7 @@ main() { [ "${cloud_hypervisor_component}" == "1" ] && cache_clh_artifacts [ "${firecracker_component}" == "1" ] && cache_firecracker_artifacts + [ "${kernel_component}" == "1" ] && cache_kernel_artifacts ls -la "${WORKSPACE}/artifacts/" popd diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index f85ba4ec1..d9f6ccd90 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -16,7 +16,7 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} -container_image="${KERNEL_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:kernel-$(get_last_modification ${script_dir})-$(uname -m)}" +container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" sudo docker pull ${container_image} || \ (sudo docker build -t "${container_image}" "${script_dir}" && \ From 7aed8f8c80c3268a7073a8ca5ac6d8377ed4889f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 12:45:20 +0100 Subject: [PATCH 26/40] tools: Add support for caching Nydus artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching Nydus artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../static-build/cache_components_main.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index 3f620887c..84ca73a0b 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -34,6 +34,12 @@ cache_kernel_artifacts() { create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}" } +cache_nydus_artifacts() { + local nydus_tarball_name="kata-static-nydus.tar.xz" + local current_nydus_version="$(get_from_kata_deps "externals.nydus.version")" + create_cache_asset "${nydus_tarball_name}" "${current_nydus_version}" "" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -60,6 +66,7 @@ Usage: $0 "[options]" -k Kernel cache * Export KERNEL_FLAVOUR="kernel|kernek-experimental|kernel-arm-experimental|kernel-dragonball-experimental" for a specific build The default KERNEL_FLAVOUR value is "kernel" + -n Nydus cache -h Shows help EOF )" @@ -69,8 +76,9 @@ main() { local cloud_hypervisor_component="${cloud_hypervisor_component:-}" local firecracker_component="${firecracker_component:-}" local kernel_component="${kernel_component:-}" + local nydus_component="${nydus_component:-}" local OPTIND - while getopts ":cFkh:" opt + while getopts ":cFknh:" opt do case "$opt" in c) @@ -82,6 +90,9 @@ main() { k) kernel_component="1" ;; + n) + nydus_component="1" + ;; h) help exit 0; @@ -98,6 +109,7 @@ main() { [[ -z "${cloud_hypervisor_component}" ]] && \ [[ -z "${firecracker_component}" ]] && \ [[ -z "${kernel_component}" ]] && \ + [[ -z "${nydus_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -107,6 +119,7 @@ main() { [ "${cloud_hypervisor_component}" == "1" ] && cache_clh_artifacts [ "${firecracker_component}" == "1" ] && cache_firecracker_artifacts [ "${kernel_component}" == "1" ] && cache_kernel_artifacts + [ "${nydus_component}" == "1" ] && cache_nydus_artifacts ls -la "${WORKSPACE}/artifacts/" popd From e90891059b035fa3dc7aa0887e4db1779cc6be65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 12:59:40 +0100 Subject: [PATCH 27/40] tools: Add support for caching QEMU artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching QEMU artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- tools/packaging/scripts/lib.sh | 41 +++++++++++++++++++ .../static-build/cache_components_main.sh | 17 +++++++- .../static-build/qemu/build-base-qemu.sh | 2 +- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index deece4ede..6f006c1f9 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -134,3 +134,44 @@ get_kernel_image_name() { kernel_script_dir="${repo_root_dir}/tools/packaging/static-build/kernel" echo "${BUILDER_REGISTRY}:kernel-$(get_last_modification ${kernel_script_dir})-$(uname -m)" } + +sha256sum_from_files() { + local files_in=${@:-} + local files="" + local shasum="" + + # Process the input files: + # - discard the files/directories that don't exist. + # - find the files if it is a directory + for f in $files_in; do + if [ -d "$f" ]; then + files+=" $(find $f -type f)" + elif [ -f "$f" ]; then + files+=" $f" + fi + done + # Return in case there is none input files. + [ -n "$files" ] || return 0 + + # Alphabetically sorting the files. + files="$(echo $files | tr ' ' '\n' | LC_ALL=C sort -u)" + # Concate the files and calculate a hash. + shasum="$(cat $files | sha256sum -b)" || true + if [ -n "$shasum" ];then + # Return only the SHA field. + echo $(awk '{ print $1 }' <<< $shasum) + fi +} + +calc_qemu_files_sha256sum() { + local files="${repo_root_dir}/tools/packaging/qemu \ + ${repo_root_dir}/tools/packaging/static-build/qemu.blacklist \ + ${repo_root_dir}/tools/packaging/static-build/scripts" + + sha256sum_from_files "$files" +} + +get_qemu_image_name() { + qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu" + echo "${BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)" +} diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index 84ca73a0b..0136d1db0 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -40,6 +40,14 @@ cache_nydus_artifacts() { create_cache_asset "${nydus_tarball_name}" "${current_nydus_version}" "" } +cache_qemu_artifacts() { + local qemu_tarball_name="kata-static-qemu.tar.xz" + local current_qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") + local qemu_sha=$(calc_qemu_files_sha256sum) + local current_qemu_image="$(get_qemu_image_name)" + create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -67,6 +75,7 @@ Usage: $0 "[options]" * Export KERNEL_FLAVOUR="kernel|kernek-experimental|kernel-arm-experimental|kernel-dragonball-experimental" for a specific build The default KERNEL_FLAVOUR value is "kernel" -n Nydus cache + -q QEMU cache -h Shows help EOF )" @@ -77,8 +86,9 @@ main() { local firecracker_component="${firecracker_component:-}" local kernel_component="${kernel_component:-}" local nydus_component="${nydus_component:-}" + local qemu_component="${qemu_component:-}" local OPTIND - while getopts ":cFknh:" opt + while getopts ":cFknqh:" opt do case "$opt" in c) @@ -93,6 +103,9 @@ main() { n) nydus_component="1" ;; + q) + qemu_component="1" + ;; h) help exit 0; @@ -110,6 +123,7 @@ main() { [[ -z "${firecracker_component}" ]] && \ [[ -z "${kernel_component}" ]] && \ [[ -z "${nydus_component}" ]] && \ + [[ -z "${qemu_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -120,6 +134,7 @@ main() { [ "${firecracker_component}" == "1" ] && cache_firecracker_artifacts [ "${kernel_component}" == "1" ] && cache_kernel_artifacts [ "${nydus_component}" == "1" ] && cache_nydus_artifacts + [ "${qemu_component}" == "1" ] && cache_qemu_artifacts ls -la "${WORKSPACE}/artifacts/" popd diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh index 55ab71d35..9767e5d54 100755 --- a/tools/packaging/static-build/qemu/build-base-qemu.sh +++ b/tools/packaging/static-build/qemu/build-base-qemu.sh @@ -38,7 +38,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d") [ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu" [ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static" -container_image="${QEMU_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:qemu-$(get_last_modification ${script_dir})-$(uname -m)}" +container_image="${QEMU_CONTAINER_BUILDER:-$(get_qemu_image_name)}" sudo docker pull ${container_image} || (sudo "${container_engine}" build \ --build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \ From 7898db5f79024c485655d375ba70d00ec837d34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:05:15 +0100 Subject: [PATCH 28/40] tools: Add support for caching RootFS artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching RootFS artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../static-build/cache_components_main.sh | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index 0136d1db0..c5d49f773 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -13,6 +13,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../scripts/lib.sh" KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-experimental | kernel-arm-experimetnal | kernel-dragonball-experimental +ROOTFS_IMAGE_TYPE="${ROOTFS_IMAGE_TYPE:-image}" # image | initrd cache_clh_artifacts() { local clh_tarball_name="kata-static-cloud-hypervisor.tar.xz" @@ -48,6 +49,19 @@ cache_qemu_artifacts() { create_cache_asset "${qemu_tarball_name}" "${current_qemu_version}-${qemu_sha}" "${current_qemu_image}" } +cache_rootfs_artifacts() { + local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local rootfs_tarball_name="kata-static-rootfs-${ROOTFS_IMAGE_TYPE}.tar.xz" + local current_rootfs_version="${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${ROOTFS_IMAGE_TYPE}" + create_cache_asset "${rootfs_tarball_name}" "${current_rootfs_version}" "" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -76,6 +90,9 @@ Usage: $0 "[options]" The default KERNEL_FLAVOUR value is "kernel" -n Nydus cache -q QEMU cache + -r RootFS cache + * Export ROOTFS_IMAGE_TYPE="image|initrd" for one of those two types + The default ROOTFS_IMAGE_TYPE value is "image" -h Shows help EOF )" @@ -87,8 +104,9 @@ main() { local kernel_component="${kernel_component:-}" local nydus_component="${nydus_component:-}" local qemu_component="${qemu_component:-}" + local rootfs_component="${rootfs_component:-}" local OPTIND - while getopts ":cFknqh:" opt + while getopts ":cFknqrh:" opt do case "$opt" in c) @@ -106,6 +124,9 @@ main() { q) qemu_component="1" ;; + r) + rootfs_component="1" + ;; h) help exit 0; @@ -124,6 +145,7 @@ main() { [[ -z "${kernel_component}" ]] && \ [[ -z "${nydus_component}" ]] && \ [[ -z "${qemu_component}" ]] && \ + [[ -z "${rootfs_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -135,6 +157,7 @@ main() { [ "${kernel_component}" == "1" ] && cache_kernel_artifacts [ "${nydus_component}" == "1" ] && cache_nydus_artifacts [ "${qemu_component}" == "1" ] && cache_qemu_artifacts + [ "${rootfs_component}" == "1" ] && cache_rootfs_artifacts ls -la "${WORKSPACE}/artifacts/" popd From a34272cf20420f630839b357433022d53c8ebc19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:14:16 +0100 Subject: [PATCH 29/40] tools: Add support for caching shim v2 artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching shim v2 artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- tools/packaging/scripts/lib.sh | 5 +++++ .../static-build/cache_components_main.sh | 21 ++++++++++++++++++- tools/packaging/static-build/shim-v2/build.sh | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 6f006c1f9..61776bcbf 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -175,3 +175,8 @@ get_qemu_image_name() { qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu" echo "${BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)" } + +get_shim_v2_image_name() { + shim_v2_script_dir="${repo_root_dir}/tools/packaging/static-build/shim-v2" + echo "${BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" +} diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index c5d49f773..ef9cf6b9c 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -62,6 +62,18 @@ cache_rootfs_artifacts() { create_cache_asset "${rootfs_tarball_name}" "${current_rootfs_version}" "" } +cache_shim_v2_artifacts() { + local shim_v2_tarball_name="kata-static-shim-v2.tar.xz" + local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" + local protocols_last_commit="$(get_last_modification "${repo_root_dir}/src/libs/protocols")" + local runtime_rs_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime-rs")" + local golang_version="$(get_from_kata_deps "languages.golang.meta.newest-version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local current_shim_v2_version="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${golang_version}-${rust_version}" + local current_shim_v2_image="$(get_shim_v2_image_name)" + create_cache_asset "${shim_v2_tarball_name}" "${current_shim_v2_version}" "${current_shim_v2_image}" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -93,6 +105,7 @@ Usage: $0 "[options]" -r RootFS cache * Export ROOTFS_IMAGE_TYPE="image|initrd" for one of those two types The default ROOTFS_IMAGE_TYPE value is "image" + -s Shim v2 cache -h Shows help EOF )" @@ -105,8 +118,9 @@ main() { local nydus_component="${nydus_component:-}" local qemu_component="${qemu_component:-}" local rootfs_component="${rootfs_component:-}" + local shim_v2_component="${shim_v2_component:-}" local OPTIND - while getopts ":cFknqrh:" opt + while getopts ":cFknqrsh:" opt do case "$opt" in c) @@ -127,6 +141,9 @@ main() { r) rootfs_component="1" ;; + s) + shim_v2_component="1" + ;; h) help exit 0; @@ -146,6 +163,7 @@ main() { [[ -z "${nydus_component}" ]] && \ [[ -z "${qemu_component}" ]] && \ [[ -z "${rootfs_component}" ]] && \ + [[ -z "${shim_v2_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -158,6 +176,7 @@ main() { [ "${nydus_component}" == "1" ] && cache_nydus_artifacts [ "${qemu_component}" == "1" ] && cache_qemu_artifacts [ "${rootfs_component}" == "1" ] && cache_rootfs_artifacts + [ "${shim_v2_component}" == "1" ] && cache_shim_v2_artifacts ls -la "${WORKSPACE}/artifacts/" popd diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index 35ebabcab..d948ae1e8 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -19,7 +19,7 @@ RUST_VERSION=${RUST_VERSION} DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} -container_image="${SHIM_V2_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:shim-v2-go-${GO_VERSION}-rust-${RUST_VERSION}-$(get_last_modification ${script_dir})-$(uname -m)}" +container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}" sudo docker pull ${container_image} || \ (sudo docker build \ From 194d5dc8a6e92a56077dff8684d8b9b49a95b83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:17:19 +0100 Subject: [PATCH 30/40] tools: Add support for caching VirtioFS artefacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's add support for caching VirtioFS artefacts that are generated using the kata-deploy local-build scripts. Right now those are not used, but we'll switch to using them very soon as part of upcoming changes of how we build the components we test in our CI. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- tools/packaging/scripts/lib.sh | 21 +++++++++++++++++++ .../static-build/cache_components_main.sh | 16 +++++++++++++- .../packaging/static-build/virtiofsd/build.sh | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 61776bcbf..c42534108 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -180,3 +180,24 @@ get_shim_v2_image_name() { shim_v2_script_dir="${repo_root_dir}/tools/packaging/static-build/shim-v2" echo "${BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" } + +get_virtiofsd_image_name() { + ARCH=$(uname -m) + case ${ARCH} in + "aarch64") + libc="musl" + ;; + "ppc64le") + libc="gnu" + ;; + "s390x") + libc="gnu" + ;; + "x86_64") + libc="musl" + ;; + esac + + virtiofsd_script_dir="${repo_root_dir}/tools/packaging/static-build/virtiofsd" + echo "${BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)" +} diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index ef9cf6b9c..e447ab4bf 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -74,6 +74,13 @@ cache_shim_v2_artifacts() { create_cache_asset "${shim_v2_tarball_name}" "${current_shim_v2_version}" "${current_shim_v2_image}" } +cache_virtiofsd_artifacts() { + local virtiofsd_tarball_name="kata-static-virtiofsd.tar.xz" + local current_virtiofsd_version="$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" + local current_virtiofsd_image="$(get_virtiofsd_image_name)" + create_cache_asset "${virtiofsd_tarball_name}" "${current_virtiofsd_version}" "${current_virtiofsd_image}" +} + create_cache_asset() { local component_name="${1}" local component_version="${2}" @@ -106,6 +113,7 @@ Usage: $0 "[options]" * Export ROOTFS_IMAGE_TYPE="image|initrd" for one of those two types The default ROOTFS_IMAGE_TYPE value is "image" -s Shim v2 cache + -v VirtioFS cache -h Shows help EOF )" @@ -119,8 +127,9 @@ main() { local qemu_component="${qemu_component:-}" local rootfs_component="${rootfs_component:-}" local shim_v2_component="${shim_v2_component:-}" + local virtiofsd_component="${virtiofsd_component:-}" local OPTIND - while getopts ":cFknqrsh:" opt + while getopts ":cFknqrsvh:" opt do case "$opt" in c) @@ -144,6 +153,9 @@ main() { s) shim_v2_component="1" ;; + v) + virtiofsd_component="1" + ;; h) help exit 0; @@ -164,6 +176,7 @@ main() { [[ -z "${qemu_component}" ]] && \ [[ -z "${rootfs_component}" ]] && \ [[ -z "${shim_v2_component}" ]] && \ + [[ -z "${virtiofsd_component}" ]] && \ help && die "Must choose at least one option" mkdir -p "${WORKSPACE}/artifacts" @@ -177,6 +190,7 @@ main() { [ "${qemu_component}" == "1" ] && cache_qemu_artifacts [ "${rootfs_component}" == "1" ] && cache_rootfs_artifacts [ "${shim_v2_component}" == "1" ] && cache_shim_v2_artifacts + [ "${virtiofsd_component}" == "1" ] && cache_virtiofsd_artifacts ls -la "${WORKSPACE}/artifacts/" popd diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh index 68b335821..6eb5ad51b 100755 --- a/tools/packaging/static-build/virtiofsd/build.sh +++ b/tools/packaging/static-build/virtiofsd/build.sh @@ -48,7 +48,7 @@ case ${ARCH} in ;; esac -container_image="${VIRTIOFSD_CONTAINER_BUILDER:-${BUILDER_REGISTRY}:virtiofsd-${virtiofsd_toolchain}-${libc}-$(get_last_modification ${script_dir})-$(uname -m)}" +container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_virtiofsd_image_name)}" sudo docker pull ${container_image} || \ (sudo docker build \ From 8a40f6f23498bcfec6c0a2ebd4f7a1e47f2e1023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:32:38 +0100 Subject: [PATCH 31/40] local-build: Use cached Cloud Hypervisor when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../local-build/dockerbuild/Dockerfile | 1 + .../local-build/kata-deploy-binaries.sh | 54 +++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile b/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile index 2ff5f5ae8..1338f482a 100644 --- a/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile +++ b/tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile @@ -43,6 +43,7 @@ RUN apt-get update && \ git \ make \ unzip \ + wget \ xz-utils && \ apt-get clean && rm -rf /var/lib/apt/lists diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 79cf5a141..8b4361097 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -32,6 +32,9 @@ readonly nydus_builder="${static_build_dir}/nydus/build.sh" readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_image.sh" +readonly jenkins_url="http://jenkins.katacontainers.io" +readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" + ARCH=$(uname -m) workdir="${WORKDIR:-$PWD}" @@ -87,6 +90,34 @@ EOF exit "${return_code}" } + +cleanup_and_fail() { + rm -f "${component_tarball_path}" + return 1 +} + +install_cached_tarball_component() { + local component="${1}" + local jenkins_build_url="${2}" + local current_version="${3}" + local current_image_version="${4}" + local component_tarball_name="${5}" + local component_tarball_path="${6}" + + local cached_version=$(curl -sfL "${jenkins_build_url}/latest" | awk '{print $1}') || cached_version="none" + local cached_image_version=$(curl -sfL "${jenkins_build_url}/latest_image" | awk '{print $1}') || cached_image_version="none" + + [ "${cached_image_version}" != "${current_image_version}" ] && return 1 + [ "${cached_version}" != "${current_version}" ] && return 1 + + info "Using cached tarball of ${component}" + echo "Downloading tarball from: ${jenkins_build_url}/${component_tarball_name}" + wget "${jenkins_build_url}/${component_tarball_name}" || return cleanup_and_fail + wget "${jenkins_build_url}/sha256sum-${component_tarball_name}" || return cleanup_and_fail + sha256sum -c "sha256sum-${component_tarball_name}" || return cleanup_and_fail + mv "${component_tarball_name}" "${component_tarball_path}" +} + #Install guest image install_image() { info "Create image" @@ -142,6 +173,15 @@ install_firecracker() { # Install static cloud-hypervisor asset install_clh() { + install_cached_tarball_component \ + "cloud-hypervisor" \ + "${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + if [[ "${ARCH}" == "x86_64" ]]; then export features="tdx" fi @@ -192,6 +232,11 @@ handle_build() { info "DESTDIR ${destdir}" local build_target build_target="$1" + + export final_tarball_path="${workdir}/kata-static-${build_target}.tar.xz" + export final_tarball_name="$(basename ${final_tarball_path})" + rm -f ${final_tarball_name} + case "${build_target}" in all) install_clh @@ -232,12 +277,11 @@ handle_build() { ;; esac - tarball_name="${workdir}/kata-static-${build_target}.tar.xz" - ( + if [ ! -f "${final_tarball_path}" ]; then cd "${destdir}" - sudo tar cvfJ "${tarball_name}" "." - ) - tar tvf "${tarball_name}" + sudo tar cvfJ "${final_tarball_path}" "." + fi + tar tvf "${final_tarball_path}" } silent_mode_error_trap() { From 04fb52f6c9ab88802971c66301aee2b20224f212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:37:07 +0100 Subject: [PATCH 32/40] local-build: Use cached Firecracker when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../kata-deploy/local-build/kata-deploy-binaries.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 8b4361097..700a550d8 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -163,6 +163,15 @@ install_qemu() { # Install static firecracker asset install_firecracker() { + install_cached_tarball_component \ + "firecracker" \ + "${jenkins_url}/job/kata-containers-main-firecracker-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "assets.hypervisor.firecracker.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static firecracker" "${firecracker_builder}" info "Install static firecracker" From 64832ab65b353ab9f7ce07f2950b0e3ad368438d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:42:15 +0100 Subject: [PATCH 33/40] local-build: Use cached Kernel when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../local-build/kata-deploy-binaries.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 700a550d8..647fd2b68 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -133,6 +133,17 @@ install_initrd() { #Install kernel asset install_kernel() { export kernel_version="$(yq r $versions_yaml assets.kernel.version)" + local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + + install_cached_tarball_component \ + "kernel" \ + "${jenkins_url}/job/kata-containers-main-kernel-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -f -v "${kernel_version}" } @@ -140,6 +151,17 @@ install_kernel() { install_dragonball_experimental_kernel() { info "build dragonball experimental kernel" export kernel_version="$(yq r $versions_yaml assets.kernel-dragonball-experimental.version)" + local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + + install_cached_tarball_component \ + "kernel-dragonball-experimental" \ + "${jenkins_url}/job/kata-containers-main-kernel-dragonball-experimental-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -e -t dragonball -v ${kernel_version} } @@ -148,6 +170,17 @@ install_dragonball_experimental_kernel() { install_experimental_kernel() { info "build experimental kernel" export kernel_version="$(yq r $versions_yaml assets.kernel-experimental.tag)" + local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + + install_cached_tarball_component \ + "kernel-experimental" \ + "${jenkins_url}/job/kata-containers-main-kernel-experimental-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "Kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -f -b experimental -v ${kernel_version} } From 1e1c843b8b65dc463a6b57e16501a79b7ea6177c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:45:37 +0100 Subject: [PATCH 34/40] local-build: Use cached Nydus when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../kata-deploy/local-build/kata-deploy-binaries.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 647fd2b68..ccf419e6f 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -246,6 +246,15 @@ install_virtiofsd() { # Install static nydus asset install_nydus() { + install_cached_tarball_component \ + "nydus" \ + "${jenkins_url}/job/kata-containers-main-nydus-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "externals.nydus.version")" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static nydus" "${nydus_builder}" info "Install static nydus" From 09ce4ab893b2dc33772fef7b8e8702000bbf0747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:48:22 +0100 Subject: [PATCH 35/40] local-build: Use cached QEMU when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../kata-deploy/local-build/kata-deploy-binaries.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index ccf419e6f..45b29afb6 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -187,9 +187,19 @@ install_experimental_kernel() { # Install static qemu asset install_qemu() { - info "build static qemu" export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" + + install_cached_tarball_component \ + "QEMU" \ + "${jenkins_url}/job/kata-containers-main-qemu-$(uname -m)/${cached_artifacts_path}" \ + "${qemu_version}-$(calc_qemu_files_sha256sum)" \ + "$(get_qemu_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + + info "build static qemu" "${qemu_builder}" tar xvf "${builddir}/kata-static-qemu.tar.gz" -C "${destdir}" } From 1b8c5474dab15246914b5ad631d1ce5aaf515eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:54:32 +0100 Subject: [PATCH 36/40] local-build: Use cached RootFS when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../local-build/kata-deploy-binaries.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 45b29afb6..9074b76d2 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -120,12 +120,52 @@ install_cached_tarball_component() { #Install guest image install_image() { + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-image-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-image" + + local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + + install_cached_tarball_component \ + "${component}" \ + "${jenkins}" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "Create image" "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" } #Install guest initrd install_initrd() { + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-initrd-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-initrd" + + local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" + local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" + local agent_last_commit="$(get_last_modification "${repo_root_dir}/src/agent")" + local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" + local gperf_version="$(get_from_kata_deps "externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + + install_cached_tarball_component \ + "${component}" \ + "${jenkins}" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-initrd" \ + "" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "Create initrd" "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" } From 3b99004897749014e86d5d88a67820c2b0499494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 13:59:34 +0100 Subject: [PATCH 37/40] local-build: Use cached shim v2 when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../local-build/kata-deploy-binaries.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 9074b76d2..545a6763a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -316,8 +316,22 @@ install_nydus() { #Install all components that are not assets install_shimv2() { - GO_VERSION="$(yq r ${versions_yaml} languages.golang.meta.newest-version)" - RUST_VERSION="$(yq r ${versions_yaml} languages.rust.meta.newest-version)" + local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" + local runtime_rs_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime-rs")" + local protocols_last_commit="$(get_last_modification "${repo_root_dir}/src/libs/protocols")" + local GO_VERSION="$(get_from_kata_deps "languages.golang.meta.newest-version")" + local RUST_VERSION="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local shim_v2_version="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${GO_VERSION}-${RUST_VERSION}" + + install_cached_tarball_component \ + "shim-v2" \ + "${jenkins_url}/job/kata-containers-main-shim-v2-$(uname -m)/${cached_artifacts_path}" \ + "${shim_v2_version}" \ + "$(get_shim_v2_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + export GO_VERSION export RUST_VERSION DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" From 82a04dbce179fe9b4f767cb889fe730e3130a03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Mar 2023 14:02:03 +0100 Subject: [PATCH 38/40] local-build: Use cached VirtioFS when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we've added the support for caching components, let's use them whenever those are available. Fixes: #6480 Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes --- .../kata-deploy/local-build/kata-deploy-binaries.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 545a6763a..0d5ef611c 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -287,6 +287,15 @@ install_clh() { # Install static virtiofsd asset install_virtiofsd() { + install_cached_tarball_component \ + "virtiofsd" \ + "${jenkins_url}/job/kata-containers-main-virtiofsd-$(uname -m)/${cached_artifacts_path}" \ + "$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" \ + "$(get_virtiofsd_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + && return 0 + info "build static virtiofsd" "${virtiofsd_builder}" info "Install static virtiofsd" From fbf891fdfff5c83780a5ca61adad4423e850117a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 17 Mar 2023 10:41:04 +0100 Subject: [PATCH 39/40] packaging: Adapt `get_last_modification()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function is returning "" when called from the script used to cache the artefacts and one difference noted between this version and the already working one from the CCv0 is that we make sure to `pushd ${repo_root_dir}` in the CCv0 version. Let's give it a try here and see if it solves the issue. Signed-off-by: Fabiano Fidêncio --- tools/packaging/scripts/lib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index c42534108..aa101b870 100644 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -106,6 +106,7 @@ get_kata_hash() { get_last_modification() { local file="${1}" + pushd ${repo_root_dir} &> /dev/null # This is a workaround needed for when running this code on Jenkins git config --global --add safe.directory ${repo_root_dir} &> /dev/null @@ -113,6 +114,7 @@ get_last_modification() { [ $(git status --porcelain | grep "${file#${repo_root_dir}/}" | wc -l) -gt 0 ] && dirty="-dirty" echo "$(git log -1 --pretty=format:"%H" ${file})${dirty}" + popd &> /dev/null } # $1 - The tag to be pushed to the registry From f31c907f460e3ba77e10ed41153bfe3deaecf3b2 Mon Sep 17 00:00:00 2001 From: Megan Wright Date: Mon, 20 Mar 2023 13:34:08 +0000 Subject: [PATCH 40/40] Fix bad merge --- src/runtime/virtcontainers/kata_agent.go | 69 ++++++++++--------- .../local-build/kata-deploy-binaries.sh | 46 ++++++------- tools/packaging/scripts/lib.sh | 4 +- 3 files changed, 60 insertions(+), 59 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 6cc114e61..f13828f36 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -80,40 +80,41 @@ const ( type customRequestTimeoutKeyType struct{} var ( - checkRequestTimeout = 30 * time.Second - defaultRequestTimeout = 60 * time.Second - remoteRequestTimeout = 300 * time.Second - customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" - defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" - defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" - mountGuestTag = "kataShared" - defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" - type9pFs = "9p" - typeVirtioFS = "virtiofs" - typeOverlayFS = "overlay" - kata9pDevType = "9p" - kataMmioBlkDevType = "mmioblk" - kataBlkDevType = "blk" - kataBlkCCWDevType = "blk-ccw" - kataSCSIDevType = "scsi" - kataNvdimmDevType = "nvdimm" - kataVirtioFSDevType = "virtio-fs" - kataOverlayDevType = "overlayfs" - kataWatchableBindDevType = "watchable-bind" - kataVfioDevType = "vfio" // VFIO device to used as VFIO in the container - kataVfioGuestKernelDevType = "vfio-gk" // VFIO device for consumption by the guest kernel - sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} - sharedDirVirtioFSOptions = []string{} - sharedDirVirtioFSDaxOptions = "dax" - shmDir = "shm" - kataEphemeralDevType = "ephemeral" - defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) - grpcMaxDataSize = int64(1024 * 1024) - localDirOptions = []string{"mode=0777"} - maxHostnameLen = 64 - GuestDNSFile = "/etc/resolv.conf" + checkRequestTimeout = 30 * time.Second + defaultRequestTimeout = 60 * time.Second + remoteRequestTimeout = 300 * time.Second + customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" + defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" + defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" + mountGuestTag = "kataShared" + defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" + type9pFs = "9p" + typeVirtioFS = "virtiofs" + typeOverlayFS = "overlay" + kata9pDevType = "9p" + kataMmioBlkDevType = "mmioblk" + kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" + kataSCSIDevType = "scsi" + kataNvdimmDevType = "nvdimm" + kataVirtioFSDevType = "virtio-fs" + kataOverlayDevType = "overlayfs" + kataWatchableBindDevType = "watchable-bind" + kataVfioPciDevType = "vfio-pci" // VFIO device to used as VFIO in the container + kataVfioPciGuestKernelDevType = "vfio-pci-gk" // VFIO device for consumption by the guest kernel + kataVfioApDevType = "vfio-ap" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} + sharedDirVirtioFSOptions = []string{} + sharedDirVirtioFSDaxOptions = "dax" + shmDir = "shm" + kataEphemeralDevType = "ephemeral" + defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) + grpcMaxDataSize = int64(1024 * 1024) + localDirOptions = []string{"mode=0777"} + maxHostnameLen = 64 + GuestDNSFile = "/etc/resolv.conf" ) const ( diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 2af6de8f6..18ac9aed9 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -118,7 +118,7 @@ cleanup_and_fail() { return 1 } -install_cached_tarball_component() { +install_cached_component() { local component="${1}" local jenkins_build_url="${2}" local current_version="${3}" @@ -197,7 +197,7 @@ install_cached_cc_shim_v2() { wget "${jenkins_build_url}/root_hash_tdx.txt" -O "shim_v2_root_hash_tdx.txt" || return 1 diff "${root_hash_tdx}" "shim_v2_root_hash_tdx.txt" > /dev/null || return 1 - install_cached_tarball_component \ + install_cached_component \ "${component}" \ "${jenkins_build_url}" \ "${current_version}" \ @@ -210,7 +210,7 @@ install_cached_cc_shim_v2() { # Install static CC cloud-hypervisor asset install_cc_clh() { - install_cached_tarball_component \ + install_cached_component \ "cloud-hypervisor" \ "${jenkins_url}/job/kata-containers-2.0-clh-cc-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ @@ -269,7 +269,7 @@ install_cc_image() { local pause_version="$(get_from_kata_deps "externals.pause.version")" local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" - install_cached_tarball_component \ + install_cached_component \ "${component}" \ "${jenkins}" \ "${osbuilder_last_commit}-${guest_image_last_commit}-${initramfs_last_commit}-${agent_last_commit}-${libs_last_commit}-${attestation_agent_version}-${gperf_version}-${libseccomp_version}-${pause_version}-${rust_version}-${image_type}-${AA_KBC}" \ @@ -315,7 +315,7 @@ install_cc_kernel() { local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - install_cached_tarball_component \ + install_cached_component \ "kernel" \ "${jenkins_url}/job/kata-containers-2.0-kernel-cc-$(uname -m)/${cached_artifacts_path}" \ "${kernel_version}-${kernel_kata_config_version}" \ @@ -335,7 +335,7 @@ install_cc_qemu() { export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" - install_cached_tarball_component \ + install_cached_component \ "QEMU" \ "${jenkins_url}/job/kata-containers-2.0-qemu-cc-$(uname -m)/${cached_artifacts_path}" \ "${qemu_version}-$(calc_qemu_files_sha256sum)" \ @@ -392,7 +392,7 @@ install_cc_shimv2() { # Install static CC virtiofsd asset install_cc_virtiofsd() { local virtiofsd_version="$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" - install_cached_tarball_component \ + install_cached_component \ "virtiofsd" \ "${jenkins_url}/job/kata-containers-2.0-virtiofsd-cc-$(uname -m)/${cached_artifacts_path}" \ "${virtiofsd_version}" \ @@ -416,7 +416,7 @@ install_cached_kernel_component() { local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - install_cached_tarball_component \ + install_cached_component \ "kernel" \ "${jenkins_url}/job/kata-containers-2.0-kernel-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \ "${kernel_version}-${kernel_kata_config_version}" \ @@ -428,7 +428,7 @@ install_cached_kernel_component() { [ "${tee}" == "tdx" ] && return 0 # SEV specific code path - install_cached_tarball_component \ + install_cached_component \ "kernel-modules" \ "${jenkins_url}/job/kata-containers-2.0-kernel-sev-cc-$(uname -m)/${cached_artifacts_path}" \ "${kernel_version}" \ @@ -484,7 +484,7 @@ install_cc_tee_qemu() { export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.${tee}.tag)" export tee="${tee}" - install_cached_tarball_component \ + install_cached_component \ "QEMU ${tee}" \ "${jenkins_url}/job/kata-containers-2.0-qemu-${tee}-cc-$(uname -m)/${cached_artifacts_path}" \ "${qemu_version}-$(calc_qemu_files_sha256sum)" \ @@ -502,7 +502,7 @@ install_cc_tdx_qemu() { } install_cc_tdx_td_shim() { - install_cached_tarball_component \ + install_cached_component \ "td-shim" \ "${jenkins_url}/job/kata-containers-2.0-td-shim-cc-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "externals.td-shim.version")-$(get_from_kata_deps "externals.td-shim.toolchain")" \ @@ -522,7 +522,7 @@ install_cc_tee_ovmf() { local component_name="ovmf" local component_version="$(get_from_kata_deps "externals.ovmf.${tee}.version")" [ "${tee}" == "tdx" ] && component_name="tdvf" - install_cached_tarball_component \ + install_cached_component \ "${component_name}" \ "${jenkins_url}/job/kata-containers-2.0-${component_name}-cc-$(uname -m)/${cached_artifacts_path}" \ "${component_version}" \ @@ -556,7 +556,7 @@ install_image() { local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" - install_cached_tarball_component \ + install_cached_component \ "${component}" \ "${jenkins}" \ "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \ @@ -582,7 +582,7 @@ install_initrd() { local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" - install_cached_tarball_component \ + install_cached_component \ "${component}" \ "${jenkins}" \ "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-initrd" \ @@ -605,7 +605,7 @@ install_kernel() { export kernel_version="$(yq r $versions_yaml assets.kernel.version)" local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - install_cached_tarball_component \ + install_cached_component \ "kernel" \ "${jenkins_url}/job/kata-containers-main-kernel-$(uname -m)/${cached_artifacts_path}" \ "${kernel_version}-${kernel_kata_config_version}" \ @@ -623,7 +623,7 @@ install_dragonball_experimental_kernel() { export kernel_version="$(yq r $versions_yaml assets.kernel-dragonball-experimental.version)" local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - install_cached_tarball_component \ + install_cached_component \ "kernel-dragonball-experimental" \ "${jenkins_url}/job/kata-containers-main-kernel-dragonball-experimental-$(uname -m)/${cached_artifacts_path}" \ "${kernel_version}-${kernel_kata_config_version}" \ @@ -642,7 +642,7 @@ install_experimental_kernel() { export kernel_version="$(yq r $versions_yaml assets.kernel-experimental.tag)" local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - install_cached_tarball_component \ + install_cached_component \ "kernel-experimental" \ "${jenkins_url}/job/kata-containers-main-kernel-experimental-$(uname -m)/${cached_artifacts_path}" \ "${kernel_version}-${kernel_kata_config_version}" \ @@ -660,7 +660,7 @@ install_qemu() { export qemu_repo="$(yq r $versions_yaml assets.hypervisor.qemu.url)" export qemu_version="$(yq r $versions_yaml assets.hypervisor.qemu.version)" - install_cached_tarball_component \ + install_cached_component \ "QEMU" \ "${jenkins_url}/job/kata-containers-main-qemu-$(uname -m)/${cached_artifacts_path}" \ "${qemu_version}-$(calc_qemu_files_sha256sum)" \ @@ -676,7 +676,7 @@ install_qemu() { # Install static firecracker asset install_firecracker() { - install_cached_tarball_component \ + install_cached_component \ "firecracker" \ "${jenkins_url}/job/kata-containers-main-firecracker-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "assets.hypervisor.firecracker.version")" \ @@ -695,7 +695,7 @@ install_firecracker() { # Install static cloud-hypervisor asset install_clh() { - install_cached_tarball_component \ + install_cached_component \ "cloud-hypervisor" \ "${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \ @@ -717,7 +717,7 @@ install_clh() { # Install static virtiofsd asset install_virtiofsd() { - install_cached_tarball_component \ + install_cached_component \ "virtiofsd" \ "${jenkins_url}/job/kata-containers-main-virtiofsd-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" \ @@ -735,7 +735,7 @@ install_virtiofsd() { # Install static nydus asset install_nydus() { - install_cached_tarball_component \ + install_cached_component \ "nydus" \ "${jenkins_url}/job/kata-containers-main-nydus-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "externals.nydus.version")" \ @@ -762,7 +762,7 @@ install_shimv2() { local RUST_VERSION="$(get_from_kata_deps "languages.rust.meta.newest-version")" local shim_v2_version="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${GO_VERSION}-${RUST_VERSION}" - install_cached_tarball_component \ + install_cached_component \ "shim-v2" \ "${jenkins_url}/job/kata-containers-main-shim-v2-$(uname -m)/${cached_artifacts_path}" \ "${shim_v2_version}" \ diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 0283596cf..4a1198842 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -209,12 +209,12 @@ get_ovmf_image_name() { get_qemu_image_name() { qemu_script_dir="${repo_root_dir}/tools/packaging/static-build/qemu" - echo "${BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)" + echo "${CC_BUILDER_REGISTRY}:qemu-$(get_last_modification ${qemu_script_dir})-$(uname -m)" } get_shim_v2_image_name() { shim_v2_script_dir="${repo_root_dir}/tools/packaging/static-build/shim-v2" - echo "${BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" + echo "${CC_BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" } get_td_shim_image_name() {