mirror of
https://github.com/aljazceru/kata-containers.git
synced 2026-02-23 15:34:28 +01:00
runtime-rs: add support direct volume.
As block/direct volume use similar steps of device adding, so making full use of block volume code is a better way to handle direct volume. the only different point is that direct volume will use DirectVolume and get_volume_mount_info to parse mountinfo.json from the direct volume path. That's to say, direct volume needs the help of `kata-ctl direct-volume ...`. Details seen at Advanced Topics: [How to run Kata Containers with kinds of Block Volumes] docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md Fixes: #5656 Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
This commit is contained in:
alex.lyn
90
src/runtime-rs/Cargo.lock
generated
90
src/runtime-rs/Cargo.lock
generated
@@ -241,6 +241,17 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599"
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi 0.1.19",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
@@ -827,6 +838,27 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-next"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys-next"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlv-list"
|
||||
version = "0.3.0"
|
||||
@@ -1212,6 +1244,15 @@ version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.2.6"
|
||||
@@ -1619,6 +1660,7 @@ dependencies = [
|
||||
"slog-async",
|
||||
"slog-json",
|
||||
"slog-scope",
|
||||
"slog-term",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1846,7 +1888,16 @@ version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.2.6",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_threads"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -2434,6 +2485,17 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
|
||||
dependencies = [
|
||||
"getrandom 0.2.8",
|
||||
"redox_syscall",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.7.1"
|
||||
@@ -2845,6 +2907,19 @@ dependencies = [
|
||||
"slog-scope",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "slog-term"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"slog",
|
||||
"term",
|
||||
"thread_local",
|
||||
"time 0.3.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.10.0"
|
||||
@@ -2963,6 +3038,17 @@ dependencies = [
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "term"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
|
||||
dependencies = [
|
||||
"dirs-next",
|
||||
"rustversion",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.2.0"
|
||||
@@ -3043,6 +3129,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"libc",
|
||||
"num_threads",
|
||||
"serde",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
|
||||
@@ -10,7 +10,10 @@ use agent::Storage;
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use hypervisor::{
|
||||
device::{device_manager::DeviceManager, DeviceConfig},
|
||||
device::{
|
||||
device_manager::{do_handle_device, DeviceManager},
|
||||
DeviceConfig, DeviceType,
|
||||
},
|
||||
BlockConfig,
|
||||
};
|
||||
use kata_types::mount::Mount;
|
||||
@@ -46,18 +49,10 @@ impl BlockRootfs {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let device_id = d
|
||||
.write()
|
||||
// create and insert block device into Kata VM
|
||||
let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone()))
|
||||
.await
|
||||
.new_device(&DeviceConfig::BlockCfg(block_device_config.clone()))
|
||||
.await
|
||||
.context("failed to create deviec")?;
|
||||
|
||||
d.write()
|
||||
.await
|
||||
.try_add_device(device_id.as_str())
|
||||
.await
|
||||
.context("failed to add deivce")?;
|
||||
.context("do handle device failed.")?;
|
||||
|
||||
let mut storage = Storage {
|
||||
fs_type: rootfs.fs_type.clone(),
|
||||
@@ -66,17 +61,11 @@ impl BlockRootfs {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// get complete device information
|
||||
let dev_info = d
|
||||
.read()
|
||||
.await
|
||||
.get_device_info(device_id.as_str())
|
||||
.await
|
||||
.context("failed to get device info")?;
|
||||
|
||||
if let DeviceConfig::BlockCfg(config) = dev_info {
|
||||
storage.driver = config.driver_option;
|
||||
storage.source = config.virt_path;
|
||||
let mut device_id: String = "".to_owned();
|
||||
if let DeviceType::Block(device) = device_info {
|
||||
storage.driver = device.config.driver_option;
|
||||
storage.source = device.config.virt_path;
|
||||
device_id = device.device_id;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
|
||||
@@ -4,29 +4,32 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use std::{collections::HashMap, fs, path::Path};
|
||||
use nix::sys::{stat, stat::SFlag};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::share_fs::{do_get_guest_path, do_get_host_path};
|
||||
|
||||
use super::{share_fs_volume::generate_mount_path, Volume};
|
||||
use agent::Storage;
|
||||
use anyhow::{anyhow, Context};
|
||||
use super::Volume;
|
||||
use crate::volume::utils::{
|
||||
generate_shared_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, KATA_DIRECT_VOLUME_TYPE,
|
||||
KATA_MOUNT_BIND_TYPE,
|
||||
};
|
||||
use hypervisor::{
|
||||
device::{device_manager::DeviceManager, DeviceConfig},
|
||||
device::{
|
||||
device_manager::{do_handle_device, DeviceManager},
|
||||
DeviceConfig, DeviceType,
|
||||
},
|
||||
BlockConfig,
|
||||
};
|
||||
use nix::sys::stat::{self, SFlag};
|
||||
use tokio::sync::RwLock;
|
||||
#[derive(Debug)]
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BlockVolume {
|
||||
storage: Option<agent::Storage>,
|
||||
mount: oci::Mount,
|
||||
device_id: String,
|
||||
}
|
||||
|
||||
/// BlockVolume: block device volume
|
||||
/// BlockVolume for bind-mount block volume and direct block volume
|
||||
impl BlockVolume {
|
||||
pub(crate) async fn new(
|
||||
d: &RwLock<DeviceManager>,
|
||||
@@ -35,54 +38,71 @@ impl BlockVolume {
|
||||
cid: &str,
|
||||
sid: &str,
|
||||
) -> Result<Self> {
|
||||
let fstat = stat::stat(m.source.as_str()).context(format!("stat {}", m.source))?;
|
||||
info!(sl!(), "device stat: {:?}", fstat);
|
||||
let mut options = HashMap::new();
|
||||
if read_only {
|
||||
options.insert("read_only".to_string(), "true".to_string());
|
||||
}
|
||||
let mnt_src: &str = &m.source;
|
||||
// default block device fs type: ext4.
|
||||
let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string();
|
||||
|
||||
let block_device_config = &mut BlockConfig {
|
||||
major: stat::major(fstat.st_rdev) as i64,
|
||||
minor: stat::minor(fstat.st_rdev) as i64,
|
||||
..Default::default()
|
||||
let block_device_config = match m.r#type.as_str() {
|
||||
KATA_MOUNT_BIND_TYPE => {
|
||||
let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?;
|
||||
|
||||
BlockConfig {
|
||||
major: stat::major(fstat.st_rdev) as i64,
|
||||
minor: stat::minor(fstat.st_rdev) as i64,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
KATA_DIRECT_VOLUME_TYPE => {
|
||||
// get volume mountinfo from mountinfo.json
|
||||
let v = volume_mount_info(mnt_src)
|
||||
.context("deserde information from mountinfo.json")?;
|
||||
// check volume type
|
||||
if v.volume_type != KATA_DIRECT_VOLUME_TYPE {
|
||||
return Err(anyhow!("volume type {:?} is invalid", v.volume_type));
|
||||
}
|
||||
|
||||
let fstat = stat::stat(v.device.as_str())
|
||||
.with_context(|| format!("stat volume device file: {}", v.device.clone()))?;
|
||||
if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG
|
||||
&& SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK
|
||||
{
|
||||
return Err(anyhow!(
|
||||
"invalid volume device {:?} for volume type {:?}",
|
||||
v.device,
|
||||
v.volume_type
|
||||
));
|
||||
}
|
||||
|
||||
blk_dev_fstype = v.fs_type.clone();
|
||||
|
||||
BlockConfig {
|
||||
path_on_host: v.device,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(anyhow!(
|
||||
"unsupport direct block volume r#type: {:?}",
|
||||
m.r#type.as_str()
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let device_id = d
|
||||
.write()
|
||||
// create and insert block device into Kata VM
|
||||
let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone()))
|
||||
.await
|
||||
.new_device(&DeviceConfig::BlockCfg(block_device_config.clone()))
|
||||
.await
|
||||
.context("failed to create deviec")?;
|
||||
.context("do handle device failed.")?;
|
||||
|
||||
d.write()
|
||||
// generate host guest shared path
|
||||
let guest_path = generate_shared_path(m.destination.clone(), read_only, cid, sid)
|
||||
.await
|
||||
.try_add_device(device_id.as_str())
|
||||
.await
|
||||
.context("failed to add deivce")?;
|
||||
|
||||
let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap();
|
||||
let file_name = generate_mount_path(cid, file_name);
|
||||
let guest_path = do_get_guest_path(&file_name, cid, true, false);
|
||||
let host_path = do_get_host_path(&file_name, sid, cid, true, read_only);
|
||||
fs::create_dir_all(&host_path)
|
||||
.map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?;
|
||||
|
||||
// get complete device information
|
||||
let dev_info = d
|
||||
.read()
|
||||
.await
|
||||
.get_device_info(&device_id)
|
||||
.await
|
||||
.context("failed to get device info")?;
|
||||
.context("generate host-guest shared path failed")?;
|
||||
|
||||
// storage
|
||||
let mut storage = Storage::default();
|
||||
|
||||
if let DeviceConfig::BlockCfg(config) = dev_info {
|
||||
storage.driver = config.driver_option;
|
||||
storage.source = config.virt_path;
|
||||
}
|
||||
let mut storage = agent::Storage {
|
||||
mount_point: guest_path.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
storage.options = if read_only {
|
||||
vec!["ro".to_string()]
|
||||
@@ -90,21 +110,32 @@ impl BlockVolume {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
storage.mount_point = guest_path.clone();
|
||||
|
||||
// If the volume had specified the filesystem type, use it. Otherwise, set it
|
||||
// to ext4 since but right now we only support it.
|
||||
if m.r#type != "bind" {
|
||||
storage.fs_type = m.r#type.clone();
|
||||
} else {
|
||||
storage.fs_type = "ext4".to_string();
|
||||
// As the true Block Device wrapped in DeviceType, we need to
|
||||
// get it out from the wrapper, and the device_id will be for
|
||||
// BlockVolume.
|
||||
// safe here, device_info is correct and only unwrap it.
|
||||
let mut device_id = String::new();
|
||||
if let DeviceType::Block(device) = device_info {
|
||||
// blk, mmioblk
|
||||
storage.driver = device.config.driver_option;
|
||||
// /dev/vdX
|
||||
storage.source = device.config.virt_path;
|
||||
device_id = device.device_id;
|
||||
}
|
||||
|
||||
// In some case, dest is device /dev/xxx
|
||||
if m.destination.clone().starts_with("/dev") {
|
||||
storage.fs_type = "bind".to_string();
|
||||
storage.options.append(&mut m.options.clone());
|
||||
} else {
|
||||
// usually, the dest is directory.
|
||||
storage.fs_type = blk_dev_fstype;
|
||||
}
|
||||
|
||||
// mount
|
||||
let mount = oci::Mount {
|
||||
destination: m.destination.clone(),
|
||||
r#type: m.r#type.clone(),
|
||||
source: guest_path.clone(),
|
||||
r#type: storage.fs_type.clone(),
|
||||
source: guest_path,
|
||||
options: m.options.clone(),
|
||||
};
|
||||
|
||||
@@ -128,6 +159,7 @@ impl Volume for BlockVolume {
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
@@ -144,13 +176,22 @@ impl Volume for BlockVolume {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_block_volume(m: &oci::Mount) -> bool {
|
||||
if m.r#type != "bind" {
|
||||
return false;
|
||||
pub(crate) fn is_block_volume(m: &oci::Mount) -> Result<bool> {
|
||||
let vol_types = vec![KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE];
|
||||
if !vol_types.contains(&m.r#type.as_str()) {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Ok(fstat) = stat::stat(m.source.as_str()).context(format!("stat {}", m.source)) {
|
||||
info!(sl!(), "device stat: {:?}", fstat);
|
||||
return SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK;
|
||||
|
||||
let fstat =
|
||||
stat::stat(m.source.as_str()).context(format!("stat mount source {} failed.", m.source))?;
|
||||
let s_flag = SFlag::from_bits_truncate(fstat.st_mode);
|
||||
|
||||
match m.r#type.as_str() {
|
||||
// case: mount bind and block device
|
||||
KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true),
|
||||
// case: directvol and directory
|
||||
KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true),
|
||||
// else: unsupported or todo for other volume type.
|
||||
_ => Ok(false),
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
@@ -9,16 +9,18 @@ mod default_volume;
|
||||
pub mod hugepage;
|
||||
mod share_fs_volume;
|
||||
mod shm_volume;
|
||||
use async_trait::async_trait;
|
||||
pub mod utils;
|
||||
|
||||
use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume};
|
||||
use agent::Agent;
|
||||
use anyhow::{Context, Result};
|
||||
use hypervisor::device::device_manager::DeviceManager;
|
||||
use std::{sync::Arc, vec::Vec};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use self::hugepage::{get_huge_page_limits_map, get_huge_page_option};
|
||||
use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume};
|
||||
use agent::Agent;
|
||||
use hypervisor::device::device_manager::DeviceManager;
|
||||
|
||||
const BIND: &str = "bind";
|
||||
|
||||
@@ -66,7 +68,7 @@ impl VolumeResource {
|
||||
shm_volume::ShmVolume::new(m, shm_size)
|
||||
.with_context(|| format!("new shm volume {:?}", m))?,
|
||||
)
|
||||
} else if is_block_volume(m) {
|
||||
} else if is_block_volume(m).context("block volume type")? {
|
||||
// handle block volume
|
||||
Arc::new(
|
||||
block_volume::BlockVolume::new(d, m, read_only, cid, sid)
|
||||
|
||||
65
src/runtime-rs/crates/resource/src/volume/utils.rs
Normal file
65
src/runtime-rs/crates/resource/src/volume/utils.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
// Copyright (c) 2022-2023 Alibaba Cloud
|
||||
// Copyright (c) 2022-2023 Ant Group
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
|
||||
use crate::{
|
||||
share_fs::{do_get_guest_path, do_get_host_path},
|
||||
volume::share_fs_volume::generate_mount_path,
|
||||
};
|
||||
use kata_sys_util::eother;
|
||||
use kata_types::mount::{get_volume_mount_info, DirectVolumeMountInfo};
|
||||
|
||||
pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4";
|
||||
pub const KATA_MOUNT_BIND_TYPE: &str = "bind";
|
||||
pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol";
|
||||
pub const KATA_VFIO_VOLUME_TYPE: &str = "vfiovol";
|
||||
pub const KATA_SPDK_VOLUME_TYPE: &str = "spdkvol";
|
||||
|
||||
// volume mount info load infomation from mountinfo.json
|
||||
pub fn volume_mount_info(volume_path: &str) -> Result<DirectVolumeMountInfo> {
|
||||
get_volume_mount_info(volume_path)
|
||||
}
|
||||
|
||||
pub fn get_file_name<P: AsRef<Path>>(src: P) -> Result<String> {
|
||||
let file_name = src
|
||||
.as_ref()
|
||||
.file_name()
|
||||
.map(|v| v.to_os_string())
|
||||
.ok_or_else(|| {
|
||||
eother!(
|
||||
"failed to get file name of path {}",
|
||||
src.as_ref().to_string_lossy()
|
||||
)
|
||||
})?
|
||||
.into_string()
|
||||
.map_err(|e| anyhow!("failed to convert to string {:?}", e))?;
|
||||
|
||||
Ok(file_name)
|
||||
}
|
||||
|
||||
pub(crate) async fn generate_shared_path(
|
||||
dest: String,
|
||||
read_only: bool,
|
||||
cid: &str,
|
||||
sid: &str,
|
||||
) -> Result<String> {
|
||||
let file_name = get_file_name(&dest).context("failed to get file name.")?;
|
||||
let mount_name = generate_mount_path(cid, file_name.as_str());
|
||||
let guest_path = do_get_guest_path(&mount_name, cid, true, false);
|
||||
let host_path = do_get_host_path(&mount_name, sid, cid, true, read_only);
|
||||
|
||||
if dest.starts_with("/dev") {
|
||||
fs::File::create(&host_path).context(format!("failed to create file {:?}", &host_path))?;
|
||||
} else {
|
||||
std::fs::create_dir_all(&host_path)
|
||||
.map_err(|e| anyhow!("failed to create dir {}: {:?}", host_path, e))?;
|
||||
}
|
||||
|
||||
Ok(guest_path)
|
||||
}
|
||||
Reference in New Issue
Block a user