aljaz/kata-containers
1
0
Fork 0
mirror of https://github.com/aljazceru/kata-containers.git synced 2025-12-19 07:14:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7219 from Apokleos/network-refactor

Browse Source 
runtime-rs: enhancement of Device Manager for network endpoints.
This commit is contained in:
Zhongtao Hu
2023-07-17 14:13:51 +08:00
committed by GitHub
parent ce14f26d82 283f809dda
commit d50f3888af
14 changed files with 325 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/runtime-rs/crates/resource/src/manager_inner.rs
View File

@@ -141,15 +141,13 @@ impl ResourceManagerInner {
// but it is not in netns. So, the previous thread would still remain in the pod netns.
// The solution is to block the future on the current thread, it is enabled by spawn an os thread, create a
// tokio runtime, and block the task on it.
let hypervisor = self.hypervisor.clone();
let device_manager = self.device_manager.clone();
let network = thread::spawn(move || -> Result<Arc<dyn Network>> {
let rt = runtime::Builder::new_current_thread().enable_io().build()?;
let d = rt
.block_on(network::new(&network_config, device_manager))
.context("new network")?;
rt.block_on(d.setup(hypervisor.as_ref()))
.context("setup network")?;
rt.block_on(d.setup()).context("setup network")?;
Ok(d)
})
.join()

60
src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs
View File

@@ -6,11 +6,12 @@
#[cfg(test)]
mod tests {
use anyhow::Context;
use std::{fs, path::Path, sync::Arc};
use anyhow::{anyhow, Context, Result};
use netlink_packet_route::MACVLAN_MODE_PRIVATE;
use scopeguard::defer;
use std::sync::Arc;
use tokio::sync::RwLock;
use crate::network::{
endpoint::{IPVlanEndpoint, MacVlanEndpoint, VlanEndpoint},
@@ -22,6 +23,41 @@ mod tests {
network_pair::{NetworkInterface, NetworkPair, TapInterface},
utils::link::net_test_utils::delete_link,
};
use hypervisor::{device::device_manager::DeviceManager, qemu::Qemu};
use kata_types::config::{QemuConfig, TomlConfig};
async fn get_device_manager() -> Result<Arc<RwLock<DeviceManager>>> {
let path = env!("CARGO_MANIFEST_DIR");
let path = Path::new(path)
.join("../../../libs/kata-types/tests/texture/configuration-anno-0.toml");
let content = fs::read_to_string(path).context("read configuration failed")?;
// just for test, use x/kata-types/tests/texture/configuration-anno-0.toml as
// the test configuration.toml which is for qemu.
let hypervisor_name: &str = "qemu";
let qemu = QemuConfig::new();
qemu.register();
let toml_config = TomlConfig::load(&content).context("load toml config failed")?;
let hypervisor_config = toml_config
.hypervisor
.get(hypervisor_name)
.ok_or_else(|| anyhow!("failed to get hypervisor for {}", &hypervisor_name))?;
let mut hypervisor = Qemu::new();
hypervisor
.set_hypervisor_config(hypervisor_config.clone())
.await;
let dm = Arc::new(RwLock::new(
DeviceManager::new(Arc::new(hypervisor))
.await
.context("device manager")?,
));
Ok(dm)
}
// this unit test tests the integrity of MacVlanEndpoint::new()
#[actix_rt::test]
@@ -33,6 +69,10 @@ mod tests {
let dummy_name = format!("dummy{}", idx);
let vlanid = 123;
let dm = get_device_manager().await;
assert!(dm.is_ok());
let d = dm.unwrap();
if let Ok((conn, handle, _)) =
rtnetlink::new_connection().context("failed to create netlink connection")
{
@@ -63,11 +103,12 @@ mod tests {
.await
.context("failed to create manual veth pair")
{
if let Ok(mut result) = VlanEndpoint::new(&handle, "", idx, 5)
if let Ok(mut result) = VlanEndpoint::new(&d, &handle, "", idx, 5)
.await
.context("failed to create new ipvlan endpoint")
{
let manual = VlanEndpoint {
d,
net_pair: NetworkPair {
tap: TapInterface {
id: String::from("uniqueTestID_kata"),
@@ -144,6 +185,9 @@ mod tests {
let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new()
let model_str = TC_FILTER_NET_MODEL_STR;
let dummy_name = format!("dummy{}", idx);
let dm = get_device_manager().await;
assert!(dm.is_ok());
let d = dm.unwrap();
if let Ok((conn, handle, _)) =
rtnetlink::new_connection().context("failed to create netlink connection")
@@ -180,6 +224,7 @@ mod tests {
{
// model here does not matter, could be any of supported models
if let Ok(mut result) = MacVlanEndpoint::new(
&d,
&handle,
manual_macvlan_iface_name.clone().as_str(),
idx,
@@ -190,6 +235,7 @@ mod tests {
.context("failed to create new macvlan endpoint")
{
let manual = MacVlanEndpoint {
d,
net_pair: NetworkPair {
tap: TapInterface {
id: String::from("uniqueTestID_kata"),
@@ -267,6 +313,9 @@ mod tests {
let mac_addr = String::from("02:00:CA:FE:00:04");
let manual_virt_iface_name = format!("eth{}", idx);
let tap_iface_name = format!("tap{}_kata", idx); // create by kata
let dm = get_device_manager().await;
assert!(dm.is_ok());
let d = dm.unwrap();
if let Ok((conn, handle, _)) =
rtnetlink::new_connection().context("failed to create netlink connection")
@@ -286,11 +335,12 @@ mod tests {
.await
.context("failed to create manual veth pair")
{
if let Ok(mut result) = IPVlanEndpoint::new(&handle, "", idx, 5)
if let Ok(mut result) = IPVlanEndpoint::new(&d, &handle, "", idx, 5)
.await
.context("failed to create new ipvlan endpoint")
{
let manual = IPVlanEndpoint {
d,
net_pair: NetworkPair {
tap: TapInterface {
id: String::from("uniqueTestID_kata"),

57
src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
View File

@@ -4,37 +4,54 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use super::endpoint_persist::{EndpointState, IpVlanEndpointState};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use tokio::sync::RwLock;
use super::Endpoint;
use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
use crate::network::{utils, NetworkPair};
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, IpVlanEndpointState},
Endpoint,
};
use crate::network::{network_model::TC_FILTER_NET_MODEL_STR, utils, NetworkPair};
// IPVlanEndpoint is the endpoint bridged to VM
#[derive(Debug)]
pub struct IPVlanEndpoint {
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl IPVlanEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
queues: usize,
) -> Result<Self> {
// tc filter network model is the only one works for ipvlan
// tc filter network model is the only for ipvlan
let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues)
.await
.context("error creating new NetworkPair")?;
Ok(IPVlanEndpoint { net_pair })
Ok(IPVlanEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -45,9 +62,12 @@ impl IPVlanEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -62,18 +82,16 @@ impl Endpoint for IPVlanEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("error adding network model")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network IPVlan endpoint device failed.")?;
Ok(())
}
@@ -86,12 +104,13 @@ impl Endpoint for IPVlanEndpoint {
let config = self
.get_network_config()
.context("error getting network config")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove IPVlan endpoint device by hypervisor failed.")?;
Ok(())
}

55
src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
View File

@@ -4,24 +4,39 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use super::endpoint_persist::{EndpointState, MacvlanEndpointState};
use super::Endpoint;
use crate::network::{utils, NetworkPair};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
use tokio::sync::RwLock;
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, MacvlanEndpointState},
Endpoint,
};
use crate::network::{utils, NetworkPair};
#[derive(Debug)]
pub struct MacVlanEndpoint {
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl MacVlanEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
@@ -31,7 +46,11 @@ impl MacVlanEndpoint {
let net_pair = NetworkPair::new(handle, idx, name, model, queues)
.await
.context("error creating new networkInterfacePair")?;
Ok(MacVlanEndpoint { net_pair })
Ok(MacVlanEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -42,9 +61,12 @@ impl MacVlanEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -59,18 +81,16 @@ impl Endpoint for MacVlanEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("add network model")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network MacVlan endpoint device failed.")?;
Ok(())
}
@@ -80,13 +100,14 @@ impl Endpoint for MacVlanEndpoint {
.del_network_model()
.await
.context("del network model")?;
let config = self.get_network_config().context("get network config")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove MacVlan endpoint device by hypervisor failed.")?;
Ok(())
}

2
src/runtime-rs/crates/resource/src/network/endpoint/mod.rs
View File

@@ -27,7 +27,7 @@ use super::EndpointState;
pub trait Endpoint: std::fmt::Debug + Send + Sync {
async fn name(&self) -> String;
async fn hardware_addr(&self) -> String;
async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
async fn attach(&self) -> Result<()>;
async fn detach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
async fn save(&self) -> Option<EndpointState>;
}

2
src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
View File

@@ -99,7 +99,7 @@ impl Endpoint for PhysicalEndpoint {
self.hard_addr.clone()
}
async fn attach(&self, _hypervisor: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
// bind physical interface from host driver and bind to vfio
driver::bind_device_to_vfio(
&self.bdf,

64
src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
View File

@@ -4,24 +4,39 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use super::endpoint_persist::{EndpointState, VethEndpointState};
use super::Endpoint;
use crate::network::{utils, NetworkPair};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
use tokio::sync::RwLock;
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, VethEndpointState},
Endpoint,
};
use crate::network::{utils, NetworkPair};
#[derive(Debug)]
pub struct VethEndpoint {
net_pair: NetworkPair,
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl VethEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
@@ -30,8 +45,12 @@ impl VethEndpoint {
) -> Result<Self> {
let net_pair = NetworkPair::new(handle, idx, name, model, queues)
.await
.context("new networkInterfacePair")?;
Ok(VethEndpoint { net_pair })
.context("new network interface pair failed.")?;
Ok(VethEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -42,9 +61,12 @@ impl VethEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -59,18 +81,17 @@ impl Endpoint for VethEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("add network model")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network Veth endpoint device failed.")?;
Ok(())
}
@@ -78,16 +99,19 @@ impl Endpoint for VethEndpoint {
self.net_pair
.del_network_model()
.await
.context("del network model")?;
.context("del network model failed.")?;
let config = self.get_network_config().context("get network config")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove Veth endpoint device by hypervisor failed.")?;
Ok(())
}
async fn save(&self) -> Option<EndpointState> {
Some(EndpointState {
veth_endpoint: Some(VethEndpointState {

64
src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
View File

@@ -4,25 +4,39 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use tokio::sync::RwLock;
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, VlanEndpointState},
Endpoint,
};
use crate::network::{network_model::TC_FILTER_NET_MODEL_STR, utils, NetworkPair};
use super::endpoint_persist::{EndpointState, VlanEndpointState};
use super::Endpoint;
use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
use crate::network::{utils, NetworkPair};
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
#[derive(Debug)]
pub struct VlanEndpoint {
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl VlanEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
@@ -30,8 +44,12 @@ impl VlanEndpoint {
) -> Result<Self> {
let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues)
.await
.context("error creating networkInterfacePair")?;
Ok(VlanEndpoint { net_pair })
.context("new network interface pair failed.")?;
Ok(VlanEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -42,9 +60,12 @@ impl VlanEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -59,18 +80,16 @@ impl Endpoint for VlanEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("error adding network model")?;
.context("add network model failed.")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network Vlan endpoint device failed.")?;
Ok(())
}
@@ -79,16 +98,17 @@ impl Endpoint for VlanEndpoint {
self.net_pair
.del_network_model()
.await
.context("error deleting network model")?;
.context("delete network model failed.")?;
let config = self
.get_network_config()
.context("error getting network config")?;
.context("get network config failed.")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove Vlan endpoint device by hypervisor failed.")?;
Ok(())
}

2
src/runtime-rs/crates/resource/src/network/mod.rs
View File

@@ -35,7 +35,7 @@ pub enum NetworkConfig {
#[async_trait]
pub trait Network: Send + Sync {
async fn setup(&self, h: &dyn Hypervisor) -> Result<()>;
async fn setup(&self) -> Result<()>;
async fn interfaces(&self) -> Result<Vec<agent::Interface>>;
async fn routes(&self) -> Result<Vec<agent::Route>>;
async fn neighs(&self) -> Result<Vec<agent::ARPNeighbor>>;

1
src/runtime-rs/crates/resource/src/network/network_pair.rs
View File

@@ -39,6 +39,7 @@ pub struct NetworkPair {
pub model: Arc<dyn network_model::NetworkModel>,
pub network_qos: bool,
}
impl NetworkPair {
pub(crate) async fn new(
handle: &rtnetlink::Handle,

10
src/runtime-rs/crates/resource/src/network/network_with_netns.rs
View File

@@ -88,11 +88,11 @@ impl NetworkWithNetns {
#[async_trait]
impl Network for NetworkWithNetns {
async fn setup(&self, h: &dyn Hypervisor) -> Result<()> {
async fn setup(&self) -> Result<()> {
let inner = self.inner.read().await;
let _netns_guard = netns::NetnsGuard::new(&inner.netns_path).context("net netns guard")?;
for e in &inner.entity_list {
e.endpoint.attach(h).await.context("attach")?;
e.endpoint.attach().await.context("attach")?;
}
Ok(())
}
@@ -241,6 +241,7 @@ async fn create_endpoint(
match link_type {
"veth" => {
let ret = VethEndpoint::new(
&d,
handle,
&attrs.name,
idx,
@@ -252,19 +253,20 @@ async fn create_endpoint(
Arc::new(ret)
}
"vlan" => {
let ret = VlanEndpoint::new(handle, &attrs.name, idx, config.queues)
let ret = VlanEndpoint::new(&d, handle, &attrs.name, idx, config.queues)
.await
.context("vlan endpoint")?;
Arc::new(ret)
}
"ipvlan" => {
let ret = IPVlanEndpoint::new(handle, &attrs.name, idx, config.queues)
let ret = IPVlanEndpoint::new(&d, handle, &attrs.name, idx, config.queues)
.await
.context("ipvlan endpoint")?;
Arc::new(ret)
}
"macvlan" => {
let ret = MacVlanEndpoint::new(
&d,
handle,
&attrs.name,
idx,