aljaz/kata-containers
1
0
Fork 0
mirror of https://github.com/aljazceru/kata-containers.git synced 2025-12-18 23:04:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

runtime-rs: Enhancing Device Manager for network endpoints.

Browse Source 
Currently, network endpoints are separate from the device manager
and need to be included for proper management. In order to do so,
we need to refactor the implementation of the network endpoints.

The first step is to restructure the NetworkConfig and NetworkDevice
structures.
Next, we will implement the virtio-net driver and add the Network
device to the Device Manager.
Finally, we'll unify entries with do_handle_device for each endpoint.

Fixes: #7215

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
This commit is contained in:
alex.lyn
2023-07-02 22:30:32 +08:00
parent d01762dc35
commit 283f809dda
15 changed files with 440 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/runtime-rs/crates/resource/src/manager_inner.rs
View File

@@ -141,15 +141,13 @@ impl ResourceManagerInner {
// but it is not in netns. So, the previous thread would still remain in the pod netns.
// The solution is to block the future on the current thread, it is enabled by spawn an os thread, create a
// tokio runtime, and block the task on it.
let hypervisor = self.hypervisor.clone();
let device_manager = self.device_manager.clone();
let network = thread::spawn(move || -> Result<Arc<dyn Network>> {
let rt = runtime::Builder::new_current_thread().enable_io().build()?;
let d = rt
.block_on(network::new(&network_config, device_manager))
.context("new network")?;
rt.block_on(d.setup(hypervisor.as_ref()))
.context("setup network")?;
rt.block_on(d.setup()).context("setup network")?;
Ok(d)
})
.join()

60
src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs
View File

@@ -6,11 +6,12 @@
#[cfg(test)]
mod tests {
use anyhow::Context;
use std::{fs, path::Path, sync::Arc};
use anyhow::{anyhow, Context, Result};
use netlink_packet_route::MACVLAN_MODE_PRIVATE;
use scopeguard::defer;
use std::sync::Arc;
use tokio::sync::RwLock;
use crate::network::{
endpoint::{IPVlanEndpoint, MacVlanEndpoint, VlanEndpoint},
@@ -22,6 +23,41 @@ mod tests {
network_pair::{NetworkInterface, NetworkPair, TapInterface},
utils::link::net_test_utils::delete_link,
};
use hypervisor::{device::device_manager::DeviceManager, qemu::Qemu};
use kata_types::config::{QemuConfig, TomlConfig};
async fn get_device_manager() -> Result<Arc<RwLock<DeviceManager>>> {
let path = env!("CARGO_MANIFEST_DIR");
let path = Path::new(path)
.join("../../../libs/kata-types/tests/texture/configuration-anno-0.toml");
let content = fs::read_to_string(path).context("read configuration failed")?;
// just for test, use x/kata-types/tests/texture/configuration-anno-0.toml as
// the test configuration.toml which is for qemu.
let hypervisor_name: &str = "qemu";
let qemu = QemuConfig::new();
qemu.register();
let toml_config = TomlConfig::load(&content).context("load toml config failed")?;
let hypervisor_config = toml_config
.hypervisor
.get(hypervisor_name)
.ok_or_else(|| anyhow!("failed to get hypervisor for {}", &hypervisor_name))?;
let mut hypervisor = Qemu::new();
hypervisor
.set_hypervisor_config(hypervisor_config.clone())
.await;
let dm = Arc::new(RwLock::new(
DeviceManager::new(Arc::new(hypervisor))
.await
.context("device manager")?,
));
Ok(dm)
}
// this unit test tests the integrity of MacVlanEndpoint::new()
#[actix_rt::test]
@@ -33,6 +69,10 @@ mod tests {
let dummy_name = format!("dummy{}", idx);
let vlanid = 123;
let dm = get_device_manager().await;
assert!(dm.is_ok());
let d = dm.unwrap();
if let Ok((conn, handle, _)) =
rtnetlink::new_connection().context("failed to create netlink connection")
{
@@ -63,11 +103,12 @@ mod tests {
.await
.context("failed to create manual veth pair")
{
if let Ok(mut result) = VlanEndpoint::new(&handle, "", idx, 5)
if let Ok(mut result) = VlanEndpoint::new(&d, &handle, "", idx, 5)
.await
.context("failed to create new ipvlan endpoint")
{
let manual = VlanEndpoint {
d,
net_pair: NetworkPair {
tap: TapInterface {
id: String::from("uniqueTestID_kata"),
@@ -144,6 +185,9 @@ mod tests {
let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new()
let model_str = TC_FILTER_NET_MODEL_STR;
let dummy_name = format!("dummy{}", idx);
let dm = get_device_manager().await;
assert!(dm.is_ok());
let d = dm.unwrap();
if let Ok((conn, handle, _)) =
rtnetlink::new_connection().context("failed to create netlink connection")
@@ -180,6 +224,7 @@ mod tests {
{
// model here does not matter, could be any of supported models
if let Ok(mut result) = MacVlanEndpoint::new(
&d,
&handle,
manual_macvlan_iface_name.clone().as_str(),
idx,
@@ -190,6 +235,7 @@ mod tests {
.context("failed to create new macvlan endpoint")
{
let manual = MacVlanEndpoint {
d,
net_pair: NetworkPair {
tap: TapInterface {
id: String::from("uniqueTestID_kata"),
@@ -267,6 +313,9 @@ mod tests {
let mac_addr = String::from("02:00:CA:FE:00:04");
let manual_virt_iface_name = format!("eth{}", idx);
let tap_iface_name = format!("tap{}_kata", idx); // create by kata
let dm = get_device_manager().await;
assert!(dm.is_ok());
let d = dm.unwrap();
if let Ok((conn, handle, _)) =
rtnetlink::new_connection().context("failed to create netlink connection")
@@ -286,11 +335,12 @@ mod tests {
.await
.context("failed to create manual veth pair")
{
if let Ok(mut result) = IPVlanEndpoint::new(&handle, "", idx, 5)
if let Ok(mut result) = IPVlanEndpoint::new(&d, &handle, "", idx, 5)
.await
.context("failed to create new ipvlan endpoint")
{
let manual = IPVlanEndpoint {
d,
net_pair: NetworkPair {
tap: TapInterface {
id: String::from("uniqueTestID_kata"),

57
src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
View File

@@ -4,37 +4,54 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use super::endpoint_persist::{EndpointState, IpVlanEndpointState};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use tokio::sync::RwLock;
use super::Endpoint;
use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
use crate::network::{utils, NetworkPair};
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, IpVlanEndpointState},
Endpoint,
};
use crate::network::{network_model::TC_FILTER_NET_MODEL_STR, utils, NetworkPair};
// IPVlanEndpoint is the endpoint bridged to VM
#[derive(Debug)]
pub struct IPVlanEndpoint {
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl IPVlanEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
queues: usize,
) -> Result<Self> {
// tc filter network model is the only one works for ipvlan
// tc filter network model is the only for ipvlan
let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues)
.await
.context("error creating new NetworkPair")?;
Ok(IPVlanEndpoint { net_pair })
Ok(IPVlanEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -45,9 +62,12 @@ impl IPVlanEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -62,18 +82,16 @@ impl Endpoint for IPVlanEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("error adding network model")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network IPVlan endpoint device failed.")?;
Ok(())
}
@@ -86,12 +104,13 @@ impl Endpoint for IPVlanEndpoint {
let config = self
.get_network_config()
.context("error getting network config")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove IPVlan endpoint device by hypervisor failed.")?;
Ok(())
}

55
src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
View File

@@ -4,24 +4,39 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use super::endpoint_persist::{EndpointState, MacvlanEndpointState};
use super::Endpoint;
use crate::network::{utils, NetworkPair};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
use tokio::sync::RwLock;
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, MacvlanEndpointState},
Endpoint,
};
use crate::network::{utils, NetworkPair};
#[derive(Debug)]
pub struct MacVlanEndpoint {
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl MacVlanEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
@@ -31,7 +46,11 @@ impl MacVlanEndpoint {
let net_pair = NetworkPair::new(handle, idx, name, model, queues)
.await
.context("error creating new networkInterfacePair")?;
Ok(MacVlanEndpoint { net_pair })
Ok(MacVlanEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -42,9 +61,12 @@ impl MacVlanEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -59,18 +81,16 @@ impl Endpoint for MacVlanEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("add network model")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network MacVlan endpoint device failed.")?;
Ok(())
}
@@ -80,13 +100,14 @@ impl Endpoint for MacVlanEndpoint {
.del_network_model()
.await
.context("del network model")?;
let config = self.get_network_config().context("get network config")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove MacVlan endpoint device by hypervisor failed.")?;
Ok(())
}

2
src/runtime-rs/crates/resource/src/network/endpoint/mod.rs
View File

@@ -27,7 +27,7 @@ use super::EndpointState;
pub trait Endpoint: std::fmt::Debug + Send + Sync {
async fn name(&self) -> String;
async fn hardware_addr(&self) -> String;
async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
async fn attach(&self) -> Result<()>;
async fn detach(&self, hypervisor: &dyn Hypervisor) -> Result<()>;
async fn save(&self) -> Option<EndpointState>;
}

2
src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
View File

@@ -99,7 +99,7 @@ impl Endpoint for PhysicalEndpoint {
self.hard_addr.clone()
}
async fn attach(&self, _hypervisor: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
// bind physical interface from host driver and bind to vfio
driver::bind_device_to_vfio(
&self.bdf,

64
src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
View File

@@ -4,24 +4,39 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use super::endpoint_persist::{EndpointState, VethEndpointState};
use super::Endpoint;
use crate::network::{utils, NetworkPair};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
use tokio::sync::RwLock;
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, VethEndpointState},
Endpoint,
};
use crate::network::{utils, NetworkPair};
#[derive(Debug)]
pub struct VethEndpoint {
net_pair: NetworkPair,
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl VethEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
@@ -30,8 +45,12 @@ impl VethEndpoint {
) -> Result<Self> {
let net_pair = NetworkPair::new(handle, idx, name, model, queues)
.await
.context("new networkInterfacePair")?;
Ok(VethEndpoint { net_pair })
.context("new network interface pair failed.")?;
Ok(VethEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -42,9 +61,12 @@ impl VethEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -59,18 +81,17 @@ impl Endpoint for VethEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("add network model")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network Veth endpoint device failed.")?;
Ok(())
}
@@ -78,16 +99,19 @@ impl Endpoint for VethEndpoint {
self.net_pair
.del_network_model()
.await
.context("del network model")?;
.context("del network model failed.")?;
let config = self.get_network_config().context("get network config")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove Veth endpoint device by hypervisor failed.")?;
Ok(())
}
async fn save(&self) -> Option<EndpointState> {
Some(EndpointState {
veth_endpoint: Some(VethEndpointState {

64
src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
View File

@@ -4,25 +4,39 @@
// SPDX-License-Identifier: Apache-2.0
//
use std::io::{self, Error};
use std::{
io::{self, Error},
sync::Arc,
};
use anyhow::{Context, Result};
use async_trait::async_trait;
use hypervisor::device::DeviceType;
use hypervisor::NetworkDevice;
use tokio::sync::RwLock;
use hypervisor::{
device::{
device_manager::{do_handle_device, DeviceManager},
driver::NetworkConfig,
DeviceConfig, DeviceType,
},
Hypervisor, NetworkDevice,
};
use super::{
endpoint_persist::{EndpointState, VlanEndpointState},
Endpoint,
};
use crate::network::{network_model::TC_FILTER_NET_MODEL_STR, utils, NetworkPair};
use super::endpoint_persist::{EndpointState, VlanEndpointState};
use super::Endpoint;
use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
use crate::network::{utils, NetworkPair};
use hypervisor::{device::driver::NetworkConfig, Hypervisor};
#[derive(Debug)]
pub struct VlanEndpoint {
pub(crate) net_pair: NetworkPair,
pub(crate) d: Arc<RwLock<DeviceManager>>,
}
impl VlanEndpoint {
pub async fn new(
d: &Arc<RwLock<DeviceManager>>,
handle: &rtnetlink::Handle,
name: &str,
idx: u32,
@@ -30,8 +44,12 @@ impl VlanEndpoint {
) -> Result<Self> {
let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues)
.await
.context("error creating networkInterfacePair")?;
Ok(VlanEndpoint { net_pair })
.context("new network interface pair failed.")?;
Ok(VlanEndpoint {
net_pair,
d: d.clone(),
})
}
fn get_network_config(&self) -> Result<NetworkConfig> {
@@ -42,9 +60,12 @@ impl VlanEndpoint {
format!("hard_addr {}", &iface.hard_addr),
)
})?;
Ok(NetworkConfig {
host_dev_name: iface.name.clone(),
virt_iface_name: self.net_pair.virt_iface.name.clone(),
guest_mac: Some(guest_mac),
..Default::default()
})
}
}
@@ -59,18 +80,16 @@ impl Endpoint for VlanEndpoint {
self.net_pair.tap.tap_iface.hard_addr.clone()
}
async fn attach(&self, h: &dyn Hypervisor) -> Result<()> {
async fn attach(&self) -> Result<()> {
self.net_pair
.add_network_model()
.await
.context("error adding network model")?;
.context("add network model failed.")?;
let config = self.get_network_config().context("get network config")?;
h.add_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
}))
.await
.context("error adding device by hypervisor")?;
do_handle_device(&self.d, &DeviceConfig::NetworkCfg(config))
.await
.context("do handle network Vlan endpoint device failed.")?;
Ok(())
}
@@ -79,16 +98,17 @@ impl Endpoint for VlanEndpoint {
self.net_pair
.del_network_model()
.await
.context("error deleting network model")?;
.context("delete network model failed.")?;
let config = self
.get_network_config()
.context("error getting network config")?;
.context("get network config failed.")?;
h.remove_device(DeviceType::Network(NetworkDevice {
id: self.net_pair.virt_iface.name.clone(),
config,
..Default::default()
}))
.await
.context("error removing device by hypervisor")?;
.context("remove Vlan endpoint device by hypervisor failed.")?;
Ok(())
}

2
src/runtime-rs/crates/resource/src/network/mod.rs
View File

@@ -35,7 +35,7 @@ pub enum NetworkConfig {
#[async_trait]
pub trait Network: Send + Sync {
async fn setup(&self, h: &dyn Hypervisor) -> Result<()>;
async fn setup(&self) -> Result<()>;
async fn interfaces(&self) -> Result<Vec<agent::Interface>>;
async fn routes(&self) -> Result<Vec<agent::Route>>;
async fn neighs(&self) -> Result<Vec<agent::ARPNeighbor>>;

1
src/runtime-rs/crates/resource/src/network/network_pair.rs
View File

@@ -39,6 +39,7 @@ pub struct NetworkPair {
pub model: Arc<dyn network_model::NetworkModel>,
pub network_qos: bool,
}
impl NetworkPair {
pub(crate) async fn new(
handle: &rtnetlink::Handle,

10
src/runtime-rs/crates/resource/src/network/network_with_netns.rs
View File

@@ -82,11 +82,11 @@ impl NetworkWithNetns {
#[async_trait]
impl Network for NetworkWithNetns {
async fn setup(&self, h: &dyn Hypervisor) -> Result<()> {
async fn setup(&self) -> Result<()> {
let inner = self.inner.read().await;
let _netns_guard = netns::NetnsGuard::new(&inner.netns_path).context("net netns guard")?;
for e in &inner.entity_list {
e.endpoint.attach(h).await.context("attach")?;
e.endpoint.attach().await.context("attach")?;
}
Ok(())
}
@@ -225,6 +225,7 @@ async fn create_endpoint(
match link_type {
"veth" => {
let ret = VethEndpoint::new(
&d,
handle,
&attrs.name,
idx,
@@ -236,19 +237,20 @@ async fn create_endpoint(
Arc::new(ret)
}
"vlan" => {
let ret = VlanEndpoint::new(handle, &attrs.name, idx, config.queues)
let ret = VlanEndpoint::new(&d, handle, &attrs.name, idx, config.queues)
.await
.context("vlan endpoint")?;
Arc::new(ret)
}
"ipvlan" => {
let ret = IPVlanEndpoint::new(handle, &attrs.name, idx, config.queues)
let ret = IPVlanEndpoint::new(&d, handle, &attrs.name, idx, config.queues)
.await
.context("ipvlan endpoint")?;
Arc::new(ret)
}
"macvlan" => {
let ret = MacVlanEndpoint::new(
&d,
handle,
&attrs.name,
idx,