Merge pull request #5875 from Ji-Xinyou/xyji/refactor-shim-mgmt

refactor(shim-mgmt): move client side to libs

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -28,6 +28,7 @@ rand = "0.8.4"
 kata-sys-util = { path = "../../../libs/kata-sys-util" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging" }
+shim-interface = { path = "../../../libs/shim-interface" }
 
 dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs"] }
 

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -20,7 +20,8 @@ use kata_types::{
     capabilities::{Capabilities, CapabilityBits},
     config::hypervisor::Hypervisor as HypervisorConfig,
 };
-use persist::{sandbox_persist::Persist, KATA_PATH};
+use persist::sandbox_persist::Persist;
+use shim_interface::KATA_PATH;
 use std::{collections::HashSet, fs::create_dir_all, path::PathBuf};
 
 const DRAGONBALL_KERNEL: &str = "vmlinux";

src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs

@@ -14,7 +14,7 @@ use kata_types::capabilities::Capabilities;
 
 use super::inner::DragonballInner;
 use crate::{utils, VcpuThreadIds, VmmState};
-use persist::KATA_PATH;
+use shim_interface::KATA_PATH;
 const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
 
 fn get_vsock_path(root: &str) -> String {

src/runtime-rs/crates/persist/Cargo.toml

@@ -10,6 +10,7 @@ async-trait = "0.1.48"
 anyhow = "^1.0"
 kata-sys-util = { path = "../../../libs/kata-sys-util"}
 kata-types = { path = "../../../libs/kata-types" }
+shim-interface = { path = "../../../libs/shim-interface" }
 libc = "0.2"
 serde = { version = "1.0.138", features = ["derive"] }
 serde_json = "1.0.82"

src/runtime-rs/crates/persist/src/lib.rs

@@ -7,9 +7,9 @@
 pub mod sandbox_persist;
 use anyhow::{anyhow, Context, Ok, Result};
 use serde::de;
+use shim_interface::KATA_PATH;
 use std::{fs::File, io::BufReader};
 
-pub const KATA_PATH: &str = "/run/kata";
 pub const PERSIST_FILE: &str = "state.json";
 use kata_sys_util::validate::verify_id;
 use safe_path::scoped_join;

src/runtime-rs/crates/runtimes/Cargo.toml

@@ -18,6 +18,7 @@ common = { path = "./common" }
 kata-types = { path = "../../../libs/kata-types" }
 logging = { path = "../../../libs/logging"}
 oci = { path = "../../../libs/oci" }
+shim-interface = { path = "../../../libs/shim-interface" }
 persist = { path = "../persist" }
 hypervisor = { path = "../hypervisor" }
 # runtime handler

src/runtime-rs/crates/runtimes/src/lib.rs

@@ -11,6 +11,6 @@ logging::logger_with_subsystem!(sl, "runtimes");
 
 pub mod manager;
 pub use manager::RuntimeHandlerManager;
+pub use shim_interface;
 mod shim_mgmt;
-pub use shim_mgmt::{client::MgmtClient, server::sb_storage_path};
 mod static_resource;

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -19,6 +19,7 @@ use kata_types::{annotations::Annotation, config::TomlConfig};
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
 use persist::sandbox_persist::Persist;
+use shim_interface::shim_mgmt::ERR_NO_SHIM_SERVER;
 use tokio::sync::{mpsc::Sender, RwLock};
 #[cfg(feature = "virt")]
 use virt_container::{
@@ -117,7 +118,9 @@ impl RuntimeHandlerManagerInner {
         let shim_mgmt_svr = MgmtServer::new(
             &self.id,
             self.runtime_instance.as_ref().unwrap().sandbox.clone(),
-        );
+        )
+        .context(ERR_NO_SHIM_SERVER)?;
+
         tokio::task::spawn(Arc::new(shim_mgmt_svr).run());
         info!(sl!(), "shim management http server starts");
 

src/runtime-rs/crates/runtimes/src/shim_mgmt/client.rs

@@ -1,79 +0,0 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-// Defines the general client functions used by other components acting like
-// clients. To be specific, a client first connect to the socket, then send
-// request to destined URL, and finally handle the request(or not)
-
-use std::{path::Path, path::PathBuf, time::Duration};
-
-use super::server::mgmt_socket_addr;
-use anyhow::{anyhow, Context, Result};
-use hyper::{Body, Client, Method, Request, Response};
-use hyperlocal::{UnixClientExt, UnixConnector, Uri};
-
-/// Shim management client with timeout
-pub struct MgmtClient {
-    /// The socket *file path* on host file system
-    sock_path: PathBuf,
-
-    /// The http client connect to the long standing shim mgmt server
-    client: Client<UnixConnector, Body>,
-
-    /// Timeout value for each dial, usually 200ms will be enough
-    /// For heavier workload, you may want longer timeout
-    timeout: Option<Duration>,
-}
-
-impl MgmtClient {
-    /// Construct a new client connecting to shim mgmt server
-    pub fn new(sid: String, timeout: Option<Duration>) -> Result<Self> {
-        let unix_socket_path = mgmt_socket_addr(sid);
-        let s_addr = unix_socket_path
-            .strip_prefix("unix:")
-            .context("failed to strix prefix")?;
-        let sock_path = Path::new("/").join(s_addr).as_path().to_owned();
-        let client = Client::unix();
-        Ok(Self {
-            sock_path,
-            client,
-            timeout,
-        })
-    }
-
-    /// The http GET method for client, return a raw response. Further handling should be done by caller.
-    /// Parameter uri should be like "/agent-url" etc.
-    pub async fn get(&self, uri: &str) -> Result<Response<Body>> {
-        let url: hyper::Uri = Uri::new(&self.sock_path, uri).into();
-        let work = self.client.get(url);
-        match self.timeout {
-            Some(timeout) => match tokio::time::timeout(timeout, work).await {
-                Ok(result) => result.map_err(|e| anyhow!(e)),
-                Err(_) => Err(anyhow!("TIMEOUT")),
-            },
-            // if timeout not set, work executes directly
-            None => work.await.context("failed to GET"),
-        }
-    }
-
-    /// The http PUT method for client
-    pub async fn put(&self, uri: &str, data: Vec<u8>) -> Result<Response<Body>> {
-        let url: hyper::Uri = Uri::new(&self.sock_path, uri).into();
-        let request = Request::builder()
-            .method(Method::PUT)
-            .uri(url)
-            .body(Body::from(data))
-            .unwrap();
-        let work = self.client.request(request);
-        match self.timeout {
-            Some(timeout) => match tokio::time::timeout(timeout, work).await {
-                Ok(result) => result.map_err(|e| anyhow!(e)),
-                Err(_) => Err(anyhow!("TIMEOUT")),
-            },
-            None => work.await.context("failed to PUT"),
-        }
-    }
-}

src/runtime-rs/crates/runtimes/src/shim_mgmt/handlers.rs

@@ -12,7 +12,7 @@ use common::Sandbox;
 use hyper::{Body, Method, Request, Response, StatusCode};
 use std::sync::Arc;
 
-use super::server::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL};
+use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL};
 
 // main router for response, this works as a multiplexer on
 // http arrival which invokes the corresponding handler function

src/runtime-rs/crates/runtimes/src/shim_mgmt/mod.rs

@@ -4,6 +4,11 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-pub mod client;
+//! The server side of shim management implementation, receive HTTP
+//! requests and multiplex them to corresponding functions inside shim
+//!
+//! To call services in a RESTful convention, use the client
+//! from libs/shim-interface library
+
 mod handlers;
 pub mod server;

src/runtime-rs/crates/runtimes/src/shim_mgmt/server.rs

@@ -1,5 +1,5 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
+// Copyright (c) 2022 Alibaba Cloud
+// Copyright (c) 2022 Ant Group
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -16,21 +16,11 @@ use std::{fs, path::Path, sync::Arc};
 use anyhow::{Context, Result};
 use common::Sandbox;
 use hyper::{server::conn::Http, service::service_fn};
-use persist::KATA_PATH;
+use shim_interface::{mgmt_socket_addr, shim_mgmt::ERR_NO_SHIM_SERVER};
 use tokio::net::UnixListener;
 
 use super::handlers::handler_mux;
 
-pub(crate) const DIRECT_VOLUMN_PATH_KEY: &str = "path";
-pub(crate) const DIRECT_VOLUMN_STATS_URL: &str = "/direct-volumn/stats";
-pub(crate) const DIRECT_VOLUMN_RESIZE_URL: &str = "/direct-volumn/resize";
-pub(crate) const AGENT_URL: &str = "/agent-url";
-pub(crate) const IP_TABLE_URL: &str = "/iptables";
-pub(crate) const IP6_TABLE_URL: &str = "/ip6tables";
-pub(crate) const METRICS_URL: &str = "/metrics";
-
-const SHIM_MGMT_SOCK_NAME: &str = "shim-monitor.sock";
-
 /// The shim management server instance
 pub struct MgmtServer {
     /// socket address(with prefix like hvsock://)
@@ -42,11 +32,11 @@ pub struct MgmtServer {
 
 impl MgmtServer {
     /// construct a new management server
-    pub fn new(sid: &str, sandbox: Arc<dyn Sandbox>) -> Self {
-        Self {
-            s_addr: mgmt_socket_addr(sid.to_owned()),
+    pub fn new(sid: &str, sandbox: Arc<dyn Sandbox>) -> Result<Self> {
+        Ok(Self {
+            s_addr: mgmt_socket_addr(sid).context(ERR_NO_SHIM_SERVER)?,
             sandbox,
-        }
+        })
     }
 
     // TODO(when metrics is supported): write metric addresses to fs
@@ -75,21 +65,6 @@ impl MgmtServer {
     }
 }
 
-// return sandbox's storage path
-pub fn sb_storage_path() -> String {
-    String::from(KATA_PATH)
-}
-
-// returns the address of the unix domain socket(UDS) for communication with shim
-// management service using http
-// normally returns "unix:///run/kata/{sid}/shim_monitor.sock"
-pub fn mgmt_socket_addr(sid: String) -> String {
-    let p = Path::new(&sb_storage_path())
-        .join(sid)
-        .join(SHIM_MGMT_SOCK_NAME);
-    format!("unix://{}", p.to_string_lossy())
-}
-
 // from path, return a unix listener corresponding to that path,
 // if the path(socket file) is not created, we create that here
 async fn listener_from_path(path: String) -> Result<UnixListener> {
@@ -104,15 +79,3 @@ async fn listener_from_path(path: String) -> Result<UnixListener> {
     info!(sl!(), "mgmt-svr: binding to path {}", path);
     UnixListener::bind(file_path).context("bind address")
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn mgmt_svr_test_sock_addr() {
-        let sid = String::from("414123");
-        let addr = mgmt_socket_addr(sid);
-        assert_eq!(addr, "unix:///run/kata/414123/shim-monitor.sock");
-    }
-}

src/runtime-rs/crates/service/Cargo.toml

@@ -16,5 +16,6 @@ ttrpc = { version = "0.6.1" }
 common = { path = "../runtimes/common" }
 containerd-shim-protos = { version = "0.2.0", features = ["async"]}
 logging = { path = "../../../libs/logging"}
+shim-interface = { path = "../../../libs/shim-interface" }
 runtimes = { path = "../runtimes" }
 persist = { path = "../persist" }

src/runtime-rs/crates/service/src/manager.rs

@@ -28,7 +28,7 @@ use ttrpc::asynchronous::Server;
 use crate::task_service::TaskService;
 /// message buffer size
 const MESSAGE_BUFFER_SIZE: usize = 8;
-use persist::KATA_PATH;
+use shim_interface::KATA_PATH;
 
 pub struct ServiceManager {
     receiver: Option<Receiver<Message>>,