Merge pull request #5788 from openanolis/runtime-rs-ocihook

runtime-rs: add oci hook support

docs/design/README.md

@@ -11,6 +11,7 @@ Kata Containers design documents:
 - [Host cgroups](host-cgroups.md)
 - [Agent systemd cgroup](agent-systemd-cgroup.md)
 - [`Inotify` support](inotify.md)
+- [`Hooks` support](hooks-handling.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
 - [Design for direct-assigned volume](direct-blk-device-assignment.md)

docs/design/hooks-handling.md

@@ -0,0 +1,63 @@
+# Kata Containers support for `Hooks`
+
+## Introduction
+
+During container's lifecycle, different Hooks can be executed to do custom actions. In Kata Containers, we support two types of Hooks, `OCI Hooks` and `Kata Hooks`.
+
+### OCI Hooks
+
+The OCI Spec stipulates six hooks that can be executed at different time points and namespaces, including `Prestart Hooks`, `CreateRuntime Hooks`, `CreateContainer Hooks`, `StartContainer Hooks`, `Poststart Hooks` and `Poststop Hooks`. We support these types of Hooks as compatible as possible in Kata Containers.
+
+The path and arguments of these hooks will be passed to Kata for execution via `bundle/config.json`. For example:
+```
+...
+"hooks": {
+  "prestart": [
+    {
+      "path": "/usr/bin/prestart-hook",
+      "args": ["prestart-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ],
+  "createRuntime": [
+    {
+      "path": "/usr/bin/createRuntime-hook",
+      "args": ["createRuntime-hook", "arg1", "arg2"],
+      "env":  [ "key1=value1"]
+    }
+  ]
+}
+...
+```
+
+### Kata Hooks
+
+In Kata, we support another three kinds of hooks executed in guest VM, including `Guest Prestart Hook`, `Guest Poststart Hook`, `Guest Poststop Hook`.
+
+The executable files for Kata Hooks must be packaged in the *guest rootfs*. The file path to those guest hooks should be specified in the configuration file, and guest hooks must be stored in a subdirectory of `guest_hook_path` according to their hook type. For example:
+
++ In configuration file:
+```
+guest_hook_path="/usr/share/hooks"
+```
++ In guest rootfs, prestart-hook is stored in `/usr/share/hooks/prestart/prestart-hook`.
+
+## Execution
+The table below summarized when and where those different hooks will be executed in Kata Containers:
+
+| Hook Name | Hook Type | Hook Path | Exec Place | Exec Time |
+|---|---|---|---|---|
+| `Prestart(deprecated)` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created. |
+| `CreateRuntime` | OCI hook | host runtime namespace | host runtime namespace | After VM is started, before container is created, after `Prestart` hooks. |
+| `CreateContainer` | OCI hook | host runtime namespace | host vmm namespace* | After VM is started, before container is created, after `CreateRuntime` hooks. |
+| `StartContainer` | OCI hook | guest container namespace | guest container namespace | After container is created, before container is started. |
+| `Poststart` | OCI hook | host runtime namespace | host runtime namespace | After container is started, before start operation returns. |
+| `Poststop` | OCI hook | host runtime namespace | host runtime namespace | After container is deleted, before delete operation returns. |
+| `Guest Prestart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, before container command is executed. |
+| `Guest Poststart` | Kata hook | guest agent namespace | guest agent namespace | During start operation, after container command is executed, before start operation returns. |
+| `Guest Poststop` | Kata hook | guest agent namespace | guest agent namespace | During delete operation, after container is deleted, before delete operation returns. |
+
++ `Hook Path` specifies where hook's path be resolved.
++ `Exec Place` specifies in which namespace those hooks can be executed.
+  + For `CreateContainer` Hooks, OCI requires to run them inside the container namespace while the hook executable path is in the host runtime, which is a non-starter for VM-based containers. So we design to keep them running in the *host vmm namespace.* 
++ `Exec Time` specifies at which time point those hooks can be executed.

src/agent/rustjail/src/container.rs

@@ -374,13 +374,18 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
     let spec: oci::Spec = serde_json::from_str(spec_str)?;
-
     log_child!(cfd_log, "notify parent to send oci process");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
     let buf = read_sync(crfd)?;
     let process_str = std::str::from_utf8(&buf)?;
     let oci_process: oci::Process = serde_json::from_str(process_str)?;
+    log_child!(cfd_log, "notify parent to send oci state");
+    write_sync(cwfd, SYNC_SUCCESS, "")?;
+
+    let buf = read_sync(crfd)?;
+    let state_str = std::str::from_utf8(&buf)?;
+    let mut state: oci::State = serde_json::from_str(state_str)?;
     log_child!(cfd_log, "notify parent to send cgroup manager");
     write_sync(cwfd, SYNC_SUCCESS, "")?;
 
@@ -743,6 +748,19 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         unistd::read(fd, buf)?;
     }
 
+    if init {
+        // StartContainer Hooks:
+        // * should be run in container namespace
+        // * should be run after container is created and before container is started (before user-specific command is executed)
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#startcontainer-hooks
+        state.pid = std::process::id() as i32;
+        state.status = oci::ContainerState::Created;
+        if let Some(hooks) = spec.hooks.as_ref() {
+            let mut start_container_states = HookStates::new();
+            start_container_states.execute_hooks(&hooks.start_container, Some(state))?;
+        }
+    }
+
     // With NoNewPrivileges, we should set seccomp as close to
     // do_exec as possible in order to reduce the amount of
     // system calls in the seccomp profiles.
@@ -1323,7 +1341,6 @@ async fn join_namespaces(
     write_async(pipe_w, SYNC_DATA, spec_str.as_str()).await?;
 
     info!(logger, "wait child received oci spec");
-
     read_async(pipe_r).await?;
 
     info!(logger, "send oci process from parent to child");
@@ -1333,6 +1350,13 @@ async fn join_namespaces(
     info!(logger, "wait child received oci process");
     read_async(pipe_r).await?;
 
+    info!(logger, "try to send state from parent to child");
+    let state_str = serde_json::to_string(st)?;
+    write_async(pipe_w, SYNC_DATA, state_str.as_str()).await?;
+
+    info!(logger, "wait child received oci state");
+    read_async(pipe_r).await?;
+
     let cm_str = if use_systemd_cgroup {
         serde_json::to_string(cm.as_any()?.downcast_ref::<SystemdManager>().unwrap())
     } else {

src/agent/rustjail/src/lib.rs

@@ -153,13 +153,17 @@ fn hook_grpc_to_oci(h: &[grpcHook]) -> Vec<oci::Hook> {
 
 fn hooks_grpc_to_oci(h: &grpc::Hooks) -> oci::Hooks {
     let prestart = hook_grpc_to_oci(h.Prestart.as_ref());
-
+    let create_runtime = hook_grpc_to_oci(h.CreateRuntime.as_ref());
+    let create_container = hook_grpc_to_oci(h.CreateContainer.as_ref());
+    let start_container = hook_grpc_to_oci(h.StartContainer.as_ref());
     let poststart = hook_grpc_to_oci(h.Poststart.as_ref());
-
     let poststop = hook_grpc_to_oci(h.Poststop.as_ref());
 
     oci::Hooks {
         prestart,
+        create_runtime,
+        create_container,
+        start_container,
         poststart,
         poststop,
     }
@@ -831,6 +835,45 @@ mod tests {
                         Timeout: 10,
                         ..Default::default()
                     }])),
+                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -860,6 +903,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
             TestData {
@@ -892,6 +953,45 @@ mod tests {
                         Timeout: 10,
                         ..Default::default()
                     }])),
+                    CreateRuntime: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createruntimepath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    CreateContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("createcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
+                    StartContainer: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
+                        Path: String::from("startcontainerpath"),
+                        Args: protobuf::RepeatedField::from(Vec::from([
+                            String::from("arg1"),
+                            String::from("arg2"),
+                        ])),
+                        Env: protobuf::RepeatedField::from(Vec::from([
+                            String::from("env1"),
+                            String::from("env2"),
+                        ])),
+                        Timeout: 10,
+                        ..Default::default()
+                    }])),
                     ..Default::default()
                 },
                 result: oci::Hooks {
@@ -908,6 +1008,24 @@ mod tests {
                         env: Vec::from([String::from("env1"), String::from("env2")]),
                         timeout: Some(10),
                     }]),
+                    create_runtime: Vec::from([oci::Hook {
+                        path: String::from("createruntimepath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    create_container: Vec::from([oci::Hook {
+                        path: String::from("createcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
+                    start_container: Vec::from([oci::Hook {
+                        path: String::from("startcontainerpath"),
+                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                        env: Vec::from([String::from("env1"), String::from("env2")]),
+                        timeout: Some(10),
+                    }]),
                 },
             },
         ];

src/dragonball/src/api/v1/instance_info.rs

@@ -50,6 +50,8 @@ pub struct InstanceInfo {
     pub vmm_version: String,
     /// The pid of the current VMM process.
     pub pid: u32,
+    /// The tid of the current VMM master thread.
+    pub master_tid: u32,
     /// The state of async actions.
     pub async_state: AsyncState,
     /// List of tids of vcpu threads (vcpu index, tid)
@@ -66,6 +68,7 @@ impl InstanceInfo {
             state: InstanceState::Uninitialized,
             vmm_version,
             pid: std::process::id(),
+            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
@@ -80,6 +83,7 @@ impl Default for InstanceInfo {
             state: InstanceState::Uninitialized,
             vmm_version: env!("CARGO_PKG_VERSION").to_string(),
             pid: std::process::id(),
+            master_tid: 0,
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,

src/libs/oci/src/lib.rs

@@ -193,6 +193,12 @@ pub struct Hooks {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub prestart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub create_runtime: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub create_container: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub start_container: Vec<Hook>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststart: Vec<Hook>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub poststop: Vec<Hook>,
@@ -1401,6 +1407,7 @@ mod tests {
                     env: vec![],
                     timeout: None,
                 }],
+                ..Default::default()
             }),
             annotations: [
                 ("com.example.key1".to_string(), "value1".to_string()),

src/libs/protocols/protos/oci.proto

@@ -166,6 +166,15 @@ message Hooks {
 
 	// Poststop is a list of hooks to be run after the container process exits.
 	repeated Hook Poststop = 3  [(gogoproto.nullable) = false];
+
+	// Createruntime is a list of hooks to be run during the creation of runtime(sandbox).
+	repeated Hook CreateRuntime = 4  [(gogoproto.nullable) = false];
+
+	// CreateContainer is a list of hooks to be run after VM is started, and before container is created.
+	repeated Hook CreateContainer = 5  [(gogoproto.nullable) = false];
+
+	// StartContainer is a list of hooks to be run after container is created, but before it is started.
+	repeated Hook StartContainer = 6  [(gogoproto.nullable) = false];
 }
 
 message Hook {

src/libs/protocols/src/trans.rs

@@ -294,6 +294,9 @@ impl From<oci::Hooks> for crate::oci::Hooks {
     fn from(from: Hooks) -> Self {
         crate::oci::Hooks {
             Prestart: from_vec(from.prestart),
+            CreateRuntime: from_vec(from.create_runtime),
+            CreateContainer: from_vec(from.create_container),
+            StartContainer: from_vec(from.start_container),
             Poststart: from_vec(from.poststart),
             Poststop: from_vec(from.poststop),
             unknown_fields: Default::default(),
@@ -970,20 +973,34 @@ impl From<crate::oci::Hook> for oci::Hook {
 
 impl From<crate::oci::Hooks> for oci::Hooks {
     fn from(mut from: crate::oci::Hooks) -> Self {
-        let mut prestart = Vec::new();
+        let prestart = from.take_Prestart().into_iter().map(|i| i.into()).collect();
-        for hook in from.take_Prestart().to_vec() {
+        let create_runtime = from
-            prestart.push(hook.into())
+            .take_CreateRuntime()
-        }
+            .into_iter()
-        let mut poststart = Vec::new();
+            .map(|i| i.into())
-        for hook in from.take_Poststart().to_vec() {
+            .collect();
-            poststart.push(hook.into());
+        let create_container = from
-        }
+            .take_CreateContainer()
-        let mut poststop = Vec::new();
+            .into_iter()
-        for hook in from.take_Poststop().to_vec() {
+            .map(|i| i.into())
-            poststop.push(hook.into());
+            .collect();
-        }
+        let start_container = from
+            .take_StartContainer()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let poststart = from
+            .take_Poststart()
+            .into_iter()
+            .map(|i| i.into())
+            .collect();
+        let poststop = from.take_Poststop().into_iter().map(|i| i.into()).collect();
+
         oci::Hooks {
             prestart,
+            create_runtime,
+            create_container,
+            start_container,
             poststart,
             poststop,
         }

src/runtime-rs/Cargo.lock

@@ -1768,6 +1768,8 @@ dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
  "libc",
+ "memoffset 0.6.5",
+ "pin-utils",
 ]
 
 [[package]]
@@ -2455,12 +2457,15 @@ dependencies = [
  "hyper",
  "hyperlocal",
  "hypervisor",
+ "kata-sys-util",
  "kata-types",
  "lazy_static",
  "linux_container",
  "logging",
+ "nix 0.25.1",
  "oci",
  "persist",
+ "serde_json",
  "shim-interface",
  "slog",
  "slog-scope",

src/runtime-rs/crates/agent/src/types.rs

@@ -124,7 +124,6 @@ pub struct CreateContainerRequest {
     pub devices: Vec<Device>,
     pub storages: Vec<Storage>,
     pub oci: Option<oci::Spec>,
-    pub guest_hooks: Option<oci::Hooks>,
     pub sandbox_pidns: bool,
     pub rootfs_mounts: Vec<oci::Mount>,
 }

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -472,6 +472,10 @@ impl CloudHypervisorInner {
         Ok(Vec::<u32>::new())
     }
 
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        todo!()
+    }
+
     pub(crate) async fn check(&self) -> Result<()> {
         Ok(())
     }

src/runtime-rs/crates/hypervisor/src/ch/mod.rs

@@ -118,6 +118,11 @@ impl Hypervisor for CloudHypervisor {
         inner.get_pids().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
     async fn check(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.check().await

src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs

@@ -127,6 +127,11 @@ impl DragonballInner {
         Ok(Vec::from_iter(pids.into_iter()))
     }
 
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let master_tid = self.vmm_instance.get_vmm_master_tid();
+        Ok(master_tid)
+    }
+
     pub(crate) async fn check(&self) -> Result<()> {
         Ok(())
     }

src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs

@@ -117,6 +117,11 @@ impl Hypervisor for Dragonball {
         inner.get_pids().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
     async fn check(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.check().await

src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs

@@ -75,6 +75,12 @@ impl VmmInstance {
         share_info_lock.write().unwrap().id = String::from(id);
     }
 
+    pub fn get_vmm_master_tid(&self) -> u32 {
+        let info = self.vmm_shared_info.clone();
+        let result = info.read().unwrap().master_tid;
+        result
+    }
+
     pub fn get_vcpu_tids(&self) -> Vec<(u8, u32)> {
         let info = self.vmm_shared_info.clone();
         let result = info.read().unwrap().tids.clone();
@@ -103,6 +109,7 @@ impl VmmInstance {
             Some(kvm.into_raw_fd()),
         )
         .expect("Failed to start vmm");
+        let vmm_shared_info = self.get_shared_info();
 
         self.vmm_thread = Some(
             thread::Builder::new()
@@ -110,6 +117,9 @@ impl VmmInstance {
                 .spawn(move || {
                     || -> Result<i32> {
                         debug!(sl!(), "run vmm thread start");
+                        let cur_tid = nix::unistd::gettid().as_raw() as u32;
+                        vmm_shared_info.write().unwrap().master_tid = cur_tid;
+
                         if let Some(netns_path) = netns {
                             info!(sl!(), "set netns for vmm master {}", &netns_path);
                             let netns_fd = File::open(&netns_path)

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -87,6 +87,7 @@ pub trait Hypervisor: Send + Sync {
     async fn hypervisor_config(&self) -> HypervisorConfig;
     async fn get_thread_ids(&self) -> Result<VcpuThreadIds>;
     async fn get_pids(&self) -> Result<Vec<u32>>;
+    async fn get_vmm_master_tid(&self) -> Result<u32>;
     async fn cleanup(&self) -> Result<()>;
     async fn check(&self) -> Result<()>;
     async fn get_jailer_root(&self) -> Result<String>;

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -89,6 +89,11 @@ impl QemuInner {
         todo!()
     }
 
+    pub(crate) async fn get_vmm_master_tid(&self) -> Result<u32> {
+        info!(sl!(), "QemuInner::get_vmm_master_tid()");
+        todo!()
+    }
+
     pub(crate) async fn cleanup(&self) -> Result<()> {
         info!(sl!(), "QemuInner::cleanup()");
         todo!()

src/runtime-rs/crates/hypervisor/src/qemu/mod.rs

@@ -103,6 +103,11 @@ impl Hypervisor for Qemu {
         inner.get_thread_ids().await
     }
 
+    async fn get_vmm_master_tid(&self) -> Result<u32> {
+        let inner = self.inner.read().await;
+        inner.get_vmm_master_tid().await
+    }
+
     async fn cleanup(&self) -> Result<()> {
         let inner = self.inner.read().await;
         inner.cleanup().await

src/runtime-rs/crates/resource/src/network/mod.rs

@@ -5,6 +5,7 @@
 //
 
 mod endpoint;
+pub use endpoint::endpoint_persist::EndpointState;
 pub use endpoint::Endpoint;
 mod network_entity;
 mod network_info;
@@ -17,7 +18,7 @@ use network_with_netns::NetworkWithNetns;
 mod network_pair;
 use network_pair::NetworkPair;
 mod utils;
-pub use endpoint::endpoint_persist::EndpointState;
+pub use utils::netns::NetnsGuard;
 
 use std::sync::Arc;
 

src/runtime-rs/crates/resource/src/network/utils/netns.rs

@@ -10,12 +10,12 @@ use anyhow::{Context, Result};
 use nix::sched::{setns, CloneFlags};
 use nix::unistd::{getpid, gettid};
 
-pub(crate) struct NetnsGuard {
+pub struct NetnsGuard {
     old_netns: Option<File>,
 }
 
 impl NetnsGuard {
-    pub(crate) fn new(new_netns_path: &str) -> Result<Self> {
+    pub fn new(new_netns_path: &str) -> Result<Self> {
         let old_netns = if !new_netns_path.is_empty() {
             let current_netns_path = format!("/proc/{}/task/{}/ns/{}", getpid(), gettid(), "net");
             let old_netns = File::open(&current_netns_path)

src/runtime-rs/crates/runtimes/Cargo.toml

@@ -13,9 +13,12 @@ slog-scope = "4.4.0"
 tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
 hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
 hyperlocal = "0.8"
+serde_json = "1.0.88"
+nix = "0.25.0"
 
 common = { path = "./common" }
 kata-types = { path = "../../../libs/kata-types" }
+kata-sys-util = { path = "../../../libs/kata-sys-util" }
 logging = { path = "../../../libs/logging"}
 oci = { path = "../../../libs/oci" }
 shim-interface = { path = "../../../libs/shim-interface" }

src/runtime-rs/crates/runtimes/common/Cargo.toml

@@ -26,3 +26,4 @@ agent = { path = "../../agent" }
 kata-sys-util = { path = "../../../../libs/kata-sys-util" }
 kata-types = { path = "../../../../libs/kata-types" }
 oci = { path = "../../../../libs/oci" }
+

src/runtime-rs/crates/runtimes/common/src/sandbox.rs

@@ -9,7 +9,13 @@ use async_trait::async_trait;
 
 #[async_trait]
 pub trait Sandbox: Send + Sync {
-    async fn start(&self, netns: Option<String>, dns: Vec<String>) -> Result<()>;
+    async fn start(
+        &self,
+        netns: Option<String>,
+        dns: Vec<String>,
+        spec: &oci::Spec,
+        state: &oci::State,
+    ) -> Result<()>;
     async fn stop(&self) -> Result<()>;
     async fn cleanup(&self) -> Result<()>;
     async fn shutdown(&self) -> Result<()>;

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -18,6 +18,7 @@ use hypervisor::Param;
 use kata_types::{
     annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig,
 };
+
 #[cfg(feature = "linux")]
 use linux_container::LinuxContainer;
 use persist::sandbox_persist::Persist;
@@ -50,6 +51,8 @@ impl RuntimeHandlerManagerInner {
 
     async fn init_runtime_handler(
         &mut self,
+        spec: &oci::Spec,
+        state: &oci::State,
         netns: Option<String>,
         dns: Vec<String>,
         config: Arc<TomlConfig>,
@@ -74,14 +77,19 @@ impl RuntimeHandlerManagerInner {
         // start sandbox
         runtime_instance
             .sandbox
-            .start(netns, dns)
+            .start(netns, dns, spec, state)
             .await
             .context("start sandbox")?;
         self.runtime_instance = Some(Arc::new(runtime_instance));
         Ok(())
     }
 
-    async fn try_init(&mut self, spec: &oci::Spec, options: &Option<Vec<u8>>) -> Result<()> {
+    async fn try_init(
+        &mut self,
+        spec: &oci::Spec,
+        state: &oci::State,
+        options: &Option<Vec<u8>>,
+    ) -> Result<()> {
         // return if runtime instance has init
         if self.runtime_instance.is_some() {
             return Ok(());
@@ -121,7 +129,7 @@ impl RuntimeHandlerManagerInner {
         }
 
         let config = load_config(spec, options).context("load config")?;
-        self.init_runtime_handler(netns, dns, Arc::new(config))
+        self.init_runtime_handler(spec, state, netns, dns, Arc::new(config))
             .await
             .context("init runtime handler")?;
 
@@ -207,10 +215,11 @@ impl RuntimeHandlerManager {
     async fn try_init_runtime_instance(
         &self,
         spec: &oci::Spec,
+        state: &oci::State,
         options: &Option<Vec<u8>>,
     ) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.try_init(spec, options).await
+        inner.try_init(spec, state, options).await
     }
 
     pub async fn handler_message(&self, req: Request) -> Result<Response> {
@@ -222,8 +231,16 @@ impl RuntimeHandlerManager {
                 oci::OCI_SPEC_CONFIG_FILE_NAME
             );
             let spec = oci::Spec::load(&bundler_path).context("load spec")?;
+            let state = oci::State {
+                version: spec.version.clone(),
+                id: container_config.container_id.to_string(),
+                status: oci::ContainerState::Creating,
+                pid: 0,
+                bundle: bundler_path,
+                annotations: spec.annotations.clone(),
+            };
 
-            self.try_init_runtime_instance(&spec, &container_config.options)
+            self.try_init_runtime_instance(&spec, &state, &container_config.options)
                 .await
                 .context("try init runtime instance")?;
             let instance = self

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs

@@ -37,6 +37,7 @@ pub struct Container {
     pid: u32,
     pub container_id: ContainerID,
     config: ContainerConfig,
+    spec: oci::Spec,
     inner: Arc<RwLock<ContainerInner>>,
     agent: Arc<dyn Agent>,
     resource_manager: Arc<ResourceManager>,
@@ -47,6 +48,7 @@ impl Container {
     pub fn new(
         pid: u32,
         config: ContainerConfig,
+        spec: oci::Spec,
         agent: Arc<dyn Agent>,
         resource_manager: Arc<ResourceManager>,
     ) -> Result<Self> {
@@ -67,6 +69,7 @@ impl Container {
             pid,
             container_id,
             config,
+            spec,
             inner: Arc::new(RwLock::new(ContainerInner::new(
                 agent.clone(),
                 init_process,
@@ -382,11 +385,31 @@ impl Container {
             .context("agent update container")?;
         Ok(())
     }
+
+    pub async fn config(&self) -> ContainerConfig {
+        self.config.clone()
+    }
+
+    pub async fn spec(&self) -> oci::Spec {
+        self.spec.clone()
+    }
 }
 
 fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> {
-    // hook should be done on host
+    // Only the StartContainer hook needs to be reserved for execution in the guest
-    spec.hooks = None;
+    let start_container_hooks = match spec.hooks.as_ref() {
+        Some(hooks) => hooks.start_container.clone(),
+        None => Vec::new(),
+    };
+
+    spec.hooks = if start_container_hooks.is_empty() {
+        None
+    } else {
+        Some(oci::Hooks {
+            start_container: start_container_hooks,
+            ..Default::default()
+        })
+    };
 
     // special process K8s ephemeral volumes.
     update_ephemeral_storage_type(spec);

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs

@@ -5,11 +5,10 @@
 //
 
 use anyhow::{anyhow, Context, Result};
-
+use async_trait::async_trait;
 use std::{collections::HashMap, sync::Arc};
 
 use agent::Agent;
-use async_trait::async_trait;
 use common::{
     error::Error,
     types::{
@@ -19,10 +18,14 @@ use common::{
     },
     ContainerManager,
 };
+use hypervisor::Hypervisor;
 use oci::Process as OCIProcess;
+use resource::network::NetnsGuard;
 use resource::ResourceManager;
 use tokio::sync::RwLock;
 
+use kata_sys_util::hooks::HookStates;
+
 use super::{logger_with_process, Container};
 
 pub struct VirtContainerManager {
@@ -31,6 +34,7 @@ pub struct VirtContainerManager {
     containers: Arc<RwLock<HashMap<String, Container>>>,
     resource_manager: Arc<ResourceManager>,
     agent: Arc<dyn Agent>,
+    hypervisor: Arc<dyn Hypervisor>,
 }
 
 impl VirtContainerManager {
@@ -38,6 +42,7 @@ impl VirtContainerManager {
         sid: &str,
         pid: u32,
         agent: Arc<dyn Agent>,
+        hypervisor: Arc<dyn Hypervisor>,
         resource_manager: Arc<ResourceManager>,
     ) -> Self {
         Self {
@@ -46,6 +51,7 @@ impl VirtContainerManager {
             containers: Default::default(),
             resource_manager,
             agent,
+            hypervisor,
         }
     }
 }
@@ -55,12 +61,37 @@ impl ContainerManager for VirtContainerManager {
     async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result<PID> {
         let container = Container::new(
             self.pid,
-            config,
+            config.clone(),
+            spec.clone(),
             self.agent.clone(),
             self.resource_manager.clone(),
         )
         .context("new container")?;
 
+        // CreateContainer Hooks:
+        // * should be run in vmm namespace (hook path in runtime namespace)
+        // * should be run after the vm is started, before container is created, and after CreateRuntime Hooks
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createcontainer-hooks
+        let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
+        let vmm_netns_path = format!("/proc/{}/task/{}/ns/{}", self.pid, vmm_master_tid, "net");
+        let state = oci::State {
+            version: spec.version.clone(),
+            id: config.container_id.clone(),
+            status: oci::ContainerState::Creating,
+            pid: vmm_master_tid as i32,
+            bundle: config.bundle.clone(),
+            annotations: spec.annotations.clone(),
+        };
+
+        // new scope, CreateContainer hooks in which will execute in a new network namespace
+        {
+            let _netns_guard = NetnsGuard::new(&vmm_netns_path).context("vmm netns guard")?;
+            if let Some(hooks) = spec.hooks.as_ref() {
+                let mut create_container_hook_states = HookStates::new();
+                create_container_hook_states.execute_hooks(&hooks.create_container, Some(state))?;
+            }
+        }
+
         let mut containers = self.containers.write().await;
         container.create(spec).await.context("create")?;
         containers.insert(container.container_id.to_string(), container);
@@ -87,6 +118,26 @@ impl ContainerManager for VirtContainerManager {
                 let c = containers
                     .remove(container_id)
                     .ok_or_else(|| Error::ContainerNotFound(container_id.to_string()))?;
+
+                // Poststop Hooks:
+                // * should be run in runtime namespace
+                // * should be run after the container is deleted but before delete operation returns
+                // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#poststop
+                let c_spec = c.spec().await;
+                let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
+                let state = oci::State {
+                    version: c_spec.version.clone(),
+                    id: c.container_id.to_string(),
+                    status: oci::ContainerState::Stopped,
+                    pid: vmm_master_tid as i32,
+                    bundle: c.config().await.bundle,
+                    annotations: c_spec.annotations.clone(),
+                };
+                if let Some(hooks) = c_spec.hooks.as_ref() {
+                    let mut poststop_hook_states = HookStates::new();
+                    poststop_hook_states.execute_hooks(&hooks.poststop, Some(state))?;
+                }
+
                 c.state_process(process).await.context("state process")
             }
             ProcessType::Exec => {
@@ -190,6 +241,26 @@ impl ContainerManager for VirtContainerManager {
             .get(container_id)
             .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?;
         c.start(process).await.context("start")?;
+
+        // Poststart Hooks:
+        // * should be run in runtime namespace
+        // * should be run after user-specific command is executed but before start operation returns
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#poststart
+        let c_spec = c.spec().await;
+        let vmm_master_tid = self.hypervisor.get_vmm_master_tid().await?;
+        let state = oci::State {
+            version: c_spec.version.clone(),
+            id: c.container_id.to_string(),
+            status: oci::ContainerState::Running,
+            pid: vmm_master_tid as i32,
+            bundle: c.config().await.bundle,
+            annotations: c_spec.annotations.clone(),
+        };
+        if let Some(hooks) = c_spec.hooks.as_ref() {
+            let mut poststart_hook_states = HookStates::new();
+            poststart_hook_states.execute_hooks(&hooks.poststart, Some(state))?;
+        }
+
         Ok(PID { pid: self.pid })
     }
 

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -86,13 +86,18 @@ impl RuntimeHandler for VirtContainer {
             sid,
             msg_sender,
             agent.clone(),
-            hypervisor,
+            hypervisor.clone(),
             resource_manager.clone(),
         )
         .await
         .context("new virt sandbox")?;
-        let container_manager =
+        let container_manager = container_manager::VirtContainerManager::new(
-            container_manager::VirtContainerManager::new(sid, pid, agent, resource_manager);
+            sid,
+            pid,
+            agent,
+            hypervisor,
+            resource_manager,
+        );
         Ok(RuntimeInstance {
             sandbox: Arc::new(sandbox),
             container_manager: Arc::new(container_manager),

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -17,6 +17,7 @@ use common::{
 };
 use containerd_shim_protos::events::task::TaskOOM;
 use hypervisor::{dragonball::Dragonball, Hypervisor, HYPERVISOR_DRAGONBALL};
+use kata_sys_util::hooks::HookStates;
 use kata_types::config::{
     default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT},
     TomlConfig,
@@ -117,11 +118,50 @@ impl VirtSandbox {
 
         Ok(resource_configs)
     }
+
+    async fn execute_oci_hook_functions(
+        &self,
+        prestart_hooks: &[oci::Hook],
+        create_runtime_hooks: &[oci::Hook],
+        state: &oci::State,
+    ) -> Result<()> {
+        let mut st = state.clone();
+        // for dragonball, we use vmm_master_tid
+        let vmm_pid = self
+            .hypervisor
+            .get_vmm_master_tid()
+            .await
+            .context("get vmm master tid")?;
+        st.pid = vmm_pid as i32;
+
+        // Prestart Hooks [DEPRECATED in newest oci spec]:
+        // * should be run in runtime namespace
+        // * should be run after vm is started, but before container is created
+        //      if Prestart Hook and CreateRuntime Hook are both supported
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#prestart
+        let mut prestart_hook_states = HookStates::new();
+        prestart_hook_states.execute_hooks(prestart_hooks, Some(st.clone()))?;
+
+        // CreateRuntime Hooks:
+        // * should be run in runtime namespace
+        // * should be run when creating the runtime
+        // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks
+        let mut create_runtime_hook_states = HookStates::new();
+        create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?;
+
+        Ok(())
+    }
 }
 
 #[async_trait]
 impl Sandbox for VirtSandbox {
-    async fn start(&self, netns: Option<String>, dns: Vec<String>) -> Result<()> {
+    async fn start(
+        &self,
+        netns: Option<String>,
+        dns: Vec<String>,
+        spec: &oci::Spec,
+        state: &oci::State,
+    ) -> Result<()> {
         let id = &self.sid;
 
         // if sandbox running, return
@@ -149,6 +189,17 @@ impl Sandbox for VirtSandbox {
         self.hypervisor.start_vm(10_000).await.context("start vm")?;
         info!(sl!(), "start vm");
 
+        // execute pre-start hook functions, including Prestart Hooks and CreateRuntime Hooks
+        let (prestart_hooks, create_runtime_hooks) = match spec.hooks.as_ref() {
+            Some(hooks) => (hooks.prestart.clone(), hooks.create_runtime.clone()),
+            None => (Vec::new(), Vec::new()),
+        };
+        self.execute_oci_hook_functions(&prestart_hooks, &create_runtime_hooks, state)
+            .await?;
+
+        // TODO: if prestart_hooks is not empty, rescan the network endpoints(rely on hotplug endpoints).
+        // see: https://github.com/kata-containers/kata-containers/issues/6378
+
         // connect agent
         // set agent socket
         let address = self