From 1c1ee8057cb7c7de7f9cbc27943c9d16e07fa33c Mon Sep 17 00:00:00 2001
From: Eduardo Berrocal <eduardo.berrocal@intel.com>
Date: Tue, 25 Apr 2023 23:34:13 +0000
Subject: [PATCH 001/150] pkg/signals: Improved test coverage 60% to 100%

Expanded tests on signals_test.go to cover more lines of code. 'go test' won't show 100% coverage (only 66.7%), because one test need to spawn a new
process (since it is testing a function that calls os.Exit(1)).

Fixes: #256

Signed-off-by: Eduardo Berrocal <eduardo.berrocal@intel.com>
---
 src/runtime/pkg/signals/signals_test.go | 68 +++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/src/runtime/pkg/signals/signals_test.go b/src/runtime/pkg/signals/signals_test.go
index fab557829..f13e02b04 100644
--- a/src/runtime/pkg/signals/signals_test.go
+++ b/src/runtime/pkg/signals/signals_test.go
@@ -7,7 +7,9 @@ package signals
 
 import (
 	"bytes"
+	"errors"
 	"os"
+	"os/exec"
 	"reflect"
 	goruntime "runtime"
 	"sort"
@@ -135,3 +137,69 @@ func TestSignalBacktrace(t *testing.T) {
 	assert.True(strings.Contains(b, "contention:"))
 	assert.True(strings.Contains(b, `level=error`))
 }
+
+func TestSignalHandlePanic(t *testing.T) {
+	assert := assert.New(t)
+
+	savedLog := signalLog
+	defer func() {
+		signalLog = savedLog
+	}()
+
+	signalLog = logrus.WithFields(logrus.Fields{
+		"name":        "name",
+		"pid":         os.Getpid(),
+		"source":      "throttler",
+		"test-logger": true})
+
+	// Create buffer to save logger output.
+	buf := &bytes.Buffer{}
+
+	savedOut := signalLog.Logger.Out
+	defer func() {
+		signalLog.Logger.Out = savedOut
+	}()
+
+	// Capture output to buffer.
+	signalLog.Logger.Out = buf
+
+	HandlePanic(nil)
+
+	b := buf.String()
+	assert.True(len(b) == 0)
+}
+
+func TestSignalHandlePanicWithError(t *testing.T) {
+	assert := assert.New(t)
+
+	if os.Getenv("CALL_EXIT") != "1" {
+		cmd := exec.Command(os.Args[0], "-test.run=TestSignalHandlePanicWithError")
+		cmd.Env = append(os.Environ(), "CALL_EXIT=1")
+
+		err := cmd.Run()
+		assert.True(err != nil)
+
+		exitError, ok := err.(*exec.ExitError)
+		assert.True(ok)
+		assert.True(exitError.ExitCode() == 1)
+
+		return
+	}
+
+	signalLog = logrus.WithFields(logrus.Fields{
+		"name":        "name",
+		"pid":         os.Getpid(),
+		"source":      "throttler",
+		"test-logger": true})
+
+	// Create buffer to save logger output.
+	buf := &bytes.Buffer{}
+
+	// Capture output to buffer.
+	signalLog.Logger.Out = buf
+
+	dieCallBack := func() {}
+	defer HandlePanic(dieCallBack)
+	e := errors.New("test-panic")
+	panic(e)
+}

From 9c38204f131b0b34941b43e92efa8b2d4726c62d Mon Sep 17 00:00:00 2001
From: Eduardo Berrocal <eduardo.berrocal@intel.com>
Date: Tue, 25 Apr 2023 23:53:46 +0000
Subject: [PATCH 002/150] virtcontainers/persist: Improved test coverage 65% to
 87.5%

Expanded tests on manager_test.go to cover more lines of code.

Fixes: #259

Signed-off-by: Eduardo Berrocal <eduardo.berrocal@intel.com>
---
 .../virtcontainers/persist/manager_test.go    | 46 ++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

diff --git a/src/runtime/virtcontainers/persist/manager_test.go b/src/runtime/virtcontainers/persist/manager_test.go
index 4347f9adc..ad7813792 100644
--- a/src/runtime/virtcontainers/persist/manager_test.go
+++ b/src/runtime/virtcontainers/persist/manager_test.go
@@ -7,7 +7,9 @@
 package persist
 
 import (
+	"errors"
 	"os"
+	"strings"
 	"testing"
 
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@@ -20,14 +22,29 @@ func TestGetDriverByName(t *testing.T) {
 	assert.NotNil(t, err)
 	assert.Nil(t, nonexist)
 
+	// testing correct driver is returned
 	fsDriver, err := GetDriverByName("fs")
 	assert.Nil(t, err)
 	assert.NotNil(t, fsDriver)
+
+	// testing case when expErr is set
+	expErr = errors.New("TEST-ERROR")
+	defer func() {
+		expErr = nil
+	}()
+
+	nonexist, err = GetDriverByName("fs")
+	assert.NotNil(t, err)
+	assert.Nil(t, nonexist)
+
+	b := err.Error()
+	assert.True(t, strings.Contains(b, "TEST-ERROR"))
 }
 
 func TestGetDriver(t *testing.T) {
 	assert := assert.New(t)
 
+	// testing correct driver is returned
 	fsd, err := GetDriver()
 	assert.NoError(err)
 
@@ -39,5 +56,32 @@ func TestGetDriver(t *testing.T) {
 	}
 
 	assert.NoError(err)
-	assert.Equal(expectedFS, fsd)
+	assert.Equal(expectedFS, fsd) // driver should match correct one for UID
+
+	// testing case when expErr is set
+	expErr = errors.New("TEST-ERROR")
+	nonexist, err := GetDriver()
+	assert.NotNil(err)
+	assert.Nil(nonexist)
+	expErr = nil
+
+	// testing case when driver can't be found on supportedDrivers variable
+	supportedDriversBU := supportedDrivers
+	supportedDrivers = nil
+	fsd, err = GetDriver()
+	assert.Nil(fsd)
+	assert.NotNil(err)
+	b := err.Error()
+	assert.True(strings.Contains(b, "Could not find a FS driver"))
+	supportedDrivers = supportedDriversBU
+
+	// testing case when mock driver is activated
+	fs.EnableMockTesting(t.TempDir())
+	mock, err := GetDriver()
+	assert.NoError(err)
+	expectedFS, err = fs.MockFSInit(fs.MockStorageRootPath())
+	assert.NoError(err)
+	assert.Equal(expectedFS, mock)
+
+	fs.EnableMockTesting("")
 }

From c18ceae10925529ca6b4259e8fd10f5e233bad20 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 20 Apr 2023 10:56:29 +0000
Subject: [PATCH 003/150] gpu: Add new struct PCIePort

For the hypervisor to distinguish between PCIe components, adding
a new enum that can be used for hot-plug and cold-plug of PCIe devices

Fixes: #6687

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 .../pkg/hypervisors/hypervisor_state.go       | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go
index b1f58e7fa..6a9fd7af2 100644
--- a/src/runtime/pkg/hypervisors/hypervisor_state.go
+++ b/src/runtime/pkg/hypervisors/hypervisor_state.go
@@ -26,6 +26,20 @@ type CPUDevice struct {
 	ID string
 }
 
+// PCIePort distinguish only between root and switch port
+type PCIePort string
+
+const (
+	// RootPort attach VFIO devices to a root-port
+	RootPort PCIePort = "root-port"
+	// SwitchPort attach VFIO devices to a switch-port
+	SwitchPort = "switch-port"
+	// BridgePort is the default
+	BridgePort = "bridge-port"
+	// NoPort is for disabling VFIO hotplug/coldplug
+	NoPort = "no-port"
+)
+
 type HypervisorState struct {
 	BlockIndexMap map[int]struct{}
 
@@ -41,10 +55,10 @@ type HypervisorState struct {
 	// HotpluggedCPUs is the list of CPUs that were hot-added
 	HotpluggedVCPUs []CPUDevice
 
-	HotpluggedMemory  int
-	VirtiofsDaemonPid int
-	Pid               int
-	PCIeRootPort      int
-
+	HotpluggedMemory     int
+	VirtiofsDaemonPid    int
+	Pid                  int
+	PCIeRootPort         int
+	ColdPlugVFIO         PCIePort
 	HotplugVFIOOnRootBus bool
 }

From 377ebc2ad1f923c6ffc982f24d6f129b4cce2c03 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 20 Apr 2023 11:24:50 +0000
Subject: [PATCH 004/150] gpu: Add configuration option for cold-plug VFIO

Users can set cold-plug="root-port" to cold plug a VFIO device in QEMU

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/cmd/kata-runtime/kata-env.go      |   2 +
 src/runtime/cmd/kata-runtime/kata-env_test.go |   3 +
 src/runtime/pkg/katatestutils/utils.go        |   2 +
 .../pkg/katautils/config-settings.go.in       |   2 +
 src/runtime/pkg/katautils/config.go           | 164 +++++++++---------
 src/runtime/virtcontainers/hypervisor.go      |   4 +
 src/runtime/virtcontainers/kata_agent.go      |   3 +-
 src/runtime/virtcontainers/persist.go         |   1 +
 .../virtcontainers/persist/api/config.go      |   5 +
 9 files changed, 104 insertions(+), 82 deletions(-)

diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go
index c129f8f43..c7e919d62 100644
--- a/src/runtime/cmd/kata-runtime/kata-env.go
+++ b/src/runtime/cmd/kata-runtime/kata-env.go
@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/procfs"
 	"github.com/urfave/cli"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
@@ -113,6 +114,7 @@ type HypervisorInfo struct {
 	Msize9p              uint32
 	MemorySlots          uint32
 	PCIeRootPort         uint32
+	ColdPlugVFIO         hv.PCIePort
 	HotplugVFIOOnRootBus bool
 	Debug                bool
 }
diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go
index 321bc507b..246889472 100644
--- a/src/runtime/cmd/kata-runtime/kata-env_test.go
+++ b/src/runtime/cmd/kata-runtime/kata-env_test.go
@@ -24,6 +24,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/urfave/cli"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
@@ -85,6 +86,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	blockStorageDriver := "virtio-scsi"
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	coldPlugVFIO := hv.RootPort
 	pcieRootPort := uint32(2)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
@@ -129,6 +131,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		ColdPlugVFIO:         coldPlugVFIO,
 		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs,
diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go
index 4e3a784a2..b973063e8 100644
--- a/src/runtime/pkg/katatestutils/utils.go
+++ b/src/runtime/pkg/katatestutils/utils.go
@@ -14,6 +14,7 @@ import (
 	"strconv"
 	"testing"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 )
@@ -224,6 +225,7 @@ type RuntimeConfigOptions struct {
 	JaegerPassword       string
 	PFlash               []string
 	PCIeRootPort         uint32
+	ColdPlugVFIO         hv.PCIePort
 	DefaultVCPUCount     uint32
 	DefaultMaxVCPUCount  uint32
 	DefaultMemSize       uint32
diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in
index 7bfab6d9f..14a2b0b58 100644
--- a/src/runtime/pkg/katautils/config-settings.go.in
+++ b/src/runtime/pkg/katautils/config-settings.go.in
@@ -103,3 +103,5 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
+
+const defaultColdPlugVFIO = "no-port"
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
index 763e9a6f4..f419e0d61 100644
--- a/src/runtime/pkg/katautils/config.go
+++ b/src/runtime/pkg/katautils/config.go
@@ -20,6 +20,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@@ -77,87 +78,88 @@ type factory struct {
 }
 
 type hypervisor struct {
-	Path                           string   `toml:"path"`
-	JailerPath                     string   `toml:"jailer_path"`
-	Kernel                         string   `toml:"kernel"`
-	CtlPath                        string   `toml:"ctlpath"`
-	Initrd                         string   `toml:"initrd"`
-	Image                          string   `toml:"image"`
-	RootfsType                     string   `toml:"rootfs_type"`
-	Firmware                       string   `toml:"firmware"`
-	FirmwareVolume                 string   `toml:"firmware_volume"`
-	MachineAccelerators            string   `toml:"machine_accelerators"`
-	CPUFeatures                    string   `toml:"cpu_features"`
-	KernelParams                   string   `toml:"kernel_params"`
-	MachineType                    string   `toml:"machine_type"`
-	BlockDeviceDriver              string   `toml:"block_device_driver"`
-	EntropySource                  string   `toml:"entropy_source"`
-	SharedFS                       string   `toml:"shared_fs"`
-	VirtioFSDaemon                 string   `toml:"virtio_fs_daemon"`
-	VirtioFSCache                  string   `toml:"virtio_fs_cache"`
-	VhostUserStorePath             string   `toml:"vhost_user_store_path"`
-	FileBackedMemRootDir           string   `toml:"file_mem_backend"`
-	GuestHookPath                  string   `toml:"guest_hook_path"`
-	GuestMemoryDumpPath            string   `toml:"guest_memory_dump_path"`
-	SeccompSandbox                 string   `toml:"seccompsandbox"`
-	BlockDeviceAIO                 string   `toml:"block_device_aio"`
-	HypervisorPathList             []string `toml:"valid_hypervisor_paths"`
-	JailerPathList                 []string `toml:"valid_jailer_paths"`
-	CtlPathList                    []string `toml:"valid_ctlpaths"`
-	VirtioFSDaemonList             []string `toml:"valid_virtio_fs_daemon_paths"`
-	VirtioFSExtraArgs              []string `toml:"virtio_fs_extra_args"`
-	PFlashList                     []string `toml:"pflashes"`
-	VhostUserStorePathList         []string `toml:"valid_vhost_user_store_paths"`
-	FileBackedMemRootList          []string `toml:"valid_file_mem_backends"`
-	EntropySourceList              []string `toml:"valid_entropy_sources"`
-	EnableAnnotations              []string `toml:"enable_annotations"`
-	RxRateLimiterMaxRate           uint64   `toml:"rx_rate_limiter_max_rate"`
-	TxRateLimiterMaxRate           uint64   `toml:"tx_rate_limiter_max_rate"`
-	MemOffset                      uint64   `toml:"memory_offset"`
-	DefaultMaxMemorySize           uint64   `toml:"default_maxmemory"`
-	DiskRateLimiterBwMaxRate       int64    `toml:"disk_rate_limiter_bw_max_rate"`
-	DiskRateLimiterBwOneTimeBurst  int64    `toml:"disk_rate_limiter_bw_one_time_burst"`
-	DiskRateLimiterOpsMaxRate      int64    `toml:"disk_rate_limiter_ops_max_rate"`
-	DiskRateLimiterOpsOneTimeBurst int64    `toml:"disk_rate_limiter_ops_one_time_burst"`
-	NetRateLimiterBwMaxRate        int64    `toml:"net_rate_limiter_bw_max_rate"`
-	NetRateLimiterBwOneTimeBurst   int64    `toml:"net_rate_limiter_bw_one_time_burst"`
-	NetRateLimiterOpsMaxRate       int64    `toml:"net_rate_limiter_ops_max_rate"`
-	NetRateLimiterOpsOneTimeBurst  int64    `toml:"net_rate_limiter_ops_one_time_burst"`
-	VirtioFSCacheSize              uint32   `toml:"virtio_fs_cache_size"`
-	VirtioFSQueueSize              uint32   `toml:"virtio_fs_queue_size"`
-	DefaultMaxVCPUs                uint32   `toml:"default_maxvcpus"`
-	MemorySize                     uint32   `toml:"default_memory"`
-	MemSlots                       uint32   `toml:"memory_slots"`
-	DefaultBridges                 uint32   `toml:"default_bridges"`
-	Msize9p                        uint32   `toml:"msize_9p"`
-	PCIeRootPort                   uint32   `toml:"pcie_root_port"`
-	NumVCPUs                       int32    `toml:"default_vcpus"`
-	BlockDeviceCacheSet            bool     `toml:"block_device_cache_set"`
-	BlockDeviceCacheDirect         bool     `toml:"block_device_cache_direct"`
-	BlockDeviceCacheNoflush        bool     `toml:"block_device_cache_noflush"`
-	EnableVhostUserStore           bool     `toml:"enable_vhost_user_store"`
-	VhostUserDeviceReconnect       uint32   `toml:"vhost_user_reconnect_timeout_sec"`
-	DisableBlockDeviceUse          bool     `toml:"disable_block_device_use"`
-	MemPrealloc                    bool     `toml:"enable_mem_prealloc"`
-	HugePages                      bool     `toml:"enable_hugepages"`
-	VirtioMem                      bool     `toml:"enable_virtio_mem"`
-	IOMMU                          bool     `toml:"enable_iommu"`
-	IOMMUPlatform                  bool     `toml:"enable_iommu_platform"`
-	Debug                          bool     `toml:"enable_debug"`
-	DisableNestingChecks           bool     `toml:"disable_nesting_checks"`
-	EnableIOThreads                bool     `toml:"enable_iothreads"`
-	DisableImageNvdimm             bool     `toml:"disable_image_nvdimm"`
-	HotplugVFIOOnRootBus           bool     `toml:"hotplug_vfio_on_root_bus"`
-	DisableVhostNet                bool     `toml:"disable_vhost_net"`
-	GuestMemoryDumpPaging          bool     `toml:"guest_memory_dump_paging"`
-	ConfidentialGuest              bool     `toml:"confidential_guest"`
-	SevSnpGuest                    bool     `toml:"sev_snp_guest"`
-	GuestSwap                      bool     `toml:"enable_guest_swap"`
-	Rootless                       bool     `toml:"rootless"`
-	DisableSeccomp                 bool     `toml:"disable_seccomp"`
-	DisableSeLinux                 bool     `toml:"disable_selinux"`
-	DisableGuestSeLinux            bool     `toml:"disable_guest_selinux"`
-	LegacySerial                   bool     `toml:"use_legacy_serial"`
+	Path                           string      `toml:"path"`
+	JailerPath                     string      `toml:"jailer_path"`
+	Kernel                         string      `toml:"kernel"`
+	CtlPath                        string      `toml:"ctlpath"`
+	Initrd                         string      `toml:"initrd"`
+	Image                          string      `toml:"image"`
+	RootfsType                     string      `toml:"rootfs_type"`
+	Firmware                       string      `toml:"firmware"`
+	FirmwareVolume                 string      `toml:"firmware_volume"`
+	MachineAccelerators            string      `toml:"machine_accelerators"`
+	CPUFeatures                    string      `toml:"cpu_features"`
+	KernelParams                   string      `toml:"kernel_params"`
+	MachineType                    string      `toml:"machine_type"`
+	BlockDeviceDriver              string      `toml:"block_device_driver"`
+	EntropySource                  string      `toml:"entropy_source"`
+	SharedFS                       string      `toml:"shared_fs"`
+	VirtioFSDaemon                 string      `toml:"virtio_fs_daemon"`
+	VirtioFSCache                  string      `toml:"virtio_fs_cache"`
+	VhostUserStorePath             string      `toml:"vhost_user_store_path"`
+	FileBackedMemRootDir           string      `toml:"file_mem_backend"`
+	GuestHookPath                  string      `toml:"guest_hook_path"`
+	GuestMemoryDumpPath            string      `toml:"guest_memory_dump_path"`
+	SeccompSandbox                 string      `toml:"seccompsandbox"`
+	BlockDeviceAIO                 string      `toml:"block_device_aio"`
+	HypervisorPathList             []string    `toml:"valid_hypervisor_paths"`
+	JailerPathList                 []string    `toml:"valid_jailer_paths"`
+	CtlPathList                    []string    `toml:"valid_ctlpaths"`
+	VirtioFSDaemonList             []string    `toml:"valid_virtio_fs_daemon_paths"`
+	VirtioFSExtraArgs              []string    `toml:"virtio_fs_extra_args"`
+	PFlashList                     []string    `toml:"pflashes"`
+	VhostUserStorePathList         []string    `toml:"valid_vhost_user_store_paths"`
+	FileBackedMemRootList          []string    `toml:"valid_file_mem_backends"`
+	EntropySourceList              []string    `toml:"valid_entropy_sources"`
+	EnableAnnotations              []string    `toml:"enable_annotations"`
+	RxRateLimiterMaxRate           uint64      `toml:"rx_rate_limiter_max_rate"`
+	TxRateLimiterMaxRate           uint64      `toml:"tx_rate_limiter_max_rate"`
+	MemOffset                      uint64      `toml:"memory_offset"`
+	DefaultMaxMemorySize           uint64      `toml:"default_maxmemory"`
+	DiskRateLimiterBwMaxRate       int64       `toml:"disk_rate_limiter_bw_max_rate"`
+	DiskRateLimiterBwOneTimeBurst  int64       `toml:"disk_rate_limiter_bw_one_time_burst"`
+	DiskRateLimiterOpsMaxRate      int64       `toml:"disk_rate_limiter_ops_max_rate"`
+	DiskRateLimiterOpsOneTimeBurst int64       `toml:"disk_rate_limiter_ops_one_time_burst"`
+	NetRateLimiterBwMaxRate        int64       `toml:"net_rate_limiter_bw_max_rate"`
+	NetRateLimiterBwOneTimeBurst   int64       `toml:"net_rate_limiter_bw_one_time_burst"`
+	NetRateLimiterOpsMaxRate       int64       `toml:"net_rate_limiter_ops_max_rate"`
+	NetRateLimiterOpsOneTimeBurst  int64       `toml:"net_rate_limiter_ops_one_time_burst"`
+	VirtioFSCacheSize              uint32      `toml:"virtio_fs_cache_size"`
+	VirtioFSQueueSize              uint32      `toml:"virtio_fs_queue_size"`
+	DefaultMaxVCPUs                uint32      `toml:"default_maxvcpus"`
+	MemorySize                     uint32      `toml:"default_memory"`
+	MemSlots                       uint32      `toml:"memory_slots"`
+	DefaultBridges                 uint32      `toml:"default_bridges"`
+	Msize9p                        uint32      `toml:"msize_9p"`
+	PCIeRootPort                   uint32      `toml:"pcie_root_port"`
+	NumVCPUs                       int32       `toml:"default_vcpus"`
+	BlockDeviceCacheSet            bool        `toml:"block_device_cache_set"`
+	BlockDeviceCacheDirect         bool        `toml:"block_device_cache_direct"`
+	BlockDeviceCacheNoflush        bool        `toml:"block_device_cache_noflush"`
+	EnableVhostUserStore           bool        `toml:"enable_vhost_user_store"`
+	VhostUserDeviceReconnect       uint32      `toml:"vhost_user_reconnect_timeout_sec"`
+	DisableBlockDeviceUse          bool        `toml:"disable_block_device_use"`
+	MemPrealloc                    bool        `toml:"enable_mem_prealloc"`
+	HugePages                      bool        `toml:"enable_hugepages"`
+	VirtioMem                      bool        `toml:"enable_virtio_mem"`
+	IOMMU                          bool        `toml:"enable_iommu"`
+	IOMMUPlatform                  bool        `toml:"enable_iommu_platform"`
+	Debug                          bool        `toml:"enable_debug"`
+	DisableNestingChecks           bool        `toml:"disable_nesting_checks"`
+	EnableIOThreads                bool        `toml:"enable_iothreads"`
+	DisableImageNvdimm             bool        `toml:"disable_image_nvdimm"`
+	HotplugVFIOOnRootBus           bool        `toml:"hotplug_vfio_on_root_bus"`
+	ColdPlugVFIO                   hv.PCIePort `toml:"cold_plug_vfio"`
+	DisableVhostNet                bool        `toml:"disable_vhost_net"`
+	GuestMemoryDumpPaging          bool        `toml:"guest_memory_dump_paging"`
+	ConfidentialGuest              bool        `toml:"confidential_guest"`
+	SevSnpGuest                    bool        `toml:"sev_snp_guest"`
+	GuestSwap                      bool        `toml:"enable_guest_swap"`
+	Rootless                       bool        `toml:"rootless"`
+	DisableSeccomp                 bool        `toml:"disable_seccomp"`
+	DisableSeLinux                 bool        `toml:"disable_selinux"`
+	DisableGuestSeLinux            bool        `toml:"disable_guest_selinux"`
+	LegacySerial                   bool        `toml:"use_legacy_serial"`
 }
 
 type runtime struct {
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
index dee5fec8f..0a490ef57 100644
--- a/src/runtime/virtcontainers/hypervisor.go
+++ b/src/runtime/virtcontainers/hypervisor.go
@@ -509,6 +509,10 @@ type HypervisorConfig struct {
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
+	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
+	// root port, switch or no port
+	ColdPlugVFIO hv.PCIePort
+
 	// NumVCPUs specifies default number of vCPUs for the VM.
 	NumVCPUs uint32
 
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
index 9e5c8b34f..5c22277d0 100644
--- a/src/runtime/virtcontainers/kata_agent.go
+++ b/src/runtime/virtcontainers/kata_agent.go
@@ -1177,7 +1177,8 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 		case config.VhostUserBlk:
 			kataDevice = k.appendVhostUserBlkDevice(dev, device, c)
 		case config.DeviceVFIO:
-			kataDevice = k.appendVfioDevice(dev, device, c)
+			k.Logger().Infof("### ColdPlugging container is not adding any VFIO devices")
+			//kataDevice = k.appendVfioDevice(dev, device, c)
 		}
 
 		if kataDevice == nil {
diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go
index 18c83e251..cbba44e60 100644
--- a/src/runtime/virtcontainers/persist.go
+++ b/src/runtime/virtcontainers/persist.go
@@ -487,6 +487,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		DisableNestingChecks:    hconf.DisableNestingChecks,
 		DisableImageNvdimm:      hconf.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    hconf.HotplugVFIOOnRootBus,
+		ColdPlugVFIO:            hconf.ColdPlugVFIO,
 		PCIeRootPort:            hconf.PCIeRootPort,
 		BootToBeTemplate:        hconf.BootToBeTemplate,
 		BootFromTemplate:        hconf.BootFromTemplate,
diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go
index 5bef01219..71533d651 100644
--- a/src/runtime/virtcontainers/persist/api/config.go
+++ b/src/runtime/virtcontainers/persist/api/config.go
@@ -7,6 +7,7 @@
 package persistapi
 
 import (
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -198,6 +199,10 @@ type HypervisorConfig struct {
 	// root bus instead of a bridge.
 	HotplugVFIOOnRootBus bool
 
+	// ColdPlugVFIO is used to indicate if devices need to be coldlugged on the
+	// root port or a switch or no-port
+	ColdPlugVFIO hv.PCIePort
+
 	// BootToBeTemplate used to indicate if the VM is created to be a template VM
 	BootToBeTemplate bool
 

From 6107c32d70377f4bc83ed22aaab3ea4e035ada48 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 20 Apr 2023 11:34:17 +0000
Subject: [PATCH 005/150] gpu: Assign default value to cold-plug

Make sure the configuration is propagated to the right structs
and the default value is assigned.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/cmd/kata-runtime/kata-env_test.go | 3 ---
 src/runtime/pkg/katautils/config.go           | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go
index 246889472..321bc507b 100644
--- a/src/runtime/cmd/kata-runtime/kata-env_test.go
+++ b/src/runtime/cmd/kata-runtime/kata-env_test.go
@@ -24,7 +24,6 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/urfave/cli"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
@@ -86,7 +85,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	blockStorageDriver := "virtio-scsi"
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
-	coldPlugVFIO := hv.RootPort
 	pcieRootPort := uint32(2)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
@@ -131,7 +129,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
-		ColdPlugVFIO:         coldPlugVFIO,
 		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs,
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
index f419e0d61..06b217a9a 100644
--- a/src/runtime/pkg/katautils/config.go
+++ b/src/runtime/pkg/katautils/config.go
@@ -856,6 +856,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Msize9p:                 h.msize9p(),
 		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
+		ColdPlugVFIO:            h.ColdPlugVFIO,
 		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         h.DisableVhostNet,
 		EnableVhostUserStore:    h.EnableVhostUserStore,
@@ -1050,6 +1051,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:                h.EnableIOThreads,
 		Msize9p:                        h.msize9p(),
 		HotplugVFIOOnRootBus:           h.HotplugVFIOOnRootBus,
+		ColdPlugVFIO:                   h.ColdPlugVFIO,
 		PCIeRootPort:                   h.PCIeRootPort,
 		DisableVhostNet:                true,
 		GuestHookPath:                  h.guestHookPath(),
@@ -1280,6 +1282,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		EnableIOThreads:          defaultEnableIOThreads,
 		Msize9p:                  defaultMsize9p,
 		HotplugVFIOOnRootBus:     defaultHotplugVFIOOnRootBus,
+		ColdPlugVFIO:             defaultColdPlugVFIO,
 		PCIeRootPort:             defaultPCIeRootPort,
 		GuestHookPath:            defaultGuestHookPath,
 		VhostUserStorePath:       defaultVhostUserStorePath,

From e2b5e7f73bbf0f4eacfedbb0fa69b1fa791198ac Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 20 Apr 2023 11:41:44 +0000
Subject: [PATCH 006/150] gpu: Add Rawdevices to hypervisor

RawDevics are used to get PCIe device info early before the sandbox
is started to make better PCIe topology decisions

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/virtcontainers/hypervisor.go | 4 ++++
 src/runtime/virtcontainers/sandbox.go    | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
index 0a490ef57..bc33cfd67 100644
--- a/src/runtime/virtcontainers/hypervisor.go
+++ b/src/runtime/virtcontainers/hypervisor.go
@@ -509,6 +509,10 @@ type HypervisorConfig struct {
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
+	// RawDevics are used to get PCIe device info early before the sandbox
+	// is started to make better PCIe topology decisions
+	RawDevices []config.DeviceInfo
+
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root port, switch or no port
 	ColdPlugVFIO hv.PCIePort
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index e7c52fcc2..da1d52e9d 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -619,6 +619,12 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 	if err := validateHypervisorConfig(&sandboxConfig.HypervisorConfig); err != nil {
 		return nil, err
 	}
+	// Aggregate all the container devices and update the HV config
+	var devices []config.DeviceInfo
+	for _, ct := range sandboxConfig.Containers {
+		devices = append(devices, ct.DeviceInfos...)
+	}
+	sandboxConfig.HypervisorConfig.RawDevices = devices
 
 	// store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil {

From c8cf7ed3bc3f9b7f40dcb66fcffc97eea1db4a57 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Fri, 21 Apr 2023 08:56:47 +0000
Subject: [PATCH 007/150] gpu: Add ColdPlug of VFIO devices with devManager

If we have a VFIO device and cold-plug is enabled
we mark each device as ColdPlug=true and let the VFIO
module do the attaching.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/pkg/device/manager/manager.go    |  2 +-
 src/runtime/pkg/device/manager/utils.go      |  4 +-
 src/runtime/pkg/device/manager/utils_test.go |  2 +-
 src/runtime/virtcontainers/hypervisor.go     |  4 +-
 src/runtime/virtcontainers/kata_agent.go     |  3 +-
 src/runtime/virtcontainers/qemu.go           | 20 +++++++++-
 src/runtime/virtcontainers/sandbox.go        | 40 +++++++++++++++++---
 7 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/src/runtime/pkg/device/manager/manager.go b/src/runtime/pkg/device/manager/manager.go
index 34a51d300..baf1209a7 100644
--- a/src/runtime/pkg/device/manager/manager.go
+++ b/src/runtime/pkg/device/manager/manager.go
@@ -116,7 +116,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
 	if devInfo.ID, err = dm.newDeviceID(); err != nil {
 		return nil, err
 	}
-	if isVFIO(devInfo.HostPath) {
+	if IsVFIO(devInfo.HostPath) {
 		return drivers.NewVFIODevice(&devInfo), nil
 	} else if isVhostUserBlk(devInfo) {
 		if devInfo.DriverOptions == nil {
diff --git a/src/runtime/pkg/device/manager/utils.go b/src/runtime/pkg/device/manager/utils.go
index 17d14741c..e78205d0c 100644
--- a/src/runtime/pkg/device/manager/utils.go
+++ b/src/runtime/pkg/device/manager/utils.go
@@ -17,8 +17,8 @@ const (
 	vfioPath = "/dev/vfio/"
 )
 
-// isVFIO checks if the device provided is a vfio group.
-func isVFIO(hostPath string) bool {
+// IsVFIO checks if the device provided is a vfio group.
+func IsVFIO(hostPath string) bool {
 	// Ignore /dev/vfio/vfio character device
 	if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) {
 		return false
diff --git a/src/runtime/pkg/device/manager/utils_test.go b/src/runtime/pkg/device/manager/utils_test.go
index 273283823..b57992b3d 100644
--- a/src/runtime/pkg/device/manager/utils_test.go
+++ b/src/runtime/pkg/device/manager/utils_test.go
@@ -31,7 +31,7 @@ func TestIsVFIO(t *testing.T) {
 	}
 
 	for _, d := range data {
-		isVFIO := isVFIO(d.path)
+		isVFIO := IsVFIO(d.path)
 		assert.Equal(t, d.expected, isVFIO)
 	}
 }
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
index bc33cfd67..f773e91d5 100644
--- a/src/runtime/virtcontainers/hypervisor.go
+++ b/src/runtime/virtcontainers/hypervisor.go
@@ -509,9 +509,9 @@ type HypervisorConfig struct {
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
-	// RawDevics are used to get PCIe device info early before the sandbox
+	// VFIODevics are used to get PCIe device info early before the sandbox
 	// is started to make better PCIe topology decisions
-	RawDevices []config.DeviceInfo
+	VFIODevices []config.DeviceInfo
 
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root port, switch or no port
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
index 5c22277d0..9e5c8b34f 100644
--- a/src/runtime/virtcontainers/kata_agent.go
+++ b/src/runtime/virtcontainers/kata_agent.go
@@ -1177,8 +1177,7 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 		case config.VhostUserBlk:
 			kataDevice = k.appendVhostUserBlkDevice(dev, device, c)
 		case config.DeviceVFIO:
-			k.Logger().Infof("### ColdPlugging container is not adding any VFIO devices")
-			//kataDevice = k.appendVfioDevice(dev, device, c)
+			kataDevice = k.appendVfioDevice(dev, device, c)
 		}
 
 		if kataDevice == nil {
diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go
index 3c8cad520..d65c93553 100644
--- a/src/runtime/virtcontainers/qemu.go
+++ b/src/runtime/virtcontainers/qemu.go
@@ -83,6 +83,7 @@ type QemuState struct {
 	VirtiofsDaemonPid    int
 	PCIeRootPort         int
 	HotplugVFIOOnRootBus bool
+	ColdPlugVFIO         hv.PCIePort
 }
 
 // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor.
@@ -282,6 +283,7 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
 		q.Logger().Debug("Creating UUID")
 		q.state.UUID = uuid.Generate().String()
 
+		q.state.ColdPlugVFIO = q.config.ColdPlugVFIO
 		q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
 		q.state.PCIeRootPort = int(q.config.PCIeRootPort)
 
@@ -708,9 +710,25 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort, memSize32bit, memSize64bit)
 	}
 
+	q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath)
+
+	// If we have a VFIO device we need to update the firmware configuration
+	// if executed in a trusted execution environment.
+	if hypervisorConfig.ConfidentialGuest {
+		// At the sandbox level we alreaady checked that we have a
+		// VFIO device, pass-through of a PCIe device needs allocated
+		// mmemory in the firmware otherwise BARs cannot be mapped
+		if len(hypervisorConfig.VFIODevices) > 0 {
+			fwCfg := govmmQemu.FwCfg{
+				Name: "opt/ovmf/X-PciMmio64Mb",
+				Str:  "262144",
+			}
+			qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg)
+		}
+	}
+
 	q.qemuConfig = qemuConfig
 
-	q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath)
 	return err
 }
 
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index da1d52e9d..35b319394 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -32,6 +32,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@@ -619,12 +620,30 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 	if err := validateHypervisorConfig(&sandboxConfig.HypervisorConfig); err != nil {
 		return nil, err
 	}
-	// Aggregate all the container devices and update the HV config
-	var devices []config.DeviceInfo
-	for _, ct := range sandboxConfig.Containers {
-		devices = append(devices, ct.DeviceInfos...)
+
+	// If we have a confidential guest we need to cold-plug the PCIe VFIO devices
+	// until we have TDISP/IDE PCIe support.
+	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
+	var devs []config.DeviceInfo
+	for cnt, containers := range sandboxConfig.Containers {
+		for dev, device := range containers.DeviceInfos {
+			if coldPlugVFIO && deviceManager.IsVFIO(device.ContainerPath) {
+				device.ColdPlug = true
+				devs = append(devs, device)
+				// We need to remove the devices marked for cold-plug
+				// otherwise at the container level the kata-agent
+				// will try to hot-plug them.
+				infos := sandboxConfig.Containers[cnt].DeviceInfos
+				infos = append(infos[:dev], infos[dev+1:]...)
+				sandboxConfig.Containers[cnt].DeviceInfos = infos
+			}
+		}
 	}
-	sandboxConfig.HypervisorConfig.RawDevices = devices
+	// If we have a confidential guest, we need to add a specific
+	// firmware configuration to the hypervisor. We cannot do it here at
+	// the sandbox level we need to do that at the hypervisor level, capturing
+	// the devices here and processing in CreateVM().
+	sandboxConfig.HypervisorConfig.VFIODevices = devs
 
 	// store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil {
@@ -635,6 +654,17 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		return nil, err
 	}
 
+	if !coldPlugVFIO {
+		return s, nil
+	}
+
+	for _, dev := range devs {
+		_, err := s.AddDevice(ctx, dev)
+		if err != nil {
+			s.Logger().WithError(err).Debug("Cannot cold-plug add device")
+			return nil, err
+		}
+	}
 	return s, nil
 }
 

From 131f056a124182dfd8e5f7dc8390080c29f9ebc7 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Fri, 21 Apr 2023 10:19:40 +0000
Subject: [PATCH 008/150] gpu: Extract VFIO Functions to drivers

Some functions may be used in other modules then only in
the VFIO module, extract them and make them available to
other layers like sandbox.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/pkg/device/drivers/utils.go | 96 +++++++++++++++++++++++++
 src/runtime/pkg/device/drivers/vfio.go  | 23 +-----
 2 files changed, 99 insertions(+), 20 deletions(-)

diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go
index 79e00adbd..7c87e6a59 100644
--- a/src/runtime/pkg/device/drivers/utils.go
+++ b/src/runtime/pkg/device/drivers/utils.go
@@ -10,10 +10,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
 )
 
@@ -133,3 +135,97 @@ func GetAPVFIODevices(sysfsdev string) ([]string, error) {
 	// Split by newlines, omitting final newline
 	return strings.Split(string(data[:len(data)-1]), "\n"), nil
 }
+
+// Ignore specific PCI devices, supply the pciClass and the bitmask to check
+// against the device class, deviceBDF for meaningfull info message
+func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (bool, error) {
+	if pciClass == "" {
+		return false, nil
+	}
+	pciClassID, err := strconv.ParseUint(pciClass, 0, 32)
+	if err != nil {
+		return false, err
+	}
+	// ClassID is 16 bits, remove the two trailing zeros
+	pciClassID = pciClassID >> 8
+	if pciClassID&bitmask == bitmask {
+		deviceLogger().Infof("Ignoring PCI (Host) Bridge deviceBDF %v Class %x", deviceBDF, pciClassID)
+		return true, nil
+	}
+	return false, nil
+}
+
+// GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group
+// We can reuse this function at various leverls, sandbox, container.
+func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIODev, error) {
+
+	vfioDevs := []*config.VFIODev{}
+
+	vfioGroup := filepath.Base(device.HostPath)
+	iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices")
+
+	deviceFiles, err := os.ReadDir(iommuDevicesPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Pass all devices in iommu group
+	for i, deviceFile := range deviceFiles {
+		//Get bdf of device eg 0000:00:1c.0
+		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath)
+		if err != nil {
+			return nil, err
+		}
+		id := utils.MakeNameID("vfio", device.ID+strconv.Itoa(i), maxDevIDSize)
+
+		pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
+		// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
+		// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
+		// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
+		ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
+		if err != nil {
+			return nil, err
+		}
+		if ignorePCIDevice {
+			continue
+		}
+
+		var vfio config.VFIODev
+
+		switch vfioDeviceType {
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			isPCIe := isPCIeDevice(deviceBDF)
+			// Do not directly assign to `vfio` -- need to access field still
+			vfioPCI := config.VFIOPCIDev{
+				ID:       id,
+				Type:     vfioDeviceType,
+				BDF:      deviceBDF,
+				SysfsDev: deviceSysfsDev,
+				IsPCIe:   isPCIe,
+				Class:    pciClass,
+			}
+			if isPCIe {
+				vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
+				AllPCIeDevs[deviceBDF] = true
+			}
+			vfio = vfioPCI
+		case config.VFIOAPDeviceMediatedType:
+			devices, err := GetAPVFIODevices(deviceSysfsDev)
+			if err != nil {
+				return nil, err
+			}
+			vfio = config.VFIOAPDev{
+				ID:        id,
+				SysfsDev:  deviceSysfsDev,
+				Type:      config.VFIOAPDeviceMediatedType,
+				APDevices: devices,
+			}
+		default:
+			return nil, fmt.Errorf("Failed to append device: VFIO device type unrecognized")
+		}
+
+		vfioDevs = append(vfioDevs, &vfio)
+	}
+
+	return vfioDevs, nil
+}
diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go
index 1099f8f0b..4d06240b4 100644
--- a/src/runtime/pkg/device/drivers/vfio.go
+++ b/src/runtime/pkg/device/drivers/vfio.go
@@ -54,25 +54,6 @@ func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice {
 	}
 }
 
-// Ignore specific PCI devices, supply the pciClass and the bitmask to check
-// against the device class, deviceBDF for meaningfull info message
-func (device *VFIODevice) checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (bool, error) {
-	if pciClass == "" {
-		return false, nil
-	}
-	pciClassID, err := strconv.ParseUint(pciClass, 0, 32)
-	if err != nil {
-		return false, err
-	}
-	// ClassID is 16 bits, remove the two trailing zeros
-	pciClassID = pciClassID >> 8
-	if pciClassID&bitmask == bitmask {
-		deviceLogger().Infof("Ignoring PCI (Host) Bridge deviceBDF %v Class %x", deviceBDF, pciClassID)
-		return true, nil
-	}
-	return false, nil
-}
-
 // Attach is standard interface of api.Device, it's used to add device to some
 // DeviceReceiver
 func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceReceiver) (retErr error) {
@@ -90,6 +71,8 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		}
 	}()
 
+	device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(device.DeviceInfo)
+
 	vfioGroup := filepath.Base(device.DeviceInfo.HostPath)
 	iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices")
 
@@ -111,7 +94,7 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
 		// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
 		// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
-		ignorePCIDevice, err := device.checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
+		ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
 		if err != nil {
 			return err
 		}

From 2a830177cab9575863fbaf4ddadba25b5e07b982 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Fri, 21 Apr 2023 10:43:56 +0000
Subject: [PATCH 009/150] gpu: Add fwcfg helper function

Added driver util function for easier handling of VFIO
devices outside of the VFIO module. At the sandbox level
we may need to set options depending if we have a VFIO/PCIe
device, like the fwCfg for confiential guests.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/cmd/kata-runtime/kata-env.go | 22 ++++----
 src/runtime/pkg/device/drivers/utils.go  |  8 +--
 src/runtime/pkg/device/drivers/vfio.go   | 66 +-----------------------
 src/runtime/pkg/katatestutils/utils.go   |  1 +
 src/runtime/virtcontainers/qemu.go       | 44 +++++++++++++---
 src/runtime/virtcontainers/sandbox.go    |  2 +-
 6 files changed, 56 insertions(+), 87 deletions(-)

diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go
index c7e919d62..f17480aba 100644
--- a/src/runtime/cmd/kata-runtime/kata-env.go
+++ b/src/runtime/cmd/kata-runtime/kata-env.go
@@ -307,17 +307,17 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 	}
 
 	return HypervisorInfo{
-		Debug:             config.HypervisorConfig.Debug,
-		MachineType:       config.HypervisorConfig.HypervisorMachineType,
-		Version:           version,
-		Path:              hypervisorPath,
-		BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver,
-		Msize9p:           config.HypervisorConfig.Msize9p,
-		MemorySlots:       config.HypervisorConfig.MemSlots,
-		EntropySource:     config.HypervisorConfig.EntropySource,
-		SharedFS:          config.HypervisorConfig.SharedFS,
-		VirtioFSDaemon:    config.HypervisorConfig.VirtioFSDaemon,
-
+		Debug:                config.HypervisorConfig.Debug,
+		MachineType:          config.HypervisorConfig.HypervisorMachineType,
+		Version:              version,
+		Path:                 hypervisorPath,
+		BlockDeviceDriver:    config.HypervisorConfig.BlockDeviceDriver,
+		Msize9p:              config.HypervisorConfig.Msize9p,
+		MemorySlots:          config.HypervisorConfig.MemSlots,
+		EntropySource:        config.HypervisorConfig.EntropySource,
+		SharedFS:             config.HypervisorConfig.SharedFS,
+		VirtioFSDaemon:       config.HypervisorConfig.VirtioFSDaemon,
+		ColdPlugVFIO:         config.HypervisorConfig.ColdPlugVFIO,
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
 		SocketPath:           socketPath,
diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go
index 7c87e6a59..bfffa31a2 100644
--- a/src/runtime/pkg/device/drivers/utils.go
+++ b/src/runtime/pkg/device/drivers/utils.go
@@ -156,8 +156,10 @@ func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (boo
 }
 
 // GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group
-// We can reuse this function at various leverls, sandbox, container.
-func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIODev, error) {
+// We can reuse this function at various levels, sandbox, container.
+// Only the VFIO module is allowed to do bus assignments, all other modules need to
+// ignore it if used as helper function to get VFIO information.
+func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignment bool) ([]*config.VFIODev, error) {
 
 	vfioDevs := []*config.VFIODev{}
 
@@ -204,7 +206,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIOD
 				IsPCIe:   isPCIe,
 				Class:    pciClass,
 			}
-			if isPCIe {
+			if isPCIe && !ignoreBusAssignment {
 				vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
 				AllPCIeDevs[deviceBDF] = true
 			}
diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go
index 4d06240b4..106220dcf 100644
--- a/src/runtime/pkg/device/drivers/vfio.go
+++ b/src/runtime/pkg/device/drivers/vfio.go
@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 
 	"github.com/sirupsen/logrus"
@@ -71,74 +70,11 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		}
 	}()
 
-	device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(device.DeviceInfo)
-
-	vfioGroup := filepath.Base(device.DeviceInfo.HostPath)
-	iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices")
-
-	deviceFiles, err := os.ReadDir(iommuDevicesPath)
+	device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo, false)
 	if err != nil {
 		return err
 	}
 
-	// Pass all devices in iommu group
-	for i, deviceFile := range deviceFiles {
-		//Get bdf of device eg 0000:00:1c.0
-		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath)
-		if err != nil {
-			return err
-		}
-		id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize)
-
-		pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
-		// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
-		// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
-		// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
-		ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
-		if err != nil {
-			return err
-		}
-		if ignorePCIDevice {
-			continue
-		}
-
-		var vfio config.VFIODev
-
-		switch vfioDeviceType {
-		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
-			isPCIe := isPCIeDevice(deviceBDF)
-			// Do not directly assign to `vfio` -- need to access field still
-			vfioPCI := config.VFIOPCIDev{
-				ID:       id,
-				Type:     vfioDeviceType,
-				BDF:      deviceBDF,
-				SysfsDev: deviceSysfsDev,
-				IsPCIe:   isPCIe,
-				Class:    pciClass,
-			}
-			if isPCIe {
-				vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
-				AllPCIeDevs[deviceBDF] = true
-			}
-			vfio = vfioPCI
-		case config.VFIOAPDeviceMediatedType:
-			devices, err := GetAPVFIODevices(deviceSysfsDev)
-			if err != nil {
-				return err
-			}
-			vfio = config.VFIOAPDev{
-				ID:        id,
-				SysfsDev:  deviceSysfsDev,
-				Type:      config.VFIOAPDeviceMediatedType,
-				APDevices: devices,
-			}
-		default:
-			return fmt.Errorf("Failed to append device: VFIO device type unrecognized")
-		}
-
-		device.VfioDevs = append(device.VfioDevs, &vfio)
-	}
-
 	coldPlug := device.DeviceInfo.ColdPlug
 	deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device")
 
diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go
index b973063e8..bec0ed70d 100644
--- a/src/runtime/pkg/katatestutils/utils.go
+++ b/src/runtime/pkg/katatestutils/utils.go
@@ -319,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
 	hotplug_vfio_on_root_bus =  ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
 	pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `
+	cold_plug_vfio =  "` + config.ColdPlugVFIO + `"
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"
diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go
index d65c93553..064aa72c7 100644
--- a/src/runtime/virtcontainers/qemu.go
+++ b/src/runtime/virtcontainers/qemu.go
@@ -718,20 +718,50 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		// At the sandbox level we alreaady checked that we have a
 		// VFIO device, pass-through of a PCIe device needs allocated
 		// mmemory in the firmware otherwise BARs cannot be mapped
-		if len(hypervisorConfig.VFIODevices) > 0 {
-			fwCfg := govmmQemu.FwCfg{
-				Name: "opt/ovmf/X-PciMmio64Mb",
-				Str:  "262144",
-			}
-			qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg)
+		// First check if we have a PCIe devices, otherwise ignore
+		err, fwCfg := q.appendFwCfgForConfidentialGuest(hypervisorConfig.VFIODevices)
+		if err != nil {
+			return err
+		}
+		if fwCfg != nil {
+			qemuConfig.FwCfg = append(qemuConfig.FwCfg, *fwCfg)
 		}
 	}
-
 	q.qemuConfig = qemuConfig
 
 	return err
 }
 
+// appendFwCfgForConfidentialGuest appends the firmware configuration for a
+// VFIO and PCIe device, otherwise it will be ignored.
+func (q *qemu) appendFwCfgForConfidentialGuest(vfioDevices []config.DeviceInfo) (error, *govmmQemu.FwCfg) {
+	var err error
+	for _, dev := range vfioDevices {
+		dev.HostPath, err = config.GetHostPath(dev, false, "")
+		if err != nil {
+			return err, nil
+		}
+		vfioDevs, err := drivers.GetAllVFIODevicesFromIOMMUGroup(dev, true)
+		if err != nil {
+			return err, nil
+		}
+		fwCfg := govmmQemu.FwCfg{}
+		for _, vfioDev := range vfioDevs {
+			switch (*vfioDev).GetType() {
+			case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+				if (*vfioDev).(config.VFIOPCIDev).IsPCIe {
+					fwCfg = govmmQemu.FwCfg{
+						Name: "opt/ovmf/X-PciMmio64Mb",
+						Str:  "262144",
+					}
+					return nil, &fwCfg
+				}
+			}
+		}
+	}
+	return nil, nil
+}
+
 func (q *qemu) checkBpfEnabled() {
 	if q.config.SeccompSandbox != "" {
 		out, err := os.ReadFile("/proc/sys/net/core/bpf_jit_enable")
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index 35b319394..ff3f7fe74 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -623,7 +623,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 
 	// If we have a confidential guest we need to cold-plug the PCIe VFIO devices
 	// until we have TDISP/IDE PCIe support.
-	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
+	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO == hv.RootPort)
 	var devs []config.DeviceInfo
 	for cnt, containers := range sandboxConfig.Containers {
 		for dev, device := range containers.DeviceInfos {

From dded731db3447712ab29b31a75f8c0c5a6f81a93 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Tue, 25 Apr 2023 09:53:08 +0000
Subject: [PATCH 010/150] gpu: Add OVMF setting for MMIO aperture

The default size of OVMFs aperture is too low to
initialized PCIe devices with huge BARs

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/virtcontainers/hypervisor.go |  4 --
 src/runtime/virtcontainers/qemu.go       | 53 +++++-------------------
 src/runtime/virtcontainers/sandbox.go    |  5 ---
 3 files changed, 10 insertions(+), 52 deletions(-)

diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
index f773e91d5..0a490ef57 100644
--- a/src/runtime/virtcontainers/hypervisor.go
+++ b/src/runtime/virtcontainers/hypervisor.go
@@ -509,10 +509,6 @@ type HypervisorConfig struct {
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
-	// VFIODevics are used to get PCIe device info early before the sandbox
-	// is started to make better PCIe topology decisions
-	VFIODevices []config.DeviceInfo
-
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root port, switch or no port
 	ColdPlugVFIO hv.PCIePort
diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go
index 064aa72c7..49b0d6abd 100644
--- a/src/runtime/virtcontainers/qemu.go
+++ b/src/runtime/virtcontainers/qemu.go
@@ -712,56 +712,23 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 
 	q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath)
 
-	// If we have a VFIO device we need to update the firmware configuration
-	// if executed in a trusted execution environment.
-	if hypervisorConfig.ConfidentialGuest {
-		// At the sandbox level we alreaady checked that we have a
-		// VFIO device, pass-through of a PCIe device needs allocated
-		// mmemory in the firmware otherwise BARs cannot be mapped
-		// First check if we have a PCIe devices, otherwise ignore
-		err, fwCfg := q.appendFwCfgForConfidentialGuest(hypervisorConfig.VFIODevices)
-		if err != nil {
-			return err
-		}
-		if fwCfg != nil {
-			qemuConfig.FwCfg = append(qemuConfig.FwCfg, *fwCfg)
+	// The default OVMF MMIO aperture is too small for some PCIe devices
+	// with huge BARs so we need to increase it.
+	// memSize64bit is in bytes, convert to MB, OVMF expects MB as a string
+	if strings.Contains(strings.ToLower(hypervisorConfig.FirmwarePath), "ovmf") {
+		pciMmio64Mb := fmt.Sprintf("%d", (memSize64bit / 1024 / 1024))
+		fwCfg := govmmQemu.FwCfg{
+			Name: "opt/ovmf/X-PciMmio64Mb",
+			Str:  pciMmio64Mb,
 		}
+		qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg)
 	}
+
 	q.qemuConfig = qemuConfig
 
 	return err
 }
 
-// appendFwCfgForConfidentialGuest appends the firmware configuration for a
-// VFIO and PCIe device, otherwise it will be ignored.
-func (q *qemu) appendFwCfgForConfidentialGuest(vfioDevices []config.DeviceInfo) (error, *govmmQemu.FwCfg) {
-	var err error
-	for _, dev := range vfioDevices {
-		dev.HostPath, err = config.GetHostPath(dev, false, "")
-		if err != nil {
-			return err, nil
-		}
-		vfioDevs, err := drivers.GetAllVFIODevicesFromIOMMUGroup(dev, true)
-		if err != nil {
-			return err, nil
-		}
-		fwCfg := govmmQemu.FwCfg{}
-		for _, vfioDev := range vfioDevs {
-			switch (*vfioDev).GetType() {
-			case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
-				if (*vfioDev).(config.VFIOPCIDev).IsPCIe {
-					fwCfg = govmmQemu.FwCfg{
-						Name: "opt/ovmf/X-PciMmio64Mb",
-						Str:  "262144",
-					}
-					return nil, &fwCfg
-				}
-			}
-		}
-	}
-	return nil, nil
-}
-
 func (q *qemu) checkBpfEnabled() {
 	if q.config.SeccompSandbox != "" {
 		out, err := os.ReadFile("/proc/sys/net/core/bpf_jit_enable")
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index ff3f7fe74..644904f75 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -639,11 +639,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 			}
 		}
 	}
-	// If we have a confidential guest, we need to add a specific
-	// firmware configuration to the hypervisor. We cannot do it here at
-	// the sandbox level we need to do that at the hypervisor level, capturing
-	// the devices here and processing in CreateVM().
-	sandboxConfig.HypervisorConfig.VFIODevices = devs
 
 	// store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil {

From 0f45b0faa92d073da4ac5e667bd61d584ab6122e Mon Sep 17 00:00:00 2001
From: Tamas K Lengyel <tamas.lengyel@intel.com>
Date: Tue, 25 Apr 2023 21:09:01 +0000
Subject: [PATCH 011/150] virtcontainers/clh_test.go: improve unit test
 coverage

Credit PR to Hackathon Team3

Fixes: #265

Signed-off-by: Tamas K Lengyel <tamas.lengyel@intel.com>
---
 src/runtime/virtcontainers/clh_test.go | 151 ++++++++++++++++---------
 1 file changed, 100 insertions(+), 51 deletions(-)

diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go
index a1c3c4b64..b5c800e95 100644
--- a/src/runtime/virtcontainers/clh_test.go
+++ b/src/runtime/virtcontainers/clh_test.go
@@ -9,6 +9,7 @@ package virtcontainers
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -422,6 +423,7 @@ func TestCloudHypervisorCleanupVM(t *testing.T) {
 	assert.Error(err, "persist.GetDriver() expected error")
 
 	clh.id = "cleanVMID"
+	clh.config.VMid = "cleanVMID"
 
 	err = clh.cleanupVM(true)
 	assert.NoError(err, "persist.GetDriver() unexpected error")
@@ -438,77 +440,86 @@ func TestCloudHypervisorCleanupVM(t *testing.T) {
 	assert.True(os.IsNotExist(err), "persist.GetDriver() unexpected error")
 }
 
-func TestClhCreateVMWithInitrd(t *testing.T) {
-	assert := assert.New(t)
-
-	clhConfig, err := newClhConfig()
-	assert.NoError(err)
-	clhConfig.ImagePath = ""
-	clhConfig.InitrdPath = testClhInitrdPath
-
-	store, err := persist.GetDriver()
-	assert.NoError(err)
-
-	clhConfig.VMStorePath = store.RunVMStoragePath()
-	clhConfig.RunStorePath = store.RunStoragePath()
-
-	network, err := NewNetwork()
-	assert.NoError(err)
-
-	clh := &cloudHypervisor{
-		config: clhConfig,
-	}
-
-	sandbox := &Sandbox{
-		ctx: context.Background(),
-		id:  "testSandbox",
-		config: &SandboxConfig{
-			HypervisorConfig: clhConfig,
-		},
-	}
-
-	err = clh.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig)
-	assert.NoError(err)
-	assert.Exactly(clhConfig, clh.config)
-}
-
 func TestClhCreateVM(t *testing.T) {
 	assert := assert.New(t)
 
-	clhConfig, err := newClhConfig()
-	assert.NoError(err)
-	assert.NotEmpty(clhConfig.ImagePath)
-
 	store, err := persist.GetDriver()
 	assert.NoError(err)
 
-	clhConfig.VMStorePath = store.RunVMStoragePath()
-	clhConfig.RunStorePath = store.RunStoragePath()
-
 	network, err := NewNetwork()
 	assert.NoError(err)
 
 	clh := &cloudHypervisor{
-		config: clhConfig,
-	}
-
-	sandbox := &Sandbox{
-		ctx: context.Background(),
-		id:  "testSandbox",
-		config: &SandboxConfig{
-			HypervisorConfig: clhConfig,
+		config: HypervisorConfig{
+			VMStorePath:  store.RunVMStoragePath(),
+			RunStorePath: store.RunStoragePath(),
 		},
 	}
 
-	err = clh.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig)
+	config0, err := newClhConfig()
 	assert.NoError(err)
-	assert.Exactly(clhConfig, clh.config)
+
+	config1, err := newClhConfig()
+	assert.NoError(err)
+	config1.ImagePath = ""
+	config1.InitrdPath = testClhInitrdPath
+
+	config2, err := newClhConfig()
+	assert.NoError(err)
+	config2.Debug = true
+
+	config3, err := newClhConfig()
+	assert.NoError(err)
+	config3.Debug = true
+	config3.ConfidentialGuest = true
+
+	config4, err := newClhConfig()
+	assert.NoError(err)
+	config4.SGXEPCSize = 1
+
+	config5, err := newClhConfig()
+	assert.NoError(err)
+	config5.SharedFS = config.VirtioFSNydus
+
+	type testData struct {
+		config      HypervisorConfig
+		expectError bool
+		configMatch bool
+	}
+
+	data := []testData{
+		{config0, false, true},
+		{config1, false, true},
+		{config2, false, true},
+		{config3, true, false},
+		{config4, false, true},
+		{config5, false, true},
+	}
+
+	for i, d := range data {
+		msg := fmt.Sprintf("test[%d]", i)
+
+		err = clh.CreateVM(context.Background(), "testSandbox", network, &d.config)
+
+		if d.expectError {
+			assert.Error(err, msg)
+			continue
+		}
+
+		assert.NoError(err, msg)
+
+		if d.configMatch {
+			assert.Exactly(d.config, clh.config, msg)
+		}
+	}
 }
 
 func TestCloudHypervisorStartSandbox(t *testing.T) {
 	assert := assert.New(t)
 	clhConfig, err := newClhConfig()
 	assert.NoError(err)
+	clhConfig.Debug = true
+	clhConfig.DisableSeccomp = true
 
 	store, err := persist.GetDriver()
 	assert.NoError(err)
@@ -530,6 +541,44 @@ func TestCloudHypervisorStartSandbox(t *testing.T) {
 
 	err = clh.StartVM(context.Background(), 10)
 	assert.NoError(err)
+
+	_, err = clh.loadVirtiofsDaemon("/tmp/xyzabc")
+	assert.NoError(err)
+
+	err = clh.stopVirtiofsDaemon(context.Background())
+	assert.NoError(err)
+
+	_, _, err = clh.GetVMConsole(context.Background(), "test")
+	assert.NoError(err)
+
+	_, err = clh.GetThreadIDs(context.Background())
+	assert.NoError(err)
+
+	assert.True(clh.getClhStopSandboxTimeout().Nanoseconds() != 0)
+
+	pid := clh.GetPids()
+	assert.True(pid[0] != 0)
+
+	pid2 := *clh.GetVirtioFsPid()
+	assert.True(pid2 == 0)
+
+	mem := clh.GetTotalMemoryMB(context.Background())
+	assert.True(mem == 0)
+
+	err = clh.PauseVM(context.Background())
+	assert.NoError(err)
+
+	err = clh.SaveVM()
+	assert.NoError(err)
+
+	err = clh.ResumeVM(context.Background())
+	assert.NoError(err)
+
+	err = clh.Check()
+	assert.NoError(err)
+
+	err = clh.Cleanup(context.Background())
+	assert.NoError(err)
 }
 
 func TestCloudHypervisorResizeMemory(t *testing.T) {

From 5226f15c8462538a3820afbb8b25b1a06b32fa85 Mon Sep 17 00:00:00 2001
From: Byron Marohn <byron.marohn@intel.com>
Date: Wed, 26 Apr 2023 16:36:52 -0400
Subject: [PATCH 012/150] gha: Fix Body Line Length action flagging empty body
 commit messages

Change the Body Line Length workflow to not trigger when the commit
message contains only a message without a body. Other workflows will
flag the missing body sections, and it was confusing to have an error
message that said 'Body line too long (max 150)' when this was not
actually the case.

Fixes: #5561

Co-authored-by: Jayant Singh <jayant.singh@intel.com>
Co-authored-by: Luke Phillips <lucas.phillips@intel.com>
Signed-off-by: Byron Marohn <byron.marohn@intel.com>
Signed-off-by: Jayant Singh <jayant.singh@intel.com>
Signed-off-by: Luke Phillips <lucas.phillips@intel.com>
Signed-off-by: Kelby Madal-Hellmuth <kelby.madal-hellmuth@intel.com>
Signed-off-by: Liz Lawrens <liz.lawrens@intel.com>
---
 .github/workflows/commit-message-check.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/commit-message-check.yaml b/.github/workflows/commit-message-check.yaml
index fbdb02b6d..3cde335bd 100644
--- a/.github/workflows/commit-message-check.yaml
+++ b/.github/workflows/commit-message-check.yaml
@@ -62,6 +62,9 @@ jobs:
         #   to be specified at the start of the regex as the action is passed
         #   the entire commit message.
         #
+        # - This check will pass if the commit message only contains a subject
+        #   line, as other body message properties are enforced elsewhere.
+        #
         # - Body lines *can* be longer than the maximum if they start
         #   with a non-alphabetic character or if there is no whitespace in
         #   the line.
@@ -75,7 +78,7 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$'
+        pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)'
         error: 'Body line too long (max 150)'
         post_error: ${{ env.error_msg }}
 

From 205909fbedae44dd4c2482c4b28ffb9e094a1e42 Mon Sep 17 00:00:00 2001
From: Feng Wang <fwang@confluent.io>
Date: Wed, 26 Apr 2023 15:38:08 -0700
Subject: [PATCH 013/150] runtime: Fix virtiofs fd leak

The kata runtime invokes removeStaleVirtiofsShareMounts after
a container is stopped to clean up the stale virtiofs file caches.

Fixes: #6455
Signed-off-by: Feng Wang <fwang@confluent.io>
---
 src/agent/rustjail/src/mount.rs               |   4 +-
 src/agent/src/rpc.rs                          |  25 +
 src/libs/protocols/protos/agent.proto         |   3 +
 src/runtime/virtcontainers/agent.go           |   3 +
 src/runtime/virtcontainers/container.go       |   4 +
 .../virtcontainers/container_linux_test.go    |   1 +
 src/runtime/virtcontainers/container_test.go  |   1 +
 src/runtime/virtcontainers/kata_agent.go      |  79 +-
 src/runtime/virtcontainers/kata_agent_test.go |   1 +
 src/runtime/virtcontainers/mock_agent.go      |   5 +
 .../pkg/agent/protocols/grpc/agent.pb.go      | 702 +++++++++++-------
 src/runtime/virtcontainers/pkg/mock/mock.go   |   4 +
 12 files changed, 521 insertions(+), 311 deletions(-)

diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs
index 76922024d..d9ba15041 100644
--- a/src/agent/rustjail/src/mount.rs
+++ b/src/agent/rustjail/src/mount.rs
@@ -35,7 +35,7 @@ use crate::log_child;
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
 #[derive(std::fmt::Debug, PartialEq)]
 pub struct Info {
-    mount_point: String,
+    pub mount_point: String,
     optional: String,
     fstype: String,
 }
@@ -553,7 +553,7 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> {
 
 // Parse /proc/self/mountinfo because comparing Dev and ino does not work from
 // bind mounts
-fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
+pub fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
     let file = File::open(mountinfo_path)?;
     let reader = BufReader::new(file);
     let mut infos = Vec::new();
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
index 478653904..447768b6b 100644
--- a/src/agent/src/rpc.rs
+++ b/src/agent/src/rpc.rs
@@ -40,6 +40,7 @@ use protocols::types::Interface;
 use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
 use rustjail::cgroups::notifier;
 use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
+use rustjail::mount::parse_mount_table;
 use rustjail::process::Process;
 use rustjail::specconv::CreateOpts;
 
@@ -96,6 +97,7 @@ const USR_IP6TABLES_SAVE: &str = "/usr/sbin/ip6tables-save";
 const IP6TABLES_SAVE: &str = "/sbin/ip6tables-save";
 const USR_IP6TABLES_RESTORE: &str = "/usr/sbin/ip6tables-save";
 const IP6TABLES_RESTORE: &str = "/sbin/ip6tables-restore";
+const KATA_GUEST_SHARE_DIR: &str = "/run/kata-containers/shared/containers/";
 
 const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer";
 const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size";
@@ -829,6 +831,29 @@ impl agent_ttrpc::AgentService for AgentService {
         Ok(Empty::new())
     }
 
+    async fn remove_stale_virtiofs_share_mounts(
+        &self,
+        ctx: &TtrpcContext,
+        req: protocols::agent::RemoveStaleVirtiofsShareMountsRequest,
+    ) -> ttrpc::Result<Empty> {
+        trace_rpc_call!(ctx, "remove_stale_virtiofs_share_mounts", req);
+        is_allowed!(req);
+        let mount_infos = parse_mount_table("/proc/self/mountinfo")
+            .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
+        for m in &mount_infos {
+            if m.mount_point.starts_with(KATA_GUEST_SHARE_DIR) {
+                // stat the mount point, virtiofs daemon will remove the stale cache and release the fds if the mount point doesn't exist any more.
+                // More details in https://github.com/kata-containers/kata-containers/issues/6455#issuecomment-1477137277
+                match stat::stat(Path::new(&m.mount_point)) {
+                    Ok(_) => info!(sl!(), "stat {} success", m.mount_point),
+                    Err(e) => info!(sl!(), "stat {} failed: {}", m.mount_point, e),
+                }
+            }
+        }
+
+        Ok(Empty::new())
+    }
+
     async fn write_stdin(
         &self,
         _ctx: &TtrpcContext,
diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto
index da4377f0d..3ad755256 100644
--- a/src/libs/protocols/protos/agent.proto
+++ b/src/libs/protocols/protos/agent.proto
@@ -38,6 +38,7 @@ service AgentService {
 	rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse);
 	rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty);
 	rpc ResumeContainer(ResumeContainerRequest) returns (google.protobuf.Empty);
+	rpc RemoveStaleVirtiofsShareMounts(RemoveStaleVirtiofsShareMountsRequest) returns (google.protobuf.Empty);
 
 	// stdio
 	rpc WriteStdin(WriteStreamRequest) returns (WriteStreamResponse);
@@ -301,6 +302,8 @@ message CreateSandboxRequest {
 message DestroySandboxRequest {
 }
 
+message RemoveStaleVirtiofsShareMountsRequest {}
+
 message Interfaces {
 	repeated types.Interface Interfaces = 1;
 }
diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go
index 6ff8f91ac..ddf11d9ce 100644
--- a/src/runtime/virtcontainers/agent.go
+++ b/src/runtime/virtcontainers/agent.go
@@ -138,6 +138,9 @@ type agent interface {
 	// resumeContainer will resume a paused container
 	resumeContainer(ctx context.Context, sandbox *Sandbox, c Container) error
 
+	// removeStaleVirtiofsShareMounts will tell the agent to remove stale virtiofs share mounts in the guest.
+	removeStaleVirtiofsShareMounts(ctx context.Context) error
+
 	// configure will update agent settings based on provided arguments
 	configure(ctx context.Context, h Hypervisor, id, sharePath string, config KataAgentConfig) error
 
diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go
index f812f0f4c..c0f0789d1 100644
--- a/src/runtime/virtcontainers/container.go
+++ b/src/runtime/virtcontainers/container.go
@@ -1020,6 +1020,10 @@ func (c *Container) stop(ctx context.Context, force bool) error {
 		}
 	}
 
+	if err := c.sandbox.agent.removeStaleVirtiofsShareMounts(ctx); err != nil && !force {
+		return err
+	}
+
 	if err := c.detachDevices(ctx); err != nil && !force {
 		return err
 	}
diff --git a/src/runtime/virtcontainers/container_linux_test.go b/src/runtime/virtcontainers/container_linux_test.go
index 581461e61..7666e451d 100644
--- a/src/runtime/virtcontainers/container_linux_test.go
+++ b/src/runtime/virtcontainers/container_linux_test.go
@@ -125,6 +125,7 @@ func TestUnmountHostMountsRemoveBindHostPath(t *testing.T) {
 		ctx:    context.Background(),
 		id:     "foobar",
 		config: &SandboxConfig{},
+		agent:  newMockAgent(),
 	}
 
 	fsShare, err := NewFilesystemShare(sandbox)
diff --git a/src/runtime/virtcontainers/container_test.go b/src/runtime/virtcontainers/container_test.go
index ac2ce540c..6a04ec1d4 100644
--- a/src/runtime/virtcontainers/container_test.go
+++ b/src/runtime/virtcontainers/container_test.go
@@ -302,6 +302,7 @@ func TestMountSharedDirMounts(t *testing.T) {
 	sandbox := &Sandbox{
 		ctx:        context.Background(),
 		id:         "foobar",
+		agent:      newMockAgent(),
 		hypervisor: &mockHypervisor{},
 		config: &SandboxConfig{
 			HypervisorConfig: HypervisorConfig{
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
index 9e5c8b34f..327f57343 100644
--- a/src/runtime/virtcontainers/kata_agent.go
+++ b/src/runtime/virtcontainers/kata_agent.go
@@ -113,41 +113,42 @@ var (
 )
 
 const (
-	grpcCheckRequest                 = "grpc.CheckRequest"
-	grpcExecProcessRequest           = "grpc.ExecProcessRequest"
-	grpcCreateSandboxRequest         = "grpc.CreateSandboxRequest"
-	grpcDestroySandboxRequest        = "grpc.DestroySandboxRequest"
-	grpcCreateContainerRequest       = "grpc.CreateContainerRequest"
-	grpcStartContainerRequest        = "grpc.StartContainerRequest"
-	grpcRemoveContainerRequest       = "grpc.RemoveContainerRequest"
-	grpcSignalProcessRequest         = "grpc.SignalProcessRequest"
-	grpcUpdateRoutesRequest          = "grpc.UpdateRoutesRequest"
-	grpcUpdateInterfaceRequest       = "grpc.UpdateInterfaceRequest"
-	grpcUpdateEphemeralMountsRequest = "grpc.UpdateEphemeralMountsRequest"
-	grpcListInterfacesRequest        = "grpc.ListInterfacesRequest"
-	grpcListRoutesRequest            = "grpc.ListRoutesRequest"
-	grpcAddARPNeighborsRequest       = "grpc.AddARPNeighborsRequest"
-	grpcOnlineCPUMemRequest          = "grpc.OnlineCPUMemRequest"
-	grpcUpdateContainerRequest       = "grpc.UpdateContainerRequest"
-	grpcWaitProcessRequest           = "grpc.WaitProcessRequest"
-	grpcTtyWinResizeRequest          = "grpc.TtyWinResizeRequest"
-	grpcWriteStreamRequest           = "grpc.WriteStreamRequest"
-	grpcCloseStdinRequest            = "grpc.CloseStdinRequest"
-	grpcStatsContainerRequest        = "grpc.StatsContainerRequest"
-	grpcPauseContainerRequest        = "grpc.PauseContainerRequest"
-	grpcResumeContainerRequest       = "grpc.ResumeContainerRequest"
-	grpcReseedRandomDevRequest       = "grpc.ReseedRandomDevRequest"
-	grpcGuestDetailsRequest          = "grpc.GuestDetailsRequest"
-	grpcMemHotplugByProbeRequest     = "grpc.MemHotplugByProbeRequest"
-	grpcCopyFileRequest              = "grpc.CopyFileRequest"
-	grpcSetGuestDateTimeRequest      = "grpc.SetGuestDateTimeRequest"
-	grpcGetOOMEventRequest           = "grpc.GetOOMEventRequest"
-	grpcGetMetricsRequest            = "grpc.GetMetricsRequest"
-	grpcAddSwapRequest               = "grpc.AddSwapRequest"
-	grpcVolumeStatsRequest           = "grpc.VolumeStatsRequest"
-	grpcResizeVolumeRequest          = "grpc.ResizeVolumeRequest"
-	grpcGetIPTablesRequest           = "grpc.GetIPTablesRequest"
-	grpcSetIPTablesRequest           = "grpc.SetIPTablesRequest"
+	grpcCheckRequest                          = "grpc.CheckRequest"
+	grpcExecProcessRequest                    = "grpc.ExecProcessRequest"
+	grpcCreateSandboxRequest                  = "grpc.CreateSandboxRequest"
+	grpcDestroySandboxRequest                 = "grpc.DestroySandboxRequest"
+	grpcCreateContainerRequest                = "grpc.CreateContainerRequest"
+	grpcStartContainerRequest                 = "grpc.StartContainerRequest"
+	grpcRemoveContainerRequest                = "grpc.RemoveContainerRequest"
+	grpcSignalProcessRequest                  = "grpc.SignalProcessRequest"
+	grpcUpdateRoutesRequest                   = "grpc.UpdateRoutesRequest"
+	grpcUpdateInterfaceRequest                = "grpc.UpdateInterfaceRequest"
+	grpcUpdateEphemeralMountsRequest          = "grpc.UpdateEphemeralMountsRequest"
+	grpcRemoveStaleVirtiofsShareMountsRequest = "grpc.RemoveStaleVirtiofsShareMountsRequest"
+	grpcListInterfacesRequest                 = "grpc.ListInterfacesRequest"
+	grpcListRoutesRequest                     = "grpc.ListRoutesRequest"
+	grpcAddARPNeighborsRequest                = "grpc.AddARPNeighborsRequest"
+	grpcOnlineCPUMemRequest                   = "grpc.OnlineCPUMemRequest"
+	grpcUpdateContainerRequest                = "grpc.UpdateContainerRequest"
+	grpcWaitProcessRequest                    = "grpc.WaitProcessRequest"
+	grpcTtyWinResizeRequest                   = "grpc.TtyWinResizeRequest"
+	grpcWriteStreamRequest                    = "grpc.WriteStreamRequest"
+	grpcCloseStdinRequest                     = "grpc.CloseStdinRequest"
+	grpcStatsContainerRequest                 = "grpc.StatsContainerRequest"
+	grpcPauseContainerRequest                 = "grpc.PauseContainerRequest"
+	grpcResumeContainerRequest                = "grpc.ResumeContainerRequest"
+	grpcReseedRandomDevRequest                = "grpc.ReseedRandomDevRequest"
+	grpcGuestDetailsRequest                   = "grpc.GuestDetailsRequest"
+	grpcMemHotplugByProbeRequest              = "grpc.MemHotplugByProbeRequest"
+	grpcCopyFileRequest                       = "grpc.CopyFileRequest"
+	grpcSetGuestDateTimeRequest               = "grpc.SetGuestDateTimeRequest"
+	grpcGetOOMEventRequest                    = "grpc.GetOOMEventRequest"
+	grpcGetMetricsRequest                     = "grpc.GetMetricsRequest"
+	grpcAddSwapRequest                        = "grpc.AddSwapRequest"
+	grpcVolumeStatsRequest                    = "grpc.VolumeStatsRequest"
+	grpcResizeVolumeRequest                   = "grpc.ResizeVolumeRequest"
+	grpcGetIPTablesRequest                    = "grpc.GetIPTablesRequest"
+	grpcSetIPTablesRequest                    = "grpc.SetIPTablesRequest"
 )
 
 // newKataAgent returns an agent from an agent type.
@@ -1947,6 +1948,11 @@ func (k *kataAgent) reseedRNG(ctx context.Context, data []byte) error {
 	return err
 }
 
+func (k *kataAgent) removeStaleVirtiofsShareMounts(ctx context.Context) error {
+	_, err := k.sendReq(ctx, &grpc.RemoveStaleVirtiofsShareMountsRequest{})
+	return err
+}
+
 type reqFunc func(context.Context, interface{}) (interface{}, error)
 
 func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
@@ -2056,6 +2062,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers[grpcSetIPTablesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.SetIPTables(ctx, req.(*grpc.SetIPTablesRequest))
 	}
+	k.reqHandlers[grpcRemoveStaleVirtiofsShareMountsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.RemoveStaleVirtiofsShareMounts(ctx, req.(*grpc.RemoveStaleVirtiofsShareMountsRequest))
+	}
 }
 
 func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) {
diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go
index b2564e5a8..c7fa059dc 100644
--- a/src/runtime/virtcontainers/kata_agent_test.go
+++ b/src/runtime/virtcontainers/kata_agent_test.go
@@ -846,6 +846,7 @@ func TestAgentCreateContainer(t *testing.T) {
 			},
 		},
 		hypervisor: &mockHypervisor{},
+		agent:      newMockAgent(),
 	}
 
 	fsShare, err := NewFilesystemShare(sandbox)
diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go
index 72e865c4a..5d6b0f9d6 100644
--- a/src/runtime/virtcontainers/mock_agent.go
+++ b/src/runtime/virtcontainers/mock_agent.go
@@ -141,6 +141,11 @@ func (n *mockAgent) waitProcess(ctx context.Context, c *Container, processID str
 	return 0, nil
 }
 
+// removeStaleVirtiofsShareMounts is the Noop agent removeStaleVirtiofsShareMounts implementation. It does nothing.
+func (n *mockAgent) removeStaleVirtiofsShareMounts(ctx context.Context) error {
+	return nil
+}
+
 // winsizeProcess is the Noop agent process tty resizer. It does nothing.
 func (n *mockAgent) winsizeProcess(ctx context.Context, c *Container, processID string, height, width uint32) error {
 	return nil
diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go
index 36af23886..a8dd81104 100644
--- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go
+++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/agent.proto
+// source: agent.proto
 
 package grpc
 
@@ -50,7 +50,7 @@ type CreateContainerRequest struct {
 func (m *CreateContainerRequest) Reset()      { *m = CreateContainerRequest{} }
 func (*CreateContainerRequest) ProtoMessage() {}
 func (*CreateContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{0}
+	return fileDescriptor_56ede974c0020f77, []int{0}
 }
 func (m *CreateContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -89,7 +89,7 @@ type StartContainerRequest struct {
 func (m *StartContainerRequest) Reset()      { *m = StartContainerRequest{} }
 func (*StartContainerRequest) ProtoMessage() {}
 func (*StartContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{1}
+	return fileDescriptor_56ede974c0020f77, []int{1}
 }
 func (m *StartContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -134,7 +134,7 @@ type RemoveContainerRequest struct {
 func (m *RemoveContainerRequest) Reset()      { *m = RemoveContainerRequest{} }
 func (*RemoveContainerRequest) ProtoMessage() {}
 func (*RemoveContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{2}
+	return fileDescriptor_56ede974c0020f77, []int{2}
 }
 func (m *RemoveContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -176,7 +176,7 @@ type ExecProcessRequest struct {
 func (m *ExecProcessRequest) Reset()      { *m = ExecProcessRequest{} }
 func (*ExecProcessRequest) ProtoMessage() {}
 func (*ExecProcessRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{3}
+	return fileDescriptor_56ede974c0020f77, []int{3}
 }
 func (m *ExecProcessRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -220,7 +220,7 @@ type SignalProcessRequest struct {
 func (m *SignalProcessRequest) Reset()      { *m = SignalProcessRequest{} }
 func (*SignalProcessRequest) ProtoMessage() {}
 func (*SignalProcessRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{4}
+	return fileDescriptor_56ede974c0020f77, []int{4}
 }
 func (m *SignalProcessRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -260,7 +260,7 @@ type WaitProcessRequest struct {
 func (m *WaitProcessRequest) Reset()      { *m = WaitProcessRequest{} }
 func (*WaitProcessRequest) ProtoMessage() {}
 func (*WaitProcessRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{5}
+	return fileDescriptor_56ede974c0020f77, []int{5}
 }
 func (m *WaitProcessRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -299,7 +299,7 @@ type WaitProcessResponse struct {
 func (m *WaitProcessResponse) Reset()      { *m = WaitProcessResponse{} }
 func (*WaitProcessResponse) ProtoMessage() {}
 func (*WaitProcessResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{6}
+	return fileDescriptor_56ede974c0020f77, []int{6}
 }
 func (m *WaitProcessResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -339,7 +339,7 @@ type UpdateContainerRequest struct {
 func (m *UpdateContainerRequest) Reset()      { *m = UpdateContainerRequest{} }
 func (*UpdateContainerRequest) ProtoMessage() {}
 func (*UpdateContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{7}
+	return fileDescriptor_56ede974c0020f77, []int{7}
 }
 func (m *UpdateContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -378,7 +378,7 @@ type StatsContainerRequest struct {
 func (m *StatsContainerRequest) Reset()      { *m = StatsContainerRequest{} }
 func (*StatsContainerRequest) ProtoMessage() {}
 func (*StatsContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{8}
+	return fileDescriptor_56ede974c0020f77, []int{8}
 }
 func (m *StatsContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -417,7 +417,7 @@ type PauseContainerRequest struct {
 func (m *PauseContainerRequest) Reset()      { *m = PauseContainerRequest{} }
 func (*PauseContainerRequest) ProtoMessage() {}
 func (*PauseContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{9}
+	return fileDescriptor_56ede974c0020f77, []int{9}
 }
 func (m *PauseContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -456,7 +456,7 @@ type ResumeContainerRequest struct {
 func (m *ResumeContainerRequest) Reset()      { *m = ResumeContainerRequest{} }
 func (*ResumeContainerRequest) ProtoMessage() {}
 func (*ResumeContainerRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{10}
+	return fileDescriptor_56ede974c0020f77, []int{10}
 }
 func (m *ResumeContainerRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -498,7 +498,7 @@ type CpuUsage struct {
 func (m *CpuUsage) Reset()      { *m = CpuUsage{} }
 func (*CpuUsage) ProtoMessage() {}
 func (*CpuUsage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{11}
+	return fileDescriptor_56ede974c0020f77, []int{11}
 }
 func (m *CpuUsage) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -539,7 +539,7 @@ type ThrottlingData struct {
 func (m *ThrottlingData) Reset()      { *m = ThrottlingData{} }
 func (*ThrottlingData) ProtoMessage() {}
 func (*ThrottlingData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{12}
+	return fileDescriptor_56ede974c0020f77, []int{12}
 }
 func (m *ThrottlingData) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -579,7 +579,7 @@ type CpuStats struct {
 func (m *CpuStats) Reset()      { *m = CpuStats{} }
 func (*CpuStats) ProtoMessage() {}
 func (*CpuStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{13}
+	return fileDescriptor_56ede974c0020f77, []int{13}
 }
 func (m *CpuStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -619,7 +619,7 @@ type PidsStats struct {
 func (m *PidsStats) Reset()      { *m = PidsStats{} }
 func (*PidsStats) ProtoMessage() {}
 func (*PidsStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{14}
+	return fileDescriptor_56ede974c0020f77, []int{14}
 }
 func (m *PidsStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -661,7 +661,7 @@ type MemoryData struct {
 func (m *MemoryData) Reset()      { *m = MemoryData{} }
 func (*MemoryData) ProtoMessage() {}
 func (*MemoryData) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{15}
+	return fileDescriptor_56ede974c0020f77, []int{15}
 }
 func (m *MemoryData) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -705,7 +705,7 @@ type MemoryStats struct {
 func (m *MemoryStats) Reset()      { *m = MemoryStats{} }
 func (*MemoryStats) ProtoMessage() {}
 func (*MemoryStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{16}
+	return fileDescriptor_56ede974c0020f77, []int{16}
 }
 func (m *MemoryStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -747,7 +747,7 @@ type BlkioStatsEntry struct {
 func (m *BlkioStatsEntry) Reset()      { *m = BlkioStatsEntry{} }
 func (*BlkioStatsEntry) ProtoMessage() {}
 func (*BlkioStatsEntry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{17}
+	return fileDescriptor_56ede974c0020f77, []int{17}
 }
 func (m *BlkioStatsEntry) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -793,7 +793,7 @@ type BlkioStats struct {
 func (m *BlkioStats) Reset()      { *m = BlkioStats{} }
 func (*BlkioStats) ProtoMessage() {}
 func (*BlkioStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{18}
+	return fileDescriptor_56ede974c0020f77, []int{18}
 }
 func (m *BlkioStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -834,7 +834,7 @@ type HugetlbStats struct {
 func (m *HugetlbStats) Reset()      { *m = HugetlbStats{} }
 func (*HugetlbStats) ProtoMessage() {}
 func (*HugetlbStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{19}
+	return fileDescriptor_56ede974c0020f77, []int{19}
 }
 func (m *HugetlbStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -877,7 +877,7 @@ type CgroupStats struct {
 func (m *CgroupStats) Reset()      { *m = CgroupStats{} }
 func (*CgroupStats) ProtoMessage() {}
 func (*CgroupStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{20}
+	return fileDescriptor_56ede974c0020f77, []int{20}
 }
 func (m *CgroupStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -924,7 +924,7 @@ type NetworkStats struct {
 func (m *NetworkStats) Reset()      { *m = NetworkStats{} }
 func (*NetworkStats) ProtoMessage() {}
 func (*NetworkStats) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{21}
+	return fileDescriptor_56ede974c0020f77, []int{21}
 }
 func (m *NetworkStats) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -964,7 +964,7 @@ type StatsContainerResponse struct {
 func (m *StatsContainerResponse) Reset()      { *m = StatsContainerResponse{} }
 func (*StatsContainerResponse) ProtoMessage() {}
 func (*StatsContainerResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{22}
+	return fileDescriptor_56ede974c0020f77, []int{22}
 }
 func (m *StatsContainerResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1005,7 +1005,7 @@ type WriteStreamRequest struct {
 func (m *WriteStreamRequest) Reset()      { *m = WriteStreamRequest{} }
 func (*WriteStreamRequest) ProtoMessage() {}
 func (*WriteStreamRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{23}
+	return fileDescriptor_56ede974c0020f77, []int{23}
 }
 func (m *WriteStreamRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1044,7 +1044,7 @@ type WriteStreamResponse struct {
 func (m *WriteStreamResponse) Reset()      { *m = WriteStreamResponse{} }
 func (*WriteStreamResponse) ProtoMessage() {}
 func (*WriteStreamResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{24}
+	return fileDescriptor_56ede974c0020f77, []int{24}
 }
 func (m *WriteStreamResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1085,7 +1085,7 @@ type ReadStreamRequest struct {
 func (m *ReadStreamRequest) Reset()      { *m = ReadStreamRequest{} }
 func (*ReadStreamRequest) ProtoMessage() {}
 func (*ReadStreamRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{25}
+	return fileDescriptor_56ede974c0020f77, []int{25}
 }
 func (m *ReadStreamRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1124,7 +1124,7 @@ type ReadStreamResponse struct {
 func (m *ReadStreamResponse) Reset()      { *m = ReadStreamResponse{} }
 func (*ReadStreamResponse) ProtoMessage() {}
 func (*ReadStreamResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{26}
+	return fileDescriptor_56ede974c0020f77, []int{26}
 }
 func (m *ReadStreamResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1164,7 +1164,7 @@ type CloseStdinRequest struct {
 func (m *CloseStdinRequest) Reset()      { *m = CloseStdinRequest{} }
 func (*CloseStdinRequest) ProtoMessage() {}
 func (*CloseStdinRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{27}
+	return fileDescriptor_56ede974c0020f77, []int{27}
 }
 func (m *CloseStdinRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1206,7 +1206,7 @@ type TtyWinResizeRequest struct {
 func (m *TtyWinResizeRequest) Reset()      { *m = TtyWinResizeRequest{} }
 func (*TtyWinResizeRequest) ProtoMessage() {}
 func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{28}
+	return fileDescriptor_56ede974c0020f77, []int{28}
 }
 func (m *TtyWinResizeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1249,7 +1249,7 @@ type KernelModule struct {
 func (m *KernelModule) Reset()      { *m = KernelModule{} }
 func (*KernelModule) ProtoMessage() {}
 func (*KernelModule) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{29}
+	return fileDescriptor_56ede974c0020f77, []int{29}
 }
 func (m *KernelModule) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1304,7 +1304,7 @@ type CreateSandboxRequest struct {
 func (m *CreateSandboxRequest) Reset()      { *m = CreateSandboxRequest{} }
 func (*CreateSandboxRequest) ProtoMessage() {}
 func (*CreateSandboxRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{30}
+	return fileDescriptor_56ede974c0020f77, []int{30}
 }
 func (m *CreateSandboxRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1342,7 +1342,7 @@ type DestroySandboxRequest struct {
 func (m *DestroySandboxRequest) Reset()      { *m = DestroySandboxRequest{} }
 func (*DestroySandboxRequest) ProtoMessage() {}
 func (*DestroySandboxRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{31}
+	return fileDescriptor_56ede974c0020f77, []int{31}
 }
 func (m *DestroySandboxRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1371,6 +1371,44 @@ func (m *DestroySandboxRequest) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_DestroySandboxRequest proto.InternalMessageInfo
 
+type RemoveStaleVirtiofsShareMountsRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *RemoveStaleVirtiofsShareMountsRequest) Reset()      { *m = RemoveStaleVirtiofsShareMountsRequest{} }
+func (*RemoveStaleVirtiofsShareMountsRequest) ProtoMessage() {}
+func (*RemoveStaleVirtiofsShareMountsRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_56ede974c0020f77, []int{32}
+}
+func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest.Merge(m, src)
+}
+func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Size() int {
+	return m.Size()
+}
+func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest proto.InternalMessageInfo
+
 type Interfaces struct {
 	Interfaces           []*protocols.Interface `protobuf:"bytes,1,rep,name=Interfaces,proto3" json:"Interfaces,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
@@ -1381,7 +1419,7 @@ type Interfaces struct {
 func (m *Interfaces) Reset()      { *m = Interfaces{} }
 func (*Interfaces) ProtoMessage() {}
 func (*Interfaces) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{32}
+	return fileDescriptor_56ede974c0020f77, []int{33}
 }
 func (m *Interfaces) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1420,7 +1458,7 @@ type Routes struct {
 func (m *Routes) Reset()      { *m = Routes{} }
 func (*Routes) ProtoMessage() {}
 func (*Routes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{33}
+	return fileDescriptor_56ede974c0020f77, []int{34}
 }
 func (m *Routes) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1459,7 +1497,7 @@ type UpdateInterfaceRequest struct {
 func (m *UpdateInterfaceRequest) Reset()      { *m = UpdateInterfaceRequest{} }
 func (*UpdateInterfaceRequest) ProtoMessage() {}
 func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{34}
+	return fileDescriptor_56ede974c0020f77, []int{35}
 }
 func (m *UpdateInterfaceRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1498,7 +1536,7 @@ type UpdateRoutesRequest struct {
 func (m *UpdateRoutesRequest) Reset()      { *m = UpdateRoutesRequest{} }
 func (*UpdateRoutesRequest) ProtoMessage() {}
 func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{35}
+	return fileDescriptor_56ede974c0020f77, []int{36}
 }
 func (m *UpdateRoutesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1537,7 +1575,7 @@ type UpdateEphemeralMountsRequest struct {
 func (m *UpdateEphemeralMountsRequest) Reset()      { *m = UpdateEphemeralMountsRequest{} }
 func (*UpdateEphemeralMountsRequest) ProtoMessage() {}
 func (*UpdateEphemeralMountsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{36}
+	return fileDescriptor_56ede974c0020f77, []int{37}
 }
 func (m *UpdateEphemeralMountsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1575,7 +1613,7 @@ type ListInterfacesRequest struct {
 func (m *ListInterfacesRequest) Reset()      { *m = ListInterfacesRequest{} }
 func (*ListInterfacesRequest) ProtoMessage() {}
 func (*ListInterfacesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{37}
+	return fileDescriptor_56ede974c0020f77, []int{38}
 }
 func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1613,7 +1651,7 @@ type ListRoutesRequest struct {
 func (m *ListRoutesRequest) Reset()      { *m = ListRoutesRequest{} }
 func (*ListRoutesRequest) ProtoMessage() {}
 func (*ListRoutesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{38}
+	return fileDescriptor_56ede974c0020f77, []int{39}
 }
 func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1652,7 +1690,7 @@ type ARPNeighbors struct {
 func (m *ARPNeighbors) Reset()      { *m = ARPNeighbors{} }
 func (*ARPNeighbors) ProtoMessage() {}
 func (*ARPNeighbors) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{39}
+	return fileDescriptor_56ede974c0020f77, []int{40}
 }
 func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1691,7 +1729,7 @@ type AddARPNeighborsRequest struct {
 func (m *AddARPNeighborsRequest) Reset()      { *m = AddARPNeighborsRequest{} }
 func (*AddARPNeighborsRequest) ProtoMessage() {}
 func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{40}
+	return fileDescriptor_56ede974c0020f77, []int{41}
 }
 func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1730,7 +1768,7 @@ type GetIPTablesRequest struct {
 func (m *GetIPTablesRequest) Reset()      { *m = GetIPTablesRequest{} }
 func (*GetIPTablesRequest) ProtoMessage() {}
 func (*GetIPTablesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{41}
+	return fileDescriptor_56ede974c0020f77, []int{42}
 }
 func (m *GetIPTablesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1770,7 +1808,7 @@ type GetIPTablesResponse struct {
 func (m *GetIPTablesResponse) Reset()      { *m = GetIPTablesResponse{} }
 func (*GetIPTablesResponse) ProtoMessage() {}
 func (*GetIPTablesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{42}
+	return fileDescriptor_56ede974c0020f77, []int{43}
 }
 func (m *GetIPTablesResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1812,7 +1850,7 @@ type SetIPTablesRequest struct {
 func (m *SetIPTablesRequest) Reset()      { *m = SetIPTablesRequest{} }
 func (*SetIPTablesRequest) ProtoMessage() {}
 func (*SetIPTablesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{43}
+	return fileDescriptor_56ede974c0020f77, []int{44}
 }
 func (m *SetIPTablesRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1852,7 +1890,7 @@ type SetIPTablesResponse struct {
 func (m *SetIPTablesResponse) Reset()      { *m = SetIPTablesResponse{} }
 func (*SetIPTablesResponse) ProtoMessage() {}
 func (*SetIPTablesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{44}
+	return fileDescriptor_56ede974c0020f77, []int{45}
 }
 func (m *SetIPTablesResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1898,7 +1936,7 @@ type OnlineCPUMemRequest struct {
 func (m *OnlineCPUMemRequest) Reset()      { *m = OnlineCPUMemRequest{} }
 func (*OnlineCPUMemRequest) ProtoMessage() {}
 func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{45}
+	return fileDescriptor_56ede974c0020f77, []int{46}
 }
 func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1938,7 +1976,7 @@ type ReseedRandomDevRequest struct {
 func (m *ReseedRandomDevRequest) Reset()      { *m = ReseedRandomDevRequest{} }
 func (*ReseedRandomDevRequest) ProtoMessage() {}
 func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{46}
+	return fileDescriptor_56ede974c0020f77, []int{47}
 }
 func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1988,7 +2026,7 @@ type AgentDetails struct {
 func (m *AgentDetails) Reset()      { *m = AgentDetails{} }
 func (*AgentDetails) ProtoMessage() {}
 func (*AgentDetails) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{47}
+	return fileDescriptor_56ede974c0020f77, []int{48}
 }
 func (m *AgentDetails) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2034,7 +2072,7 @@ type GuestDetailsRequest struct {
 func (m *GuestDetailsRequest) Reset()      { *m = GuestDetailsRequest{} }
 func (*GuestDetailsRequest) ProtoMessage() {}
 func (*GuestDetailsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{48}
+	return fileDescriptor_56ede974c0020f77, []int{49}
 }
 func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2076,7 +2114,7 @@ type GuestDetailsResponse struct {
 func (m *GuestDetailsResponse) Reset()      { *m = GuestDetailsResponse{} }
 func (*GuestDetailsResponse) ProtoMessage() {}
 func (*GuestDetailsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{49}
+	return fileDescriptor_56ede974c0020f77, []int{50}
 }
 func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2117,7 +2155,7 @@ type MemHotplugByProbeRequest struct {
 func (m *MemHotplugByProbeRequest) Reset()      { *m = MemHotplugByProbeRequest{} }
 func (*MemHotplugByProbeRequest) ProtoMessage() {}
 func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{50}
+	return fileDescriptor_56ede974c0020f77, []int{51}
 }
 func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2159,7 +2197,7 @@ type SetGuestDateTimeRequest struct {
 func (m *SetGuestDateTimeRequest) Reset()      { *m = SetGuestDateTimeRequest{} }
 func (*SetGuestDateTimeRequest) ProtoMessage() {}
 func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{51}
+	return fileDescriptor_56ede974c0020f77, []int{52}
 }
 func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2205,7 +2243,7 @@ type FSGroup struct {
 func (m *FSGroup) Reset()      { *m = FSGroup{} }
 func (*FSGroup) ProtoMessage() {}
 func (*FSGroup) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{52}
+	return fileDescriptor_56ede974c0020f77, []int{53}
 }
 func (m *FSGroup) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2273,7 +2311,7 @@ type Storage struct {
 func (m *Storage) Reset()      { *m = Storage{} }
 func (*Storage) ProtoMessage() {}
 func (*Storage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{53}
+	return fileDescriptor_56ede974c0020f77, []int{54}
 }
 func (m *Storage) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2344,7 +2382,7 @@ type Device struct {
 func (m *Device) Reset()      { *m = Device{} }
 func (*Device) ProtoMessage() {}
 func (*Device) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{54}
+	return fileDescriptor_56ede974c0020f77, []int{55}
 }
 func (m *Device) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2385,7 +2423,7 @@ type StringUser struct {
 func (m *StringUser) Reset()      { *m = StringUser{} }
 func (*StringUser) ProtoMessage() {}
 func (*StringUser) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{55}
+	return fileDescriptor_56ede974c0020f77, []int{56}
 }
 func (m *StringUser) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2442,7 +2480,7 @@ type CopyFileRequest struct {
 func (m *CopyFileRequest) Reset()      { *m = CopyFileRequest{} }
 func (*CopyFileRequest) ProtoMessage() {}
 func (*CopyFileRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{56}
+	return fileDescriptor_56ede974c0020f77, []int{57}
 }
 func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2480,7 +2518,7 @@ type GetOOMEventRequest struct {
 func (m *GetOOMEventRequest) Reset()      { *m = GetOOMEventRequest{} }
 func (*GetOOMEventRequest) ProtoMessage() {}
 func (*GetOOMEventRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{57}
+	return fileDescriptor_56ede974c0020f77, []int{58}
 }
 func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2519,7 +2557,7 @@ type OOMEvent struct {
 func (m *OOMEvent) Reset()      { *m = OOMEvent{} }
 func (*OOMEvent) ProtoMessage() {}
 func (*OOMEvent) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{58}
+	return fileDescriptor_56ede974c0020f77, []int{59}
 }
 func (m *OOMEvent) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2558,7 +2596,7 @@ type AddSwapRequest struct {
 func (m *AddSwapRequest) Reset()      { *m = AddSwapRequest{} }
 func (*AddSwapRequest) ProtoMessage() {}
 func (*AddSwapRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{59}
+	return fileDescriptor_56ede974c0020f77, []int{60}
 }
 func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2596,7 +2634,7 @@ type GetMetricsRequest struct {
 func (m *GetMetricsRequest) Reset()      { *m = GetMetricsRequest{} }
 func (*GetMetricsRequest) ProtoMessage() {}
 func (*GetMetricsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{60}
+	return fileDescriptor_56ede974c0020f77, []int{61}
 }
 func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2635,7 +2673,7 @@ type Metrics struct {
 func (m *Metrics) Reset()      { *m = Metrics{} }
 func (*Metrics) ProtoMessage() {}
 func (*Metrics) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{61}
+	return fileDescriptor_56ede974c0020f77, []int{62}
 }
 func (m *Metrics) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2675,7 +2713,7 @@ type VolumeStatsRequest struct {
 func (m *VolumeStatsRequest) Reset()      { *m = VolumeStatsRequest{} }
 func (*VolumeStatsRequest) ProtoMessage() {}
 func (*VolumeStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{62}
+	return fileDescriptor_56ede974c0020f77, []int{63}
 }
 func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2716,7 +2754,7 @@ type ResizeVolumeRequest struct {
 func (m *ResizeVolumeRequest) Reset()      { *m = ResizeVolumeRequest{} }
 func (*ResizeVolumeRequest) ProtoMessage() {}
 func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_712ce9a559fda969, []int{63}
+	return fileDescriptor_56ede974c0020f77, []int{64}
 }
 func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -2780,6 +2818,7 @@ func init() {
 	proto.RegisterType((*KernelModule)(nil), "grpc.KernelModule")
 	proto.RegisterType((*CreateSandboxRequest)(nil), "grpc.CreateSandboxRequest")
 	proto.RegisterType((*DestroySandboxRequest)(nil), "grpc.DestroySandboxRequest")
+	proto.RegisterType((*RemoveStaleVirtiofsShareMountsRequest)(nil), "grpc.RemoveStaleVirtiofsShareMountsRequest")
 	proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces")
 	proto.RegisterType((*Routes)(nil), "grpc.Routes")
 	proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest")
@@ -2814,216 +2853,215 @@ func init() {
 	proto.RegisterType((*ResizeVolumeRequest)(nil), "grpc.ResizeVolumeRequest")
 }
 
-func init() {
-	proto.RegisterFile("github.com/kata-containers/kata-containers/src/libs/protocols/protos/agent.proto", fileDescriptor_712ce9a559fda969)
-}
+func init() { proto.RegisterFile("agent.proto", fileDescriptor_56ede974c0020f77) }
 
-var fileDescriptor_712ce9a559fda969 = []byte{
-	// 3249 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0xcb, 0x72, 0x1c, 0x47,
-	0x72, 0x9a, 0x07, 0x30, 0x33, 0x39, 0x2f, 0x4c, 0x0f, 0x08, 0x0e, 0x47, 0x14, 0x44, 0x35, 0x25,
-	0x0a, 0x94, 0x4c, 0x40, 0xa2, 0x14, 0xa2, 0x1e, 0x21, 0xd3, 0x00, 0x08, 0x01, 0x90, 0x04, 0x71,
-	0xdc, 0x43, 0x58, 0x0e, 0x3b, 0xec, 0x8e, 0x46, 0x77, 0x61, 0xa6, 0x84, 0xe9, 0xae, 0x56, 0x75,
-	0x35, 0x08, 0xc8, 0x11, 0x0e, 0x9f, 0xec, 0x9b, 0x8f, 0xbe, 0xe9, 0x07, 0x1c, 0xfe, 0x03, 0x5f,
-	0x7d, 0x50, 0xf8, 0xb4, 0xc7, 0xbd, 0xec, 0xc6, 0x8a, 0x9f, 0xb0, 0x5f, 0xb0, 0x51, 0xaf, 0x7e,
-	0xcc, 0x03, 0xd2, 0x22, 0x18, 0xb1, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x56, 0x56, 0x66,
-	0xd5, 0xc0, 0x60, 0x84, 0xd9, 0x38, 0x3e, 0xd9, 0x74, 0x89, 0xbf, 0x75, 0xe6, 0x30, 0xe7, 0x81,
-	0x4b, 0x02, 0xe6, 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x35, 0xc1, 0x27, 0xd1, 0x56,
-	0x48, 0x09, 0x23, 0x2e, 0x99, 0xa8, 0xaf, 0x68, 0xcb, 0x19, 0xa1, 0x80, 0x6d, 0x0a, 0xc0, 0x28,
-	0x8f, 0x68, 0xe8, 0xf6, 0x6b, 0xc4, 0xc5, 0x12, 0xd1, 0xaf, 0xb9, 0x91, 0xfe, 0xac, 0xb3, 0xcb,
-	0x10, 0x45, 0x0a, 0x78, 0x75, 0x44, 0xc8, 0x68, 0x82, 0x24, 0x8f, 0x93, 0xf8, 0x74, 0x0b, 0xf9,
-	0x21, 0xbb, 0x94, 0x83, 0xe6, 0x8f, 0x45, 0x58, 0xdb, 0xa5, 0xc8, 0x61, 0x68, 0x57, 0x2b, 0x60,
-	0xa1, 0xef, 0x63, 0x14, 0x31, 0xe3, 0x0d, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x15, 0xee, 0x14,
-	0x36, 0x6a, 0x56, 0x3d, 0xc1, 0x1d, 0x7a, 0xc6, 0x4d, 0xa8, 0xa0, 0x0b, 0xe4, 0xf2, 0xd1, 0xa2,
-	0x18, 0x5d, 0xe6, 0xe0, 0xa1, 0x67, 0xbc, 0x0f, 0xf5, 0x88, 0x51, 0x1c, 0x8c, 0xec, 0x38, 0x42,
-	0xb4, 0x57, 0xba, 0x53, 0xd8, 0xa8, 0x3f, 0x5c, 0xd9, 0xe4, 0x2a, 0x6f, 0x0e, 0xc5, 0xc0, 0x71,
-	0x84, 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x83, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x5e, 0xf9,
-	0x4e, 0x69, 0xa3, 0xfe, 0xb0, 0x21, 0xc9, 0x9f, 0x08, 0xa4, 0xa5, 0x07, 0x8d, 0xfb, 0x50, 0x8d,
-	0x18, 0xa1, 0xce, 0x08, 0x45, 0xbd, 0x25, 0x41, 0xd8, 0xd4, 0x7c, 0x05, 0xd6, 0x4a, 0x86, 0x8d,
-	0xdb, 0x50, 0x7a, 0xba, 0x7b, 0xd8, 0x5b, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda,
-	0xb8, 0x0b, 0xcd, 0xc8, 0x09, 0xbc, 0x13, 0x72, 0x61, 0x87, 0xd8, 0x0b, 0xa2, 0x5e, 0xe5, 0x4e,
-	0x61, 0xa3, 0x6a, 0x35, 0x14, 0x72, 0xc0, 0x71, 0xe6, 0xa7, 0x70, 0x63, 0xc8, 0x1c, 0xca, 0xae,
-	0xe1, 0x1d, 0xf3, 0x18, 0xd6, 0x2c, 0xe4, 0x93, 0xf3, 0x6b, 0xb9, 0xb6, 0x07, 0x15, 0x86, 0x7d,
-	0x44, 0x62, 0x26, 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0x4f, 0x01, 0x8c, 0xbd, 0x0b, 0xe4, 0x0e,
-	0x28, 0x71, 0x51, 0x14, 0xfd, 0x85, 0x96, 0xeb, 0x6d, 0xa8, 0x84, 0x52, 0x81, 0x5e, 0x59, 0x90,
-	0xab, 0x55, 0xd0, 0x5a, 0xe9, 0x51, 0xf3, 0x3b, 0x58, 0x1d, 0xe2, 0x51, 0xe0, 0x4c, 0x5e, 0xa2,
-	0xbe, 0x6b, 0xb0, 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0x73, 0x00, 0xc6, 0xb7, 0x0e,
-	0x66, 0x2f, 0x4f, 0x92, 0xf9, 0x00, 0xba, 0x39, 0x8e, 0x51, 0x48, 0x82, 0x08, 0x09, 0x05, 0x98,
-	0xc3, 0xe2, 0x48, 0x30, 0x5b, 0xb2, 0x14, 0x64, 0x12, 0x58, 0x3b, 0x0e, 0xbd, 0x6b, 0xee, 0xa6,
-	0x87, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x7b, 0xa0, 0x28, 0x9c, 0xba, 0x2a, 0x9d, 0xfa, 0x35,
-	0x0e, 0xe2, 0x0b, 0x4b, 0x8f, 0x59, 0x29, 0x99, 0x8a, 0x4f, 0x16, 0x5d, 0x27, 0x3e, 0x3f, 0x85,
-	0x1b, 0x03, 0x27, 0x8e, 0xae, 0xa3, 0xab, 0xf9, 0x19, 0x8f, 0xed, 0x28, 0xf6, 0xaf, 0x35, 0xf9,
-	0xbf, 0x0b, 0x50, 0xdd, 0x0d, 0xe3, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0x75, 0xa8, 0x33, 0xc2, 0x9c,
-	0x89, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, 0x49, 0xf0, 0x06, 0x34, 0x42, 0x44, 0xdd,
-	0x30, 0x56, 0x14, 0xc5, 0x3b, 0xa5, 0x8d, 0xb2, 0x55, 0x97, 0x38, 0x49, 0xb2, 0x09, 0x5d, 0x31,
-	0x66, 0xe3, 0xc0, 0x3e, 0x43, 0x34, 0x40, 0x13, 0x9f, 0x78, 0x48, 0x04, 0x47, 0xd9, 0xea, 0x88,
-	0xa1, 0xc3, 0xe0, 0xab, 0x64, 0xc0, 0x78, 0x07, 0x3a, 0x09, 0x3d, 0x8f, 0x78, 0x41, 0x5d, 0x16,
-	0xd4, 0x6d, 0x45, 0x7d, 0xac, 0xd0, 0xe6, 0xbf, 0x42, 0xeb, 0xd9, 0x98, 0x12, 0xc6, 0x26, 0x38,
-	0x18, 0x3d, 0x71, 0x98, 0xc3, 0xb7, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1,
-	0x2e, 0x74, 0x98, 0xa4, 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x25, 0x19, 0x18, 0x28, 0xe2,
-	0xb7, 0xa0, 0x95, 0x12, 0xf3, 0xcd, 0xad, 0xf4, 0x6d, 0x26, 0xd8, 0x67, 0xd8, 0x47, 0xe6, 0xb9,
-	0xf0, 0x95, 0x58, 0x64, 0xe3, 0x5d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x08, 0x69, 0xc9, 0x08, 0xd1,
-	0xee, 0xb4, 0xaa, 0x89, 0x53, 0x3e, 0x87, 0x36, 0x4b, 0x14, 0xb7, 0x3d, 0x87, 0x39, 0xf9, 0xa0,
-	0xca, 0x5b, 0x65, 0xb5, 0x58, 0x0e, 0x36, 0x3f, 0x83, 0xda, 0x00, 0x7b, 0x91, 0x14, 0xdc, 0x83,
-	0x8a, 0x1b, 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x0a, 0x4b, 0x13, 0xec, 0x63, 0xa6,
-	0xcc, 0x94, 0x80, 0x49, 0x00, 0x8e, 0x90, 0x4f, 0xe8, 0xa5, 0x70, 0xd8, 0x2a, 0x2c, 0x65, 0x17,
-	0x57, 0x02, 0xc6, 0xab, 0x50, 0xf3, 0x9d, 0x8b, 0x64, 0x51, 0xf9, 0x48, 0xd5, 0x77, 0x2e, 0xa4,
-	0xf2, 0x3d, 0xa8, 0x9c, 0x3a, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x72, 0x56,
-	0xe0, 0xff, 0x15, 0xa1, 0x2e, 0x25, 0x4a, 0x85, 0x57, 0x61, 0xc9, 0x75, 0xdc, 0x71, 0x22, 0x52,
-	0x00, 0xc6, 0x3d, 0xad, 0x48, 0x31, 0x9b, 0xe1, 0x52, 0x4d, 0xb5, 0x6a, 0x5b, 0x00, 0xd1, 0x73,
-	0x27, 0x54, 0xba, 0x95, 0x16, 0x10, 0xd7, 0x38, 0x8d, 0x54, 0xf7, 0x03, 0x68, 0xc8, 0xb8, 0x53,
-	0x53, 0xca, 0x0b, 0xa6, 0xd4, 0x25, 0x95, 0x9c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x31, 0x46,
-	0xd4, 0xa1, 0xee, 0xf8, 0xb2, 0xb7, 0x24, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0x3c, 0x84,
-	0x25, 0x9e, 0x5b, 0xa2, 0xde, 0xb2, 0x38, 0xeb, 0x6e, 0x67, 0x59, 0x0a, 0x53, 0x37, 0xc5, 0xef,
-	0x5e, 0xc0, 0xe8, 0xa5, 0x25, 0x49, 0xfb, 0x1f, 0x03, 0xa4, 0x48, 0x63, 0x05, 0x4a, 0x67, 0xe8,
-	0x52, 0xed, 0x43, 0xfe, 0xc9, 0x9d, 0x73, 0xee, 0x4c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x5a, 0xfc,
-	0xb8, 0x60, 0xba, 0xd0, 0xde, 0x99, 0x9c, 0x61, 0x92, 0x99, 0xbe, 0x0a, 0x4b, 0xbe, 0xf3, 0x1d,
-	0xa1, 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84,
-	0xc2, 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x7d, 0x19, 0x20, 0x95,
-	0x62, 0x58, 0xd0, 0xc7, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x72, 0xc9, 0x50, 0x64, 0x53,
-	0xe4, 0xc6, 0x34, 0xc2, 0xe7, 0x7c, 0xfd, 0xb8, 0xd9, 0x37, 0xa4, 0xd9, 0x53, 0xba, 0x59, 0x37,
-	0x31, 0x19, 0xca, 0x79, 0x3b, 0x7c, 0x9a, 0xa5, 0x67, 0x19, 0x87, 0x70, 0x23, 0xe5, 0xe9, 0x65,
-	0xd8, 0x15, 0xaf, 0x62, 0xd7, 0x4d, 0xd8, 0x79, 0x29, 0xab, 0x3d, 0xe8, 0x62, 0x62, 0x7f, 0x1f,
-	0xa3, 0x38, 0xc7, 0xa8, 0x74, 0x15, 0xa3, 0x0e, 0x26, 0x7f, 0x2b, 0x26, 0xa4, 0x6c, 0x06, 0x70,
-	0x2b, 0x63, 0x25, 0xdf, 0xee, 0x19, 0x66, 0xe5, 0xab, 0x98, 0xad, 0x25, 0x5a, 0xf1, 0x7c, 0x90,
-	0x72, 0xfc, 0x12, 0xd6, 0x30, 0xb1, 0x9f, 0x3b, 0x98, 0x4d, 0xb3, 0x5b, 0xfa, 0x05, 0x23, 0xf9,
-	0x89, 0x96, 0xe7, 0x25, 0x8d, 0xf4, 0x11, 0x1d, 0xe5, 0x8c, 0x5c, 0xfe, 0x05, 0x23, 0x8f, 0xc4,
-	0x84, 0x94, 0xcd, 0x36, 0x74, 0x30, 0x99, 0xd6, 0xa6, 0x72, 0x15, 0x93, 0x36, 0x26, 0x79, 0x4d,
-	0x76, 0xa0, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x0d, 0x82, 0xea, 0x55, 0x2c, 0x56, 0x14, 0x7d, 0xc2,
-	0xc3, 0xfc, 0x47, 0x68, 0x1c, 0xc4, 0x23, 0xc4, 0x26, 0x27, 0x49, 0x32, 0x78, 0x69, 0xf9, 0xc7,
-	0xfc, 0x63, 0x11, 0xea, 0xbb, 0x23, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xce, 0xc9,
-	0x82, 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x21, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d,
-	0x99, 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x04, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d,
-	0xb5, 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x3e, 0xd4, 0x4f, 0xb8, 0x93,
-	0xd4, 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x49, 0xba, 0x09, 0x0f, 0xa0, 0x39, 0x96, 0x2e,
-	0x53, 0x93, 0x64, 0x0c, 0xdd, 0x55, 0x96, 0xa4, 0xf6, 0x6e, 0x66, 0x3d, 0x2b, 0x17, 0xa0, 0x31,
-	0xce, 0xa0, 0xfa, 0x43, 0xe8, 0xcc, 0x90, 0xcc, 0xc9, 0x41, 0x1b, 0xd9, 0x1c, 0x54, 0x7f, 0x68,
-	0x48, 0x41, 0xd9, 0x99, 0xd9, 0xbc, 0xf4, 0x9f, 0x45, 0x68, 0x7c, 0x83, 0xd8, 0x73, 0x42, 0xcf,
-	0xa4, 0xbe, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa3, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e, 0xc8,
-	0x04, 0xa2, 0xd6, 0xb3, 0x42, 0x2f, 0x44, 0x62, 0x30, 0x5e, 0x03, 0xa0, 0x17, 0x76, 0xe8, 0xb8,
-	0x67, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xc5, 0x40, 0x22, 0x78, 0x28, 0xd0, 0x0b, 0x1b, 0x51,
-	0x4a, 0x68, 0xa4, 0x72, 0x55, 0x95, 0x5e, 0xec, 0x09, 0x58, 0xcd, 0xf5, 0x28, 0x09, 0x43, 0xe4,
-	0x89, 0x1c, 0x2d, 0xe6, 0x3e, 0x91, 0x08, 0x2e, 0x95, 0x69, 0xa9, 0xcb, 0x52, 0x2a, 0x4b, 0xa5,
-	0xb2, 0x54, 0x6a, 0x45, 0xce, 0x64, 0x59, 0xa9, 0x2c, 0x91, 0x5a, 0x95, 0x52, 0x59, 0x46, 0x2a,
-	0x4b, 0xa5, 0xd6, 0xf4, 0x5c, 0x25, 0xd5, 0xfc, 0x8f, 0x02, 0xac, 0x4d, 0x17, 0x7e, 0xaa, 0x36,
-	0xfd, 0x10, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0x66, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30,
-	0x7e, 0x04, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20,
-	0x03, 0x99, 0x1e, 0x18, 0xdf, 0x52, 0xcc, 0xd0, 0x90, 0x51, 0xe4, 0xf8, 0x2f, 0xa3, 0xba, 0x37,
-	0xa0, 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0x43, 0x37, 0x27, 0x45, 0xd9,
-	0xba, 0x02, 0xa5, 0x09, 0x0a, 0x04, 0xf7, 0xa6, 0xc5, 0x3f, 0x4d, 0x07, 0x3a, 0x16, 0x72, 0xbc,
-	0x97, 0xa7, 0x8d, 0x12, 0x51, 0x4a, 0x45, 0x6c, 0x80, 0x91, 0x15, 0xa1, 0x54, 0xd1, 0x5a, 0x17,
-	0x32, 0x5a, 0x3f, 0x85, 0xce, 0xee, 0x84, 0x44, 0x68, 0xc8, 0x3c, 0x1c, 0xbc, 0x8c, 0x76, 0xe4,
-	0x5f, 0xa0, 0xfb, 0x8c, 0x5d, 0x7e, 0xcb, 0x99, 0x45, 0xf8, 0x07, 0xf4, 0x92, 0xec, 0xa3, 0xe4,
-	0xb9, 0xb6, 0x8f, 0x92, 0xe7, 0xbc, 0xb9, 0x71, 0xc9, 0x24, 0xf6, 0x03, 0xb1, 0x15, 0x9a, 0x96,
-	0x82, 0xcc, 0x1d, 0x68, 0xc8, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0xcd, 0xdd, 0x83, 0xeb, 0x00,
-	0xa1, 0x43, 0x1d, 0x1f, 0x31, 0x44, 0x65, 0x0c, 0xd5, 0xac, 0x0c, 0xc6, 0xfc, 0xaf, 0x22, 0xac,
-	0xca, 0xfb, 0x86, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x7d, 0xa8, 0x8e, 0x49, 0xc4, 0x32, 0x0c, 0x13,
-	0x98, 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0xe6, 0x2e, 0x01, 0x4a, 0x57, 0x5f, 0x02, 0xcc,
-	0xb4, 0xf9, 0xe5, 0xd9, 0x36, 0x9f, 0xef, 0x36, 0x4d, 0x84, 0xe5, 0x1e, 0xaf, 0x59, 0x35, 0x85,
-	0x39, 0xf4, 0x8c, 0x7b, 0xd0, 0x1e, 0x71, 0x2d, 0xed, 0x31, 0x21, 0x67, 0x76, 0xe8, 0xb0, 0xb1,
-	0xd8, 0xea, 0x35, 0xab, 0x29, 0xd0, 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0x3e, 0x81, 0x96,
-	0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, 0x1d, 0x7e, 0x6a, 0x17, 0x65, 0xbd, 0x67, 0x35, 0xcf, 0x32,
-	0x50, 0x64, 0xde, 0x84, 0x1b, 0x4f, 0x50, 0xc4, 0x28, 0xb9, 0xcc, 0x3b, 0xc6, 0xfc, 0x6b, 0x80,
-	0xc3, 0x80, 0x21, 0x7a, 0xea, 0xb8, 0x28, 0x32, 0xde, 0xcb, 0x42, 0xaa, 0x38, 0x5a, 0xd9, 0x94,
-	0xd7, 0x3d, 0xc9, 0x80, 0x95, 0xa1, 0x31, 0x37, 0x61, 0xd9, 0x22, 0x31, 0x4f, 0x47, 0x6f, 0xea,
-	0x2f, 0x35, 0xaf, 0xa1, 0xe6, 0x09, 0xa4, 0xa5, 0xc6, 0xcc, 0x03, 0xdd, 0xc2, 0xa6, 0xec, 0xd4,
-	0x12, 0x6d, 0x42, 0x0d, 0x6b, 0x9c, 0xca, 0x2a, 0xb3, 0xa2, 0x53, 0x12, 0xf3, 0x33, 0xe8, 0x4a,
-	0x4e, 0x92, 0xb3, 0x66, 0xf3, 0x26, 0x2c, 0x53, 0xad, 0x46, 0x21, 0xbd, 0xe7, 0x51, 0x44, 0x6a,
-	0xcc, 0x3c, 0x84, 0xdb, 0x72, 0xf2, 0x5e, 0x38, 0x46, 0x3e, 0xa2, 0xce, 0xe4, 0x88, 0xc4, 0x01,
-	0x4b, 0xb8, 0x64, 0x23, 0xa0, 0x70, 0x65, 0x04, 0x70, 0xd7, 0x7e, 0x8d, 0x23, 0x96, 0xfa, 0x44,
-	0xbb, 0xb6, 0x0b, 0x1d, 0x3e, 0x90, 0x53, 0xcf, 0xfc, 0x02, 0x1a, 0xdb, 0xd6, 0xe0, 0x1b, 0x84,
-	0x47, 0xe3, 0x13, 0x9e, 0x88, 0x3f, 0xca, 0xc3, 0x4a, 0x98, 0xa1, 0x0c, 0xcf, 0x0c, 0x59, 0x39,
-	0x3a, 0xf3, 0x4b, 0x58, 0xdb, 0xf6, 0xbc, 0x2c, 0x4a, 0xab, 0xfe, 0x1e, 0xd4, 0x82, 0x0c, 0xbb,
-	0xcc, 0xf1, 0x97, 0xa3, 0x4e, 0x89, 0xcc, 0x07, 0x60, 0xec, 0x23, 0x76, 0x38, 0x78, 0xe6, 0x9c,
-	0x4c, 0x52, 0x47, 0xde, 0x84, 0x0a, 0x8e, 0x6c, 0x1c, 0x9e, 0x7f, 0x24, 0xb8, 0x54, 0xad, 0x65,
-	0x1c, 0x1d, 0x86, 0xe7, 0x1f, 0x99, 0xf7, 0xa1, 0x9b, 0x23, 0xbf, 0x22, 0x43, 0x6d, 0x83, 0x31,
-	0xfc, 0xf5, 0x9c, 0x13, 0x16, 0xc5, 0x0c, 0x8b, 0xfb, 0xd0, 0x1d, 0xfe, 0x4a, 0x69, 0xff, 0x04,
-	0xdd, 0xa7, 0xc1, 0x04, 0x07, 0x68, 0x77, 0x70, 0x7c, 0x84, 0x92, 0xf4, 0x6c, 0x40, 0x99, 0x97,
-	0xb1, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0x27, 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xff, 0x5a, 0x0e,
-	0x4e, 0x76, 0xc3, 0x38, 0xe2, 0xe7, 0x2d, 0xaf, 0xb7, 0x48, 0x30, 0xb9, 0x14, 0x49, 0xab, 0x6a,
-	0x55, 0xdc, 0x30, 0x7e, 0x1a, 0x4c, 0x2e, 0xcd, 0xbf, 0x12, 0x97, 0x12, 0x08, 0x79, 0x96, 0x13,
-	0x78, 0xc4, 0x7f, 0x82, 0xce, 0x33, 0x12, 0x66, 0xf4, 0xfe, 0xa9, 0x00, 0x8d, 0xed, 0x11, 0x0a,
-	0xd8, 0x13, 0xc4, 0x1c, 0x3c, 0x11, 0x4d, 0xee, 0x39, 0xa2, 0x11, 0x26, 0x81, 0xca, 0x40, 0x1a,
-	0x34, 0x5e, 0x87, 0x3a, 0x0e, 0x30, 0xb3, 0x3d, 0x07, 0xf9, 0x24, 0x10, 0x5c, 0xaa, 0x16, 0x70,
-	0xd4, 0x13, 0x81, 0x31, 0xde, 0x86, 0xb6, 0xbc, 0x9f, 0xb4, 0xc7, 0x4e, 0xe0, 0x4d, 0x78, 0xee,
-	0x2b, 0x89, 0x6c, 0xd5, 0x92, 0xe8, 0x03, 0x85, 0x35, 0xee, 0xc3, 0x8a, 0x8a, 0xcb, 0x94, 0xb2,
-	0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2,
-	0x87, 0xaa, 0x43, 0x6c, 0x6b, 0xfc, 0x50, 0xa2, 0xcd, 0x11, 0x74, 0xf7, 0xb9, 0x9d, 0xca, 0x92,
-	0x74, 0xa7, 0xb5, 0x7c, 0xe4, 0xdb, 0x27, 0x13, 0xe2, 0x9e, 0xd9, 0xfc, 0xbc, 0x50, 0x1e, 0xe6,
-	0x35, 0xe8, 0x0e, 0x47, 0x0e, 0xf1, 0x0f, 0xe2, 0x32, 0x84, 0x53, 0x8d, 0x09, 0x0b, 0x27, 0xf1,
-	0xc8, 0x0e, 0x29, 0x39, 0x41, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9,
-	0xbf, 0x05, 0x58, 0xcd, 0x4b, 0x52, 0xab, 0xbd, 0x05, 0xab, 0x79, 0x51, 0xaa, 0x22, 0x92, 0x15,
-	0x77, 0x27, 0x2b, 0x50, 0xd6, 0x46, 0x8f, 0xa0, 0x29, 0x6e, 0xb3, 0x6d, 0x4f, 0x72, 0xca, 0xd7,
-	0x81, 0xd9, 0x75, 0xb1, 0x1a, 0x4e, 0x76, 0x95, 0x3e, 0x81, 0x5b, 0xca, 0x7c, 0x7b, 0x56, 0x6d,
-	0x19, 0x10, 0x6b, 0x8a, 0xe0, 0x68, 0x4a, 0xfb, 0xaf, 0xa1, 0x97, 0xa2, 0x76, 0x2e, 0x05, 0x32,
-	0xdd, 0x94, 0xdd, 0x29, 0x63, 0xb7, 0x3d, 0x8f, 0x8a, 0xdd, 0x5e, 0xb6, 0xe6, 0x0d, 0x99, 0x8f,
-	0xe1, 0xe6, 0x10, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xe6, 0x4c, 0x32, 0x5b, 0x81, 0xd2, 0x10, 0xb9,
-	0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, 0x47, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d,
-	0x86, 0x50, 0xf9, 0x62, 0xb8, 0xcf, 0x4b, 0x30, 0x1e, 0xd4, 0xb2, 0x64, 0x53, 0xc7, 0x73, 0xd3,
-	0xaa, 0x08, 0xf8, 0xd0, 0x33, 0xbe, 0x84, 0xae, 0x1c, 0x72, 0xc7, 0x4e, 0x30, 0x42, 0x76, 0x48,
-	0x26, 0xd8, 0x95, 0xa1, 0xdf, 0x7a, 0xd8, 0x57, 0x69, 0x48, 0xf1, 0xd9, 0x15, 0x24, 0x03, 0x41,
-	0x61, 0x75, 0x46, 0xd3, 0x28, 0xf3, 0x77, 0x05, 0xa8, 0xa8, 0xfc, 0xc8, 0x4f, 0x79, 0x8f, 0xe2,
-	0x73, 0x44, 0x55, 0xb0, 0x2b, 0xc8, 0x78, 0x0b, 0x5a, 0xf2, 0xcb, 0x26, 0x21, 0xc3, 0x24, 0x39,
-	0x77, 0x9b, 0x12, 0xfb, 0x54, 0x22, 0xc5, 0x0d, 0xa8, 0xb8, 0x83, 0x54, 0xed, 0xbe, 0x82, 0x38,
-	0xfe, 0x34, 0xe2, 0x4a, 0x89, 0x73, 0xb6, 0x66, 0x29, 0x88, 0x6f, 0x2e, 0xcd, 0x6f, 0x49, 0xf0,
-	0xd3, 0x20, 0xdf, 0x5c, 0x3e, 0x4f, 0xed, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0xc1, 0x0a, 0x02, 0x35,
-	0xe0, 0x18, 0x63, 0x03, 0xaa, 0xa7, 0x91, 0x2d, 0xac, 0x11, 0x45, 0x74, 0x92, 0xea, 0x95, 0xd5,
-	0x56, 0xe5, 0x34, 0x12, 0x1f, 0xe6, 0xbf, 0x17, 0x60, 0x59, 0xbe, 0x17, 0x18, 0x2d, 0x28, 0x26,
-	0x85, 0x50, 0x11, 0x8b, 0xa2, 0x52, 0x68, 0x25, 0x8b, 0x1f, 0xf1, 0xcd, 0x73, 0xcc, 0xb9, 0x2f,
-	0x8f, 0x73, 0x65, 0xc4, 0xb9, 0x2f, 0xce, 0xf1, 0xb7, 0xa0, 0x95, 0xd6, 0x53, 0x62, 0x5c, 0x1a,
-	0xd3, 0x4c, 0xb0, 0x82, 0x6c, 0xa1, 0x4d, 0xe6, 0xdf, 0x03, 0xa4, 0xf7, 0xe6, 0x3c, 0x1c, 0xe2,
-	0x44, 0x19, 0xfe, 0xc9, 0x31, 0xa3, 0xa4, 0x12, 0xe3, 0x9f, 0xc6, 0x3d, 0x68, 0x39, 0x9e, 0x87,
-	0xf9, 0x74, 0x67, 0xb2, 0x8f, 0xbd, 0x24, 0x81, 0xe4, 0xb1, 0xe6, 0xff, 0x17, 0xa0, 0xbd, 0x4b,
-	0xc2, 0xcb, 0x2f, 0xf0, 0x04, 0x65, 0xb2, 0x9b, 0x50, 0x52, 0x15, 0x62, 0xfc, 0x9b, 0x37, 0x17,
-	0xa7, 0x78, 0x82, 0xe4, 0xb6, 0x97, 0x51, 0x57, 0xe5, 0x08, 0xb1, 0xe5, 0xf5, 0x60, 0x72, 0x4b,
-	0xda, 0x94, 0x83, 0x47, 0xc4, 0x13, 0x6d, 0x94, 0x87, 0xa9, 0x9d, 0xdc, 0x89, 0x36, 0xad, 0x8a,
-	0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x3b, 0xef, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59,
-	0x83, 0x65, 0x72, 0x7a, 0x1a, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c,
-	0x0a, 0x5e, 0x15, 0xe7, 0xda, 0xd3, 0xa7, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0xfa, 0x04, 0x7e, 0x00,
-	0x55, 0x8d, 0xfa, 0x35, 0xb7, 0xc9, 0xef, 0x40, 0x6b, 0xdb, 0xf3, 0x86, 0xcf, 0x9d, 0x50, 0xfb,
-	0xa3, 0x07, 0x95, 0xc1, 0xee, 0xe1, 0x40, 0xba, 0xa4, 0xc4, 0x0d, 0x50, 0x20, 0x3f, 0xf1, 0xf7,
-	0x11, 0x3b, 0x42, 0x8c, 0x62, 0x37, 0x39, 0xf1, 0xef, 0x42, 0x45, 0x61, 0xf8, 0x4c, 0x5f, 0x7e,
-	0xea, 0x23, 0x40, 0x81, 0xe6, 0xdf, 0x80, 0xf1, 0x77, 0xbc, 0x0c, 0x46, 0xb2, 0x07, 0x52, 0x92,
-	0xde, 0x81, 0xce, 0xb9, 0xc0, 0xda, 0xb2, 0x3e, 0xcc, 0x2c, 0x43, 0x5b, 0x0e, 0x88, 0xfc, 0x20,
-	0x64, 0x1f, 0x43, 0x57, 0x56, 0xed, 0x92, 0xcf, 0x35, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6,
-	0xc4, 0xf7, 0xc3, 0x1f, 0xbb, 0xea, 0x18, 0x53, 0x97, 0x44, 0xc6, 0x3e, 0xb4, 0xa7, 0x5e, 0xf4,
-	0x0c, 0x75, 0x6b, 0x38, 0xff, 0xa1, 0xaf, 0xbf, 0xb6, 0x29, 0x5f, 0x08, 0x37, 0xf5, 0x0b, 0xe1,
-	0xe6, 0x9e, 0x1f, 0xb2, 0x4b, 0x63, 0x0f, 0x5a, 0xf9, 0xb7, 0x2f, 0xe3, 0x55, 0x5d, 0x62, 0xcd,
-	0x79, 0x11, 0x5b, 0xc8, 0x66, 0x1f, 0xda, 0x53, 0xcf, 0x60, 0x5a, 0x9f, 0xf9, 0xaf, 0x63, 0x0b,
-	0x19, 0x3d, 0x86, 0x7a, 0xe6, 0xdd, 0xcb, 0xe8, 0x49, 0x26, 0xb3, 0x4f, 0x61, 0x0b, 0x19, 0xec,
-	0x42, 0x33, 0xf7, 0x14, 0x65, 0xf4, 0x95, 0x3d, 0x73, 0xde, 0xa7, 0x16, 0x32, 0xd9, 0x81, 0x7a,
-	0xe6, 0x45, 0x48, 0x6b, 0x31, 0xfb, 0xec, 0xd4, 0xbf, 0x35, 0x67, 0x44, 0x9d, 0x96, 0xfb, 0xd0,
-	0x9e, 0x7a, 0x26, 0xd2, 0x2e, 0x99, 0xff, 0x7a, 0xb4, 0x50, 0x99, 0x21, 0xdc, 0x98, 0x5b, 0x25,
-	0x1b, 0x66, 0x96, 0xdd, 0xfc, 0x12, 0x7a, 0x21, 0xd3, 0xaf, 0xc4, 0xba, 0x67, 0xae, 0x16, 0x32,
-	0xeb, 0x3e, 0xfb, 0xd2, 0xd4, 0xbf, 0x3d, 0x7f, 0x50, 0x99, 0xba, 0x07, 0xad, 0xfc, 0x23, 0x93,
-	0x66, 0x36, 0xf7, 0xe9, 0xe9, 0xea, 0x20, 0xca, 0xbd, 0x37, 0xa5, 0x41, 0x34, 0xef, 0x19, 0x6a,
-	0x21, 0xa3, 0x6d, 0x00, 0x75, 0x91, 0xe0, 0xe1, 0x20, 0x59, 0xbd, 0x99, 0x0b, 0x8c, 0x64, 0xf5,
-	0xe6, 0x5c, 0x3a, 0x3c, 0x06, 0x90, 0xfd, 0xbf, 0x47, 0x62, 0x66, 0xdc, 0xd4, 0x6a, 0x4c, 0x5d,
-	0x3a, 0xf4, 0x7b, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x39, 0x40, 0x7a, 0xaf,
-	0xa0, 0x19, 0xcc, 0xdc, 0x34, 0x5c, 0xe1, 0x83, 0x46, 0xf6, 0x16, 0xc1, 0x50, 0xb6, 0xce, 0xb9,
-	0x59, 0xb8, 0x82, 0x45, 0x7b, 0xaa, 0x4b, 0xcc, 0x47, 0xf0, 0x74, 0xf3, 0xd8, 0x9f, 0xe9, 0x14,
-	0x8d, 0x47, 0xd0, 0xc8, 0xb6, 0x87, 0x5a, 0x8b, 0x39, 0x2d, 0x63, 0x3f, 0xd7, 0x22, 0x1a, 0x8f,
-	0xa1, 0x95, 0xef, 0xe7, 0x74, 0x48, 0xcd, 0xed, 0xf2, 0xfa, 0xea, 0xe2, 0x33, 0x43, 0xfe, 0x01,
-	0x40, 0xda, 0xf7, 0x69, 0xf7, 0xcd, 0x74, 0x82, 0x53, 0x52, 0xf7, 0xa1, 0x3d, 0xd5, 0xcf, 0x69,
-	0x8b, 0xe7, 0xb7, 0x79, 0x57, 0x25, 0x90, 0x4c, 0x77, 0xa6, 0x43, 0x70, 0xb6, 0xbf, 0xd3, 0x21,
-	0x38, 0xaf, 0x95, 0xdb, 0x81, 0xfa, 0x70, 0x96, 0xc7, 0x70, 0x21, 0x8f, 0x79, 0x0d, 0xda, 0x87,
-	0x00, 0xe9, 0x59, 0xa8, 0xbd, 0x30, 0x73, 0x3a, 0xf6, 0x9b, 0xfa, 0x72, 0x5a, 0xd2, 0xed, 0x42,
-	0x33, 0x77, 0x7f, 0xa3, 0x73, 0xe8, 0xbc, 0x4b, 0x9d, 0xab, 0x4e, 0x96, 0xfc, 0x65, 0x87, 0x5e,
-	0xc1, 0xb9, 0x57, 0x20, 0x57, 0xc5, 0x71, 0xb6, 0x9d, 0xd4, 0x11, 0x34, 0xa7, 0xc5, 0xfc, 0x85,
-	0xbc, 0x92, 0x6d, 0x19, 0x33, 0x79, 0x65, 0x4e, 0x27, 0xb9, 0x90, 0xd1, 0x01, 0xb4, 0xf7, 0x75,
-	0x37, 0xa0, 0x3a, 0x15, 0xbd, 0x7e, 0xb3, 0x9d, 0x59, 0xbf, 0x3f, 0x6f, 0x48, 0xad, 0xcb, 0x57,
-	0xd0, 0x99, 0xe9, 0x52, 0x8c, 0xf5, 0xe4, 0x89, 0x60, 0x6e, 0xfb, 0xb2, 0x50, 0xad, 0x43, 0x58,
-	0x99, 0x6e, 0x52, 0x8c, 0xd7, 0x92, 0x98, 0x98, 0xd7, 0xbc, 0x2c, 0x64, 0xf5, 0x09, 0x54, 0x75,
-	0xe1, 0x69, 0xa8, 0xa7, 0x98, 0xa9, 0x42, 0x74, 0xe1, 0xd4, 0x47, 0x22, 0xe4, 0x93, 0xa2, 0x2e,
-	0x0d, 0xf9, 0xa9, 0xd2, 0xaf, 0xaf, 0x5e, 0x4e, 0x12, 0xca, 0x47, 0x50, 0x51, 0xb5, 0x9d, 0xb1,
-	0x9a, 0x6c, 0xb6, 0x4c, 0xa9, 0x77, 0x55, 0x84, 0xed, 0x23, 0x96, 0xa9, 0xd8, 0xb4, 0xd0, 0xd9,
-	0x22, 0x4e, 0xef, 0x91, 0xdc, 0x88, 0x5a, 0x8b, 0x6d, 0x68, 0x64, 0x6b, 0x36, 0xbd, 0xa4, 0x73,
-	0xea, 0xb8, 0x45, 0x9a, 0xec, 0x5c, 0xfc, 0xf4, 0xf3, 0xfa, 0x2b, 0xbf, 0xfd, 0x79, 0xfd, 0x95,
-	0x7f, 0x7b, 0xb1, 0x5e, 0xf8, 0xe9, 0xc5, 0x7a, 0xe1, 0x37, 0x2f, 0xd6, 0x0b, 0x7f, 0x78, 0xb1,
-	0x5e, 0xf8, 0x87, 0x7f, 0xfe, 0x33, 0xff, 0x13, 0x46, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x9d, 0x63,
-	0xca, 0x32, 0x43, 0xe1, 0xd9, 0x48, 0xfe, 0x31, 0x2c, 0xf3, 0x7f, 0x31, 0xae, 0xe5, 0xc9, 0xb2,
-	0x80, 0x3f, 0xf8, 0x53, 0x00, 0x00, 0x00, 0xff, 0xff, 0xeb, 0xb0, 0x39, 0x1d, 0x7c, 0x26, 0x00,
-	0x00,
+var fileDescriptor_56ede974c0020f77 = []byte{
+	// 3275 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x1c, 0x47,
+	0x73, 0xdf, 0x3e, 0xc8, 0xdd, 0xad, 0x7d, 0x71, 0x87, 0x14, 0xb5, 0x5a, 0xeb, 0xa3, 0xe5, 0x91,
+	0x25, 0x51, 0x76, 0x44, 0xda, 0xb2, 0x61, 0xf9, 0x01, 0x47, 0x21, 0x29, 0x9a, 0xa4, 0x6d, 0x5a,
+	0x9b, 0x59, 0xd1, 0x0e, 0x12, 0x24, 0x83, 0xe1, 0x4c, 0x73, 0xb7, 0xcd, 0x9d, 0xe9, 0x71, 0x4f,
+	0x0f, 0x45, 0x3a, 0x40, 0x90, 0x43, 0x90, 0xdc, 0x72, 0xcc, 0x2d, 0x7f, 0x20, 0xc8, 0x3f, 0xc8,
+	0x35, 0x07, 0x23, 0xa7, 0x1c, 0x73, 0x49, 0x10, 0xeb, 0x27, 0xe4, 0x17, 0x04, 0xfd, 0x9a, 0xc7,
+	0xbe, 0x64, 0x10, 0x02, 0xbe, 0xcb, 0x62, 0xba, 0xba, 0xba, 0x5e, 0x5d, 0x5d, 0x5d, 0x55, 0xbd,
+	0x50, 0x77, 0x86, 0x28, 0x60, 0x5b, 0x21, 0x25, 0x8c, 0x18, 0xe5, 0x21, 0x0d, 0xdd, 0x5e, 0x8d,
+	0xb8, 0x58, 0x02, 0x7a, 0x35, 0x37, 0xd2, 0x9f, 0x75, 0x76, 0x15, 0xa2, 0x48, 0x0d, 0xde, 0x1a,
+	0x12, 0x32, 0x1c, 0xa3, 0x6d, 0x31, 0x3a, 0x8d, 0xcf, 0xb6, 0x91, 0x1f, 0xb2, 0x2b, 0x39, 0x69,
+	0xfe, 0x73, 0x11, 0xd6, 0xf7, 0x28, 0x72, 0x18, 0xda, 0x23, 0x01, 0x73, 0x70, 0x80, 0xa8, 0x85,
+	0x7e, 0x8a, 0x51, 0xc4, 0x8c, 0x77, 0xa0, 0xe1, 0x6a, 0x98, 0x8d, 0xbd, 0x6e, 0xe1, 0x4e, 0x61,
+	0xb3, 0x66, 0xd5, 0x13, 0xd8, 0x91, 0x67, 0xdc, 0x84, 0x0a, 0xba, 0x44, 0x2e, 0x9f, 0x2d, 0x8a,
+	0xd9, 0x65, 0x3e, 0x3c, 0xf2, 0x8c, 0x0f, 0xa1, 0x1e, 0x31, 0x8a, 0x83, 0xa1, 0x1d, 0x47, 0x88,
+	0x76, 0x4b, 0x77, 0x0a, 0x9b, 0xf5, 0xc7, 0x2b, 0x5b, 0x5c, 0xe4, 0xad, 0x81, 0x98, 0x38, 0x89,
+	0x10, 0xb5, 0x20, 0x4a, 0xbe, 0x8d, 0xfb, 0x50, 0xf1, 0xd0, 0x05, 0x76, 0x51, 0xd4, 0x2d, 0xdf,
+	0x29, 0x6d, 0xd6, 0x1f, 0x37, 0x24, 0xfa, 0x33, 0x01, 0xb4, 0xf4, 0xa4, 0xf1, 0x10, 0xaa, 0x11,
+	0x23, 0xd4, 0x19, 0xa2, 0xa8, 0xbb, 0x24, 0x10, 0x9b, 0x9a, 0xae, 0x80, 0x5a, 0xc9, 0xb4, 0x71,
+	0x1b, 0x4a, 0xcf, 0xf7, 0x8e, 0xba, 0xcb, 0x82, 0x3b, 0x28, 0xac, 0x10, 0xb9, 0x16, 0x07, 0x1b,
+	0x77, 0xa1, 0x19, 0x39, 0x81, 0x77, 0x4a, 0x2e, 0xed, 0x10, 0x7b, 0x41, 0xd4, 0xad, 0xdc, 0x29,
+	0x6c, 0x56, 0xad, 0x86, 0x02, 0xf6, 0x39, 0xcc, 0xfc, 0x1c, 0x6e, 0x0c, 0x98, 0x43, 0xd9, 0x35,
+	0xac, 0x63, 0x9e, 0xc0, 0xba, 0x85, 0x7c, 0x72, 0x71, 0x2d, 0xd3, 0x76, 0xa1, 0xc2, 0xb0, 0x8f,
+	0x48, 0xcc, 0x84, 0x69, 0x9b, 0x96, 0x1e, 0x9a, 0xff, 0x5a, 0x00, 0x63, 0xff, 0x12, 0xb9, 0x7d,
+	0x4a, 0x5c, 0x14, 0x45, 0x7f, 0xa0, 0xed, 0x7a, 0x00, 0x95, 0x50, 0x0a, 0xd0, 0x2d, 0x0b, 0x74,
+	0xb5, 0x0b, 0x5a, 0x2a, 0x3d, 0x6b, 0xfe, 0x08, 0x6b, 0x03, 0x3c, 0x0c, 0x9c, 0xf1, 0x1b, 0x94,
+	0x77, 0x1d, 0x96, 0x23, 0x41, 0x53, 0x88, 0xda, 0xb4, 0xd4, 0xc8, 0xec, 0x83, 0xf1, 0x83, 0x83,
+	0xd9, 0x9b, 0xe3, 0x64, 0x3e, 0x82, 0xd5, 0x1c, 0xc5, 0x28, 0x24, 0x41, 0x84, 0x84, 0x00, 0xcc,
+	0x61, 0x71, 0x24, 0x88, 0x2d, 0x59, 0x6a, 0x64, 0x12, 0x58, 0x3f, 0x09, 0xbd, 0x6b, 0x9e, 0xa6,
+	0xc7, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x67, 0xa0, 0x28, 0x8c, 0xba, 0x26, 0x8d, 0xfa, 0x2d,
+	0x0e, 0xe2, 0x4b, 0x4b, 0xcf, 0x59, 0x29, 0x9a, 0xf2, 0x4f, 0x16, 0x5d, 0xc7, 0x3f, 0x3f, 0x87,
+	0x1b, 0x7d, 0x27, 0x8e, 0xae, 0x23, 0xab, 0xf9, 0x05, 0xf7, 0xed, 0x28, 0xf6, 0xaf, 0xb5, 0xf8,
+	0x5f, 0x0a, 0x50, 0xdd, 0x0b, 0xe3, 0x93, 0xc8, 0x19, 0x22, 0xe3, 0x6d, 0xa8, 0x33, 0xc2, 0x9c,
+	0xb1, 0x1d, 0xf3, 0xa1, 0x40, 0x2f, 0x5b, 0x20, 0x40, 0x12, 0xe1, 0x1d, 0x68, 0x84, 0x88, 0xba,
+	0x61, 0xac, 0x30, 0x8a, 0x77, 0x4a, 0x9b, 0x65, 0xab, 0x2e, 0x61, 0x12, 0x65, 0x0b, 0x56, 0xc5,
+	0x9c, 0x8d, 0x03, 0xfb, 0x1c, 0xd1, 0x00, 0x8d, 0x7d, 0xe2, 0x21, 0xe1, 0x1c, 0x65, 0xab, 0x23,
+	0xa6, 0x8e, 0x82, 0x6f, 0x92, 0x09, 0xe3, 0x3d, 0xe8, 0x24, 0xf8, 0xdc, 0xe3, 0x05, 0x76, 0x59,
+	0x60, 0xb7, 0x15, 0xf6, 0x89, 0x02, 0x9b, 0x7f, 0x03, 0xad, 0x17, 0x23, 0x4a, 0x18, 0x1b, 0xe3,
+	0x60, 0xf8, 0xcc, 0x61, 0x0e, 0x3f, 0x9a, 0x21, 0xa2, 0x98, 0x78, 0x91, 0x92, 0x56, 0x0f, 0x8d,
+	0xf7, 0xa1, 0xc3, 0x24, 0x2e, 0xf2, 0x6c, 0x8d, 0x53, 0x14, 0x38, 0x2b, 0xc9, 0x44, 0x5f, 0x21,
+	0xdf, 0x83, 0x56, 0x8a, 0xcc, 0x0f, 0xb7, 0x92, 0xb7, 0x99, 0x40, 0x5f, 0x60, 0x1f, 0x99, 0x17,
+	0xc2, 0x56, 0x62, 0x93, 0x8d, 0xf7, 0xa1, 0x96, 0xda, 0xa1, 0x20, 0x3c, 0xa4, 0x25, 0x3d, 0x44,
+	0x9b, 0xd3, 0xaa, 0x26, 0x46, 0xf9, 0x12, 0xda, 0x2c, 0x11, 0xdc, 0xf6, 0x1c, 0xe6, 0xe4, 0x9d,
+	0x2a, 0xaf, 0x95, 0xd5, 0x62, 0xb9, 0xb1, 0xf9, 0x05, 0xd4, 0xfa, 0xd8, 0x8b, 0x24, 0xe3, 0x2e,
+	0x54, 0xdc, 0x98, 0x52, 0x14, 0x30, 0xad, 0xb2, 0x1a, 0x1a, 0x6b, 0xb0, 0x34, 0xc6, 0x3e, 0x66,
+	0x4a, 0x4d, 0x39, 0x30, 0x09, 0xc0, 0x31, 0xf2, 0x09, 0xbd, 0x12, 0x06, 0x5b, 0x83, 0xa5, 0xec,
+	0xe6, 0xca, 0x81, 0xf1, 0x16, 0xd4, 0x7c, 0xe7, 0x32, 0xd9, 0x54, 0x3e, 0x53, 0xf5, 0x9d, 0x4b,
+	0x29, 0x7c, 0x17, 0x2a, 0x67, 0x0e, 0x1e, 0xbb, 0x01, 0x53, 0x56, 0xd1, 0xc3, 0x94, 0x61, 0x39,
+	0xcb, 0xf0, 0xdf, 0x8b, 0x50, 0x97, 0x1c, 0xa5, 0xc0, 0x6b, 0xb0, 0xe4, 0x3a, 0xee, 0x28, 0x61,
+	0x29, 0x06, 0xc6, 0x7d, 0x2d, 0x48, 0x31, 0x1b, 0xe1, 0x52, 0x49, 0xb5, 0x68, 0xdb, 0x00, 0xd1,
+	0x4b, 0x27, 0x54, 0xb2, 0x95, 0xe6, 0x20, 0xd7, 0x38, 0x8e, 0x14, 0xf7, 0x23, 0x68, 0x48, 0xbf,
+	0x53, 0x4b, 0xca, 0x73, 0x96, 0xd4, 0x25, 0x96, 0x5c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x11,
+	0x46, 0xd4, 0xa1, 0xee, 0xe8, 0xaa, 0xbb, 0x24, 0x2f, 0xa0, 0x38, 0x42, 0x87, 0x1a, 0x66, 0x3c,
+	0x86, 0x25, 0x1e, 0x5b, 0xa2, 0xee, 0xb2, 0xb8, 0xeb, 0x6e, 0x67, 0x49, 0x0a, 0x55, 0xb7, 0xc4,
+	0xef, 0x7e, 0xc0, 0xe8, 0x95, 0x25, 0x51, 0x7b, 0x9f, 0x02, 0xa4, 0x40, 0x63, 0x05, 0x4a, 0xe7,
+	0xe8, 0x4a, 0x9d, 0x43, 0xfe, 0xc9, 0x8d, 0x73, 0xe1, 0x8c, 0x63, 0x6d, 0x75, 0x39, 0xf8, 0xbc,
+	0xf8, 0x69, 0xc1, 0x74, 0xa1, 0xbd, 0x3b, 0x3e, 0xc7, 0x24, 0xb3, 0x7c, 0x0d, 0x96, 0x7c, 0xe7,
+	0x47, 0x42, 0xb5, 0x25, 0xc5, 0x40, 0x40, 0x71, 0x40, 0xa8, 0x26, 0x21, 0x06, 0x46, 0x0b, 0x8a,
+	0x24, 0x14, 0xf6, 0xaa, 0x59, 0x45, 0x12, 0xa6, 0x8c, 0xca, 0x19, 0x46, 0xe6, 0xff, 0x94, 0x01,
+	0x52, 0x2e, 0x86, 0x05, 0x3d, 0x4c, 0xec, 0x08, 0x51, 0x7e, 0xbf, 0xdb, 0xa7, 0x57, 0x0c, 0x45,
+	0x36, 0x45, 0x6e, 0x4c, 0x23, 0x7c, 0xc1, 0xf7, 0x8f, 0xab, 0x7d, 0x43, 0xaa, 0x3d, 0x21, 0x9b,
+	0x75, 0x13, 0x93, 0x81, 0x5c, 0xb7, 0xcb, 0x97, 0x59, 0x7a, 0x95, 0x71, 0x04, 0x37, 0x52, 0x9a,
+	0x5e, 0x86, 0x5c, 0x71, 0x11, 0xb9, 0xd5, 0x84, 0x9c, 0x97, 0x92, 0xda, 0x87, 0x55, 0x4c, 0xec,
+	0x9f, 0x62, 0x14, 0xe7, 0x08, 0x95, 0x16, 0x11, 0xea, 0x60, 0xf2, 0xa7, 0x62, 0x41, 0x4a, 0xa6,
+	0x0f, 0xb7, 0x32, 0x5a, 0xf2, 0xe3, 0x9e, 0x21, 0x56, 0x5e, 0x44, 0x6c, 0x3d, 0x91, 0x8a, 0xc7,
+	0x83, 0x94, 0xe2, 0xd7, 0xb0, 0x8e, 0x89, 0xfd, 0xd2, 0xc1, 0x6c, 0x92, 0xdc, 0xd2, 0x6b, 0x94,
+	0xe4, 0x37, 0x5a, 0x9e, 0x96, 0x54, 0xd2, 0x47, 0x74, 0x98, 0x53, 0x72, 0xf9, 0x35, 0x4a, 0x1e,
+	0x8b, 0x05, 0x29, 0x99, 0x1d, 0xe8, 0x60, 0x32, 0x29, 0x4d, 0x65, 0x11, 0x91, 0x36, 0x26, 0x79,
+	0x49, 0x76, 0xa1, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x75, 0x82, 0xea, 0x22, 0x12, 0x2b, 0x0a, 0x3f,
+	0xa1, 0x61, 0xfe, 0x05, 0x34, 0x0e, 0xe3, 0x21, 0x62, 0xe3, 0xd3, 0x24, 0x18, 0xbc, 0xb1, 0xf8,
+	0x63, 0xfe, 0x5f, 0x11, 0xea, 0x7b, 0x43, 0x4a, 0xe2, 0x30, 0x17, 0x93, 0xe5, 0x21, 0x9d, 0x8c,
+	0xc9, 0x02, 0x45, 0xc4, 0x64, 0x89, 0xfc, 0x31, 0x34, 0x7c, 0x71, 0x74, 0x15, 0xbe, 0x8c, 0x43,
+	0x9d, 0xa9, 0x43, 0x6d, 0xd5, 0xfd, 0x4c, 0x30, 0xdb, 0x02, 0x08, 0xb1, 0x17, 0xa9, 0x35, 0x32,
+	0x1c, 0xb5, 0x55, 0xba, 0xa5, 0x43, 0xb4, 0x55, 0x0b, 0x93, 0x68, 0xfd, 0x21, 0xd4, 0x4f, 0xb9,
+	0x91, 0xd4, 0x82, 0x5c, 0x30, 0x4a, 0xad, 0x67, 0xc1, 0x69, 0x7a, 0x08, 0x0f, 0xa1, 0x39, 0x92,
+	0x26, 0x53, 0x8b, 0xa4, 0x0f, 0xdd, 0x55, 0x9a, 0xa4, 0xfa, 0x6e, 0x65, 0x2d, 0x2b, 0x37, 0xa0,
+	0x31, 0xca, 0x80, 0x7a, 0x03, 0xe8, 0x4c, 0xa1, 0xcc, 0x88, 0x41, 0x9b, 0xd9, 0x18, 0x54, 0x7f,
+	0x6c, 0x48, 0x46, 0xd9, 0x95, 0xd9, 0xb8, 0xf4, 0x8f, 0x45, 0x68, 0x7c, 0x87, 0xd8, 0x4b, 0x42,
+	0xcf, 0xa5, 0xbc, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa2, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e,
+	0xca, 0x00, 0xa2, 0xf6, 0xb3, 0x42, 0x2f, 0x45, 0x60, 0x30, 0x7e, 0x0f, 0x40, 0x2f, 0xed, 0xd0,
+	0x71, 0xcf, 0x91, 0xb2, 0x60, 0xd9, 0xaa, 0xd1, 0xcb, 0xbe, 0x04, 0x70, 0x57, 0xa0, 0x97, 0x36,
+	0xa2, 0x94, 0xd0, 0x48, 0xc5, 0xaa, 0x2a, 0xbd, 0xdc, 0x17, 0x63, 0xb5, 0xd6, 0xa3, 0x24, 0x0c,
+	0x91, 0x27, 0x62, 0xb4, 0x58, 0xfb, 0x4c, 0x02, 0x38, 0x57, 0xa6, 0xb9, 0x2e, 0x4b, 0xae, 0x2c,
+	0xe5, 0xca, 0x52, 0xae, 0x15, 0xb9, 0x92, 0x65, 0xb9, 0xb2, 0x84, 0x6b, 0x55, 0x72, 0x65, 0x19,
+	0xae, 0x2c, 0xe5, 0x5a, 0xd3, 0x6b, 0x15, 0x57, 0xf3, 0x1f, 0x0a, 0xb0, 0x3e, 0x99, 0xf8, 0xa9,
+	0xdc, 0xf4, 0x63, 0x68, 0xb8, 0x62, 0xbf, 0x72, 0x3e, 0xd9, 0x99, 0xda, 0x49, 0xab, 0xee, 0x66,
+	0xdc, 0xf8, 0x09, 0x34, 0x03, 0x69, 0xe0, 0xc4, 0x35, 0x4b, 0xe9, 0xbe, 0x64, 0x6d, 0x6f, 0x35,
+	0x82, 0xcc, 0xc8, 0xf4, 0xc0, 0xf8, 0x81, 0x62, 0x86, 0x06, 0x8c, 0x22, 0xc7, 0x7f, 0x13, 0xd9,
+	0xbd, 0x01, 0x65, 0x91, 0xad, 0xf0, 0x6d, 0x6a, 0x58, 0xe2, 0xdb, 0x7c, 0x00, 0xab, 0x39, 0x2e,
+	0x4a, 0xd7, 0x15, 0x28, 0x8d, 0x51, 0x20, 0xa8, 0x37, 0x2d, 0xfe, 0x69, 0x3a, 0xd0, 0xb1, 0x90,
+	0xe3, 0xbd, 0x39, 0x69, 0x14, 0x8b, 0x52, 0xca, 0x62, 0x13, 0x8c, 0x2c, 0x0b, 0x25, 0x8a, 0x96,
+	0xba, 0x90, 0x91, 0xfa, 0x39, 0x74, 0xf6, 0xc6, 0x24, 0x42, 0x03, 0xe6, 0xe1, 0xe0, 0x4d, 0x94,
+	0x23, 0x7f, 0x0d, 0xab, 0x2f, 0xd8, 0xd5, 0x0f, 0x9c, 0x58, 0x84, 0x7f, 0x46, 0x6f, 0x48, 0x3f,
+	0x4a, 0x5e, 0x6a, 0xfd, 0x28, 0x79, 0xc9, 0x8b, 0x1b, 0x97, 0x8c, 0x63, 0x3f, 0x10, 0x47, 0xa1,
+	0x69, 0xa9, 0x91, 0xb9, 0x0b, 0x0d, 0x99, 0x43, 0x1f, 0x13, 0x2f, 0x1e, 0xa3, 0x99, 0x67, 0x70,
+	0x03, 0x20, 0x74, 0xa8, 0xe3, 0x23, 0x86, 0xa8, 0xf4, 0xa1, 0x9a, 0x95, 0x81, 0x98, 0xff, 0x54,
+	0x84, 0x35, 0xd9, 0x6f, 0x18, 0xc8, 0x32, 0x5b, 0xab, 0xd0, 0x83, 0xea, 0x88, 0x44, 0x2c, 0x43,
+	0x30, 0x19, 0x73, 0x11, 0x79, 0x7d, 0x2e, 0xa9, 0xf1, 0xcf, 0x5c, 0x13, 0xa0, 0xb4, 0xb8, 0x09,
+	0x30, 0x55, 0xe6, 0x97, 0xa7, 0xcb, 0x7c, 0x7e, 0xda, 0x34, 0x12, 0x96, 0x67, 0xbc, 0x66, 0xd5,
+	0x14, 0xe4, 0xc8, 0x33, 0xee, 0x43, 0x7b, 0xc8, 0xa5, 0xb4, 0x47, 0x84, 0x9c, 0xdb, 0xa1, 0xc3,
+	0x46, 0xe2, 0xa8, 0xd7, 0xac, 0xa6, 0x00, 0x1f, 0x12, 0x72, 0xde, 0x77, 0xd8, 0xc8, 0xf8, 0x0c,
+	0x5a, 0x2a, 0x0d, 0xf4, 0x85, 0x89, 0x22, 0x75, 0xf9, 0xa9, 0x53, 0x94, 0xb5, 0x9e, 0xd5, 0x3c,
+	0xcf, 0x8c, 0x22, 0xf3, 0x26, 0xdc, 0x78, 0x86, 0x22, 0x46, 0xc9, 0x55, 0xde, 0x30, 0xe6, 0x03,
+	0xb8, 0x27, 0xbb, 0x08, 0x03, 0xe6, 0x8c, 0xd1, 0xf7, 0x98, 0x32, 0x4c, 0xce, 0xa2, 0xc1, 0xc8,
+	0xa1, 0xe8, 0x98, 0xc4, 0x01, 0xd3, 0x65, 0xae, 0xf9, 0xc7, 0x00, 0x47, 0x01, 0x43, 0xf4, 0xcc,
+	0x71, 0x51, 0x64, 0x7c, 0x90, 0x1d, 0xa9, 0x2c, 0x6a, 0x65, 0x4b, 0xf6, 0x85, 0x92, 0x09, 0x2b,
+	0x83, 0x63, 0x6e, 0xc1, 0xb2, 0x45, 0x62, 0x1e, 0xb7, 0xde, 0xd5, 0x5f, 0x6a, 0x5d, 0x43, 0xad,
+	0x13, 0x40, 0x4b, 0xcd, 0x99, 0x87, 0xba, 0xd6, 0x4d, 0xc9, 0xa9, 0xbd, 0xdc, 0x82, 0x1a, 0xd6,
+	0x30, 0x15, 0x7e, 0xa6, 0x59, 0xa7, 0x28, 0xe6, 0x17, 0xb0, 0x2a, 0x29, 0x49, 0xca, 0x9a, 0xcc,
+	0xbb, 0xb0, 0x4c, 0xb5, 0x18, 0x85, 0xb4, 0x21, 0xa4, 0x90, 0xd4, 0x9c, 0x79, 0x04, 0xb7, 0xe5,
+	0xe2, 0xfd, 0x70, 0x84, 0x7c, 0x44, 0x9d, 0x71, 0xce, 0x2c, 0x39, 0x57, 0x29, 0x2c, 0x74, 0x15,
+	0xbe, 0x07, 0xdf, 0xe2, 0x88, 0xa5, 0x36, 0xd1, 0xa6, 0x5d, 0x85, 0x0e, 0x9f, 0xc8, 0x89, 0x67,
+	0x7e, 0x05, 0x8d, 0x1d, 0xab, 0xff, 0x1d, 0xc2, 0xc3, 0xd1, 0x29, 0x8f, 0xd8, 0x9f, 0xe4, 0xc7,
+	0x8a, 0x99, 0xa1, 0x14, 0xcf, 0x4c, 0x59, 0x39, 0x3c, 0xf3, 0x6b, 0x58, 0xdf, 0xf1, 0xbc, 0x2c,
+	0x48, 0x8b, 0xfe, 0x01, 0xd4, 0x82, 0x0c, 0xb9, 0xcc, 0x3d, 0x99, 0xc3, 0x4e, 0x91, 0xcc, 0x47,
+	0x60, 0x1c, 0x20, 0x76, 0xd4, 0x7f, 0xe1, 0x9c, 0x8e, 0x53, 0x43, 0xde, 0x84, 0x0a, 0x8e, 0x6c,
+	0x1c, 0x5e, 0x7c, 0x22, 0xa8, 0x54, 0xad, 0x65, 0x1c, 0x1d, 0x85, 0x17, 0x9f, 0x98, 0x0f, 0x61,
+	0x35, 0x87, 0xbe, 0x20, 0x94, 0xed, 0x80, 0x31, 0xf8, 0xed, 0x94, 0x13, 0x12, 0xc5, 0x0c, 0x89,
+	0x87, 0xb0, 0x3a, 0xf8, 0x8d, 0xdc, 0xfe, 0x12, 0x56, 0x9f, 0x07, 0x63, 0x1c, 0xa0, 0xbd, 0xfe,
+	0xc9, 0x31, 0x4a, 0xe2, 0xb8, 0x01, 0x65, 0x9e, 0xef, 0x2a, 0x5e, 0xe2, 0x9b, 0x8b, 0x10, 0x9c,
+	0xda, 0x6e, 0x18, 0x47, 0xaa, 0x51, 0xb6, 0x1c, 0x9c, 0xee, 0x85, 0x71, 0xc4, 0x2f, 0x66, 0x9e,
+	0x98, 0x91, 0x60, 0x7c, 0x25, 0xa2, 0x5b, 0xd5, 0xaa, 0xb8, 0x61, 0xfc, 0x3c, 0x18, 0x5f, 0x99,
+	0x7f, 0x24, 0xba, 0x17, 0x08, 0x79, 0x96, 0x13, 0x78, 0xc4, 0x7f, 0x86, 0x2e, 0x32, 0x1c, 0xa6,
+	0xe4, 0xfe, 0xa5, 0x00, 0x8d, 0x9d, 0x21, 0x0a, 0xd8, 0x33, 0xc4, 0x1c, 0x3c, 0x16, 0xd5, 0xf0,
+	0x05, 0xa2, 0x11, 0x26, 0x81, 0x0a, 0x55, 0x7a, 0x68, 0xbc, 0x0d, 0x75, 0x1c, 0x60, 0x66, 0x7b,
+	0x0e, 0xf2, 0x49, 0x20, 0xa8, 0x54, 0x2d, 0xe0, 0xa0, 0x67, 0x02, 0x62, 0x3c, 0x80, 0xb6, 0x6c,
+	0x64, 0xda, 0x23, 0x27, 0xf0, 0xc6, 0x3c, 0x48, 0x96, 0x44, 0x58, 0x6b, 0x49, 0xf0, 0xa1, 0x82,
+	0x1a, 0x0f, 0x61, 0x45, 0xf9, 0x65, 0x8a, 0x59, 0x16, 0x98, 0x6d, 0x05, 0xcf, 0xa1, 0xc6, 0x61,
+	0x48, 0x28, 0x8b, 0xec, 0x08, 0xb9, 0x2e, 0xf1, 0x43, 0x55, 0x4a, 0xb6, 0x35, 0x7c, 0x20, 0xc1,
+	0xe6, 0x10, 0x56, 0x0f, 0xb8, 0x9e, 0x4a, 0x93, 0xf4, 0xa4, 0xb5, 0x7c, 0xe4, 0xdb, 0xa7, 0x63,
+	0xe2, 0x9e, 0xdb, 0xfc, 0x62, 0x51, 0x16, 0xe6, 0xc9, 0xea, 0x2e, 0x07, 0x0e, 0xf0, 0xcf, 0xa2,
+	0x6b, 0xc2, 0xb1, 0x46, 0x84, 0x85, 0xe3, 0x78, 0x68, 0x87, 0x94, 0x9c, 0x22, 0xa5, 0x62, 0xdb,
+	0x47, 0xfe, 0xa1, 0x84, 0xf7, 0x39, 0xd8, 0xfc, 0xb7, 0x02, 0xac, 0xe5, 0x39, 0xa9, 0xdd, 0xde,
+	0x86, 0xb5, 0x3c, 0x2b, 0x95, 0x3a, 0xc9, 0xd4, 0xbc, 0x93, 0x65, 0x28, 0x93, 0xa8, 0x27, 0xd0,
+	0x14, 0x6d, 0x6f, 0xdb, 0x93, 0x94, 0xf2, 0x09, 0x63, 0x76, 0x5f, 0xac, 0x86, 0x93, 0xdd, 0xa5,
+	0xcf, 0xe0, 0x96, 0x52, 0xdf, 0x9e, 0x16, 0x5b, 0x3a, 0xc4, 0xba, 0x42, 0x38, 0x9e, 0x90, 0xfe,
+	0x5b, 0xe8, 0xa6, 0xa0, 0xdd, 0x2b, 0x01, 0x4c, 0x0f, 0xe5, 0xea, 0x84, 0xb2, 0x3b, 0x9e, 0x47,
+	0xc5, 0x69, 0x2f, 0x5b, 0xb3, 0xa6, 0xcc, 0xa7, 0x70, 0x73, 0x80, 0x98, 0xb4, 0x86, 0xc3, 0x54,
+	0x15, 0x27, 0x89, 0xad, 0x40, 0x69, 0x80, 0x5c, 0xa1, 0x7c, 0xc9, 0xe2, 0x9f, 0xdc, 0x01, 0x4f,
+	0x22, 0xe4, 0x0a, 0x2d, 0x4b, 0x96, 0xf8, 0x36, 0x43, 0xa8, 0x7c, 0x35, 0x38, 0xe0, 0xb9, 0x1a,
+	0x77, 0x6a, 0x99, 0xdb, 0xa9, 0x7b, 0xbc, 0x69, 0x55, 0xc4, 0xf8, 0xc8, 0x33, 0xbe, 0x86, 0x55,
+	0x39, 0xe5, 0x8e, 0x9c, 0x60, 0x88, 0xec, 0x90, 0x8c, 0xb1, 0x2b, 0x5d, 0xbf, 0xf5, 0xb8, 0xa7,
+	0xc2, 0x90, 0xa2, 0xb3, 0x27, 0x50, 0xfa, 0x02, 0xc3, 0xea, 0x0c, 0x27, 0x41, 0xe6, 0x7f, 0x17,
+	0xa0, 0xa2, 0xe2, 0x23, 0x4f, 0x07, 0x3c, 0x8a, 0x2f, 0x10, 0x55, 0xce, 0xae, 0x46, 0xc6, 0x3d,
+	0x68, 0xc9, 0x2f, 0x9b, 0x84, 0x0c, 0x93, 0xe4, 0x82, 0x6e, 0x4a, 0xe8, 0x73, 0x09, 0x14, 0xad,
+	0x52, 0xd1, 0xac, 0x54, 0x7d, 0x01, 0x35, 0xe2, 0xf0, 0xb3, 0x88, 0x0b, 0x25, 0x2e, 0xe4, 0x9a,
+	0xa5, 0x46, 0xfc, 0x70, 0x69, 0x7a, 0x4b, 0x82, 0x9e, 0x1e, 0xf2, 0xc3, 0xe5, 0xf3, 0xd0, 0x6e,
+	0x87, 0x04, 0x07, 0x4c, 0xdd, 0xc0, 0x20, 0x40, 0x7d, 0x0e, 0x31, 0x36, 0xa1, 0x7a, 0x16, 0xd9,
+	0x42, 0x1b, 0x91, 0x6d, 0x27, 0xa1, 0x5e, 0x69, 0x6d, 0x55, 0xce, 0x22, 0xf1, 0x61, 0xfe, 0x7d,
+	0x01, 0x96, 0xe5, 0xc3, 0x82, 0xd1, 0x82, 0x62, 0x92, 0x31, 0x15, 0xb1, 0xc8, 0x3e, 0x85, 0x54,
+	0x32, 0x4b, 0x12, 0xdf, 0x3c, 0xc6, 0x5c, 0xf8, 0xf2, 0xde, 0x57, 0x4a, 0x5c, 0xf8, 0xe2, 0xc2,
+	0xbf, 0x07, 0xad, 0x34, 0xf1, 0x12, 0xf3, 0x52, 0x99, 0x66, 0x02, 0x15, 0x68, 0x73, 0x75, 0x32,
+	0xff, 0x0c, 0x20, 0x6d, 0xb0, 0x73, 0x77, 0x88, 0x13, 0x61, 0xf8, 0x27, 0x87, 0x0c, 0x93, 0x94,
+	0x8d, 0x7f, 0x1a, 0xf7, 0xa1, 0xe5, 0x78, 0x1e, 0xe6, 0xcb, 0x9d, 0xf1, 0x01, 0xf6, 0x92, 0x00,
+	0x92, 0x87, 0x9a, 0xff, 0x51, 0x80, 0xf6, 0x1e, 0x09, 0xaf, 0xbe, 0xc2, 0x63, 0x94, 0x89, 0x6e,
+	0x42, 0x48, 0x95, 0xb1, 0xf1, 0x6f, 0x5e, 0x85, 0x9c, 0xe1, 0x31, 0x92, 0xc7, 0x5e, 0x7a, 0x5d,
+	0x95, 0x03, 0xc4, 0x91, 0xd7, 0x93, 0x49, 0x3b, 0xb5, 0x29, 0x27, 0x8f, 0x89, 0x27, 0xea, 0x2d,
+	0x0f, 0x53, 0x3b, 0x69, 0x9e, 0x36, 0xad, 0x8a, 0x87, 0xa9, 0x98, 0x52, 0x8a, 0x2c, 0x89, 0xe6,
+	0x78, 0x56, 0x91, 0x65, 0x09, 0xe1, 0x8a, 0xac, 0xc3, 0x32, 0x39, 0x3b, 0x8b, 0x10, 0x13, 0x7b,
+	0x55, 0xb2, 0xd4, 0x28, 0x09, 0xc1, 0xd5, 0x4c, 0x08, 0x5e, 0x13, 0xf7, 0xda, 0xf3, 0xe7, 0xc7,
+	0xfb, 0x17, 0x28, 0x60, 0xfa, 0x06, 0x7e, 0x04, 0x55, 0x0d, 0xfa, 0x2d, 0x6d, 0xe7, 0xf7, 0xa0,
+	0xb5, 0xe3, 0x79, 0x83, 0x97, 0x4e, 0xa8, 0xed, 0xd1, 0x85, 0x4a, 0x7f, 0xef, 0xa8, 0x2f, 0x4d,
+	0x52, 0xe2, 0x0a, 0xa8, 0x21, 0xbf, 0xf1, 0x0f, 0x10, 0x3b, 0x46, 0x8c, 0x62, 0x37, 0xb9, 0xf1,
+	0xef, 0x42, 0x45, 0x41, 0xf8, 0x4a, 0x5f, 0x7e, 0xea, 0x2b, 0x40, 0x0d, 0xcd, 0x3f, 0x01, 0xe3,
+	0x7b, 0x9e, 0x2f, 0x23, 0x59, 0x2c, 0x29, 0x4e, 0xef, 0x41, 0xe7, 0x42, 0x40, 0x6d, 0x99, 0x48,
+	0x66, 0xb6, 0xa1, 0x2d, 0x27, 0x44, 0x7c, 0x10, 0xbc, 0x4f, 0x60, 0x55, 0xa6, 0xf7, 0x92, 0xce,
+	0x35, 0x48, 0x70, 0x1b, 0x26, 0xfb, 0x59, 0xb6, 0xc4, 0xf7, 0xe3, 0xbf, 0x5b, 0x53, 0xd7, 0x98,
+	0xea, 0x26, 0x19, 0x07, 0xd0, 0x9e, 0x78, 0xfa, 0x33, 0x54, 0x7b, 0x71, 0xf6, 0x8b, 0x60, 0x6f,
+	0x7d, 0x4b, 0x3e, 0x25, 0x6e, 0xe9, 0xa7, 0xc4, 0xad, 0x7d, 0x3f, 0x64, 0x57, 0xc6, 0x3e, 0xb4,
+	0xf2, 0x8f, 0x64, 0xc6, 0x5b, 0x3a, 0xc5, 0x9a, 0xf1, 0x74, 0x36, 0x97, 0xcc, 0x01, 0xb4, 0x27,
+	0xde, 0xcb, 0xb4, 0x3c, 0xb3, 0x9f, 0xd1, 0xe6, 0x12, 0x7a, 0x0a, 0xf5, 0xcc, 0x03, 0x99, 0xd1,
+	0x95, 0x44, 0xa6, 0xdf, 0xcc, 0xe6, 0x12, 0xd8, 0x83, 0x66, 0xee, 0xcd, 0xca, 0xe8, 0x29, 0x7d,
+	0x66, 0x3c, 0x64, 0xcd, 0x25, 0xb2, 0x0b, 0xf5, 0xcc, 0xd3, 0x91, 0x96, 0x62, 0xfa, 0x7d, 0xaa,
+	0x77, 0x6b, 0xc6, 0x8c, 0xba, 0x2d, 0x0f, 0xa0, 0x3d, 0xf1, 0x9e, 0xa4, 0x4d, 0x32, 0xfb, 0x99,
+	0x69, 0xae, 0x30, 0x03, 0xb8, 0x31, 0x33, 0x4b, 0x36, 0xcc, 0x2c, 0xb9, 0xd9, 0x29, 0xf4, 0x5c,
+	0xa2, 0xdf, 0x88, 0x7d, 0xcf, 0xf4, 0x20, 0x32, 0xfb, 0x3e, 0xfd, 0x24, 0xd5, 0xbb, 0x3d, 0x7b,
+	0x52, 0xa9, 0xba, 0x0f, 0xad, 0xfc, 0x6b, 0x94, 0x26, 0x36, 0xf3, 0x8d, 0x6a, 0xb1, 0x13, 0xe5,
+	0x1e, 0xa6, 0x52, 0x27, 0x9a, 0xf5, 0x5e, 0x35, 0x97, 0x10, 0x82, 0x8d, 0xc5, 0x75, 0x97, 0xf1,
+	0x7e, 0xd6, 0x39, 0x5f, 0x53, 0x9d, 0xcd, 0x65, 0xb3, 0x03, 0xa0, 0x1a, 0x1b, 0x1e, 0x0e, 0x12,
+	0x27, 0x99, 0x6a, 0xa8, 0x24, 0x4e, 0x32, 0xa3, 0x09, 0xf2, 0x14, 0x40, 0xf6, 0x23, 0x3c, 0x12,
+	0x33, 0xe3, 0xa6, 0x96, 0x6a, 0xa2, 0x09, 0xd2, 0xeb, 0x4e, 0x4f, 0x4c, 0x11, 0x40, 0x94, 0x5e,
+	0x87, 0xc0, 0x97, 0x00, 0x69, 0x9f, 0x43, 0x13, 0x98, 0xea, 0x7c, 0x2c, 0xb0, 0x41, 0x23, 0xdb,
+	0xd5, 0x30, 0x94, 0xae, 0x33, 0x3a, 0x1d, 0x0b, 0x48, 0xb4, 0x27, 0x8a, 0xd1, 0xfc, 0x41, 0x99,
+	0xac, 0x51, 0x7b, 0x53, 0x05, 0xa9, 0xf1, 0x04, 0x1a, 0xd9, 0x2a, 0x54, 0x4b, 0x31, 0xa3, 0x32,
+	0xed, 0xe5, 0x2a, 0x51, 0xe3, 0x29, 0xb4, 0xf2, 0x65, 0xa3, 0xf6, 0xdc, 0x99, 0xc5, 0x64, 0x4f,
+	0x35, 0x62, 0x33, 0xe8, 0x1f, 0x01, 0xa4, 0xe5, 0xa5, 0x36, 0xdf, 0x54, 0xc1, 0x39, 0xc1, 0xf5,
+	0x00, 0xda, 0x13, 0x65, 0xa3, 0xd6, 0x78, 0x76, 0x35, 0xb9, 0x28, 0x4e, 0x65, 0x8a, 0x40, 0xed,
+	0x82, 0xd3, 0x65, 0xa4, 0x76, 0xc1, 0x59, 0x15, 0xe3, 0x2e, 0xd4, 0x07, 0xd3, 0x34, 0x06, 0x73,
+	0x69, 0xcc, 0xaa, 0x03, 0x3f, 0x06, 0x48, 0xaf, 0x5c, 0x6d, 0x85, 0xa9, 0x4b, 0xb8, 0xd7, 0xd4,
+	0xcd, 0x72, 0x89, 0xb7, 0x07, 0xcd, 0x5c, 0x3f, 0x49, 0x87, 0xea, 0x59, 0x4d, 0xa6, 0x45, 0x17,
+	0x58, 0xbe, 0xf9, 0xa2, 0x77, 0x70, 0x66, 0x4b, 0x66, 0x91, 0x1f, 0x67, 0xab, 0x56, 0xed, 0x41,
+	0x33, 0x2a, 0xd9, 0xd7, 0x84, 0xaf, 0x6c, 0x65, 0x9a, 0x09, 0x5f, 0x33, 0x0a, 0xd6, 0xb9, 0x84,
+	0x0e, 0xa1, 0x7d, 0xa0, 0x8b, 0x0e, 0x55, 0x10, 0xe9, 0xfd, 0x9b, 0x2e, 0x00, 0x7b, 0xbd, 0x59,
+	0x53, 0x6a, 0x5f, 0xbe, 0x81, 0xce, 0x54, 0x31, 0x64, 0x6c, 0x24, 0x4f, 0x16, 0x33, 0xab, 0xa4,
+	0xb9, 0x62, 0x1d, 0xc1, 0xca, 0x64, 0x2d, 0x64, 0xfc, 0x3e, 0xf1, 0x89, 0x59, 0x35, 0xd2, 0x5c,
+	0x52, 0x9f, 0x41, 0x55, 0xe7, 0xb7, 0x86, 0x7a, 0x1a, 0x9a, 0xc8, 0x77, 0xe7, 0x2e, 0x7d, 0x22,
+	0x5c, 0x3e, 0xc9, 0x1d, 0x53, 0x97, 0x9f, 0xc8, 0x30, 0x7b, 0xea, 0x25, 0x27, 0xc1, 0x7c, 0x02,
+	0x15, 0x95, 0x42, 0x1a, 0x6b, 0xc9, 0x61, 0xcb, 0x64, 0x94, 0x8b, 0x3c, 0xec, 0x00, 0xb1, 0x4c,
+	0x62, 0xa8, 0x99, 0x4e, 0xe7, 0x8a, 0xfa, 0x8c, 0xe4, 0x66, 0xd4, 0x5e, 0xec, 0x40, 0x23, 0x9b,
+	0x1a, 0xea, 0x2d, 0x9d, 0x91, 0x2e, 0xce, 0x93, 0x64, 0xf7, 0xf2, 0x97, 0x5f, 0x37, 0x7e, 0xf7,
+	0x5f, 0xbf, 0x6e, 0xfc, 0xee, 0x6f, 0x5f, 0x6d, 0x14, 0x7e, 0x79, 0xb5, 0x51, 0xf8, 0xcf, 0x57,
+	0x1b, 0x85, 0xff, 0x7d, 0xb5, 0x51, 0xf8, 0xf3, 0xbf, 0x1a, 0x62, 0x36, 0x8a, 0x4f, 0xb7, 0x5c,
+	0xe2, 0x6f, 0x9f, 0x3b, 0xcc, 0x79, 0x94, 0x24, 0xcf, 0xd1, 0xd4, 0x38, 0xa2, 0xee, 0x36, 0x8d,
+	0x03, 0x86, 0x7d, 0xb4, 0x7d, 0x81, 0x29, 0xcb, 0x4c, 0x85, 0xe7, 0xc3, 0x6d, 0x51, 0x88, 0xcb,
+	0x7f, 0x9c, 0xb9, 0x64, 0x1c, 0x6d, 0x73, 0x29, 0x4f, 0x97, 0xc5, 0xf8, 0xa3, 0xff, 0x0f, 0x00,
+	0x00, 0xff, 0xff, 0x76, 0x47, 0x9c, 0x5b, 0xc7, 0x26, 0x00, 0x00,
 }
 
 func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) {
@@ -4741,6 +4779,33 @@ func (m *DestroySandboxRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *RemoveStaleVirtiofsShareMountsRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *RemoveStaleVirtiofsShareMountsRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *RemoveStaleVirtiofsShareMountsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	return len(dAtA) - i, nil
+}
+
 func (m *Interfaces) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -6917,6 +6982,18 @@ func (m *DestroySandboxRequest) Size() (n int) {
 	return n
 }
 
+func (m *RemoveStaleVirtiofsShareMountsRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
 func (m *Interfaces) Size() (n int) {
 	if m == nil {
 		return 0
@@ -8056,6 +8133,16 @@ func (this *DestroySandboxRequest) String() string {
 	}, "")
 	return s
 }
+func (this *RemoveStaleVirtiofsShareMountsRequest) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&RemoveStaleVirtiofsShareMountsRequest{`,
+		`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func (this *Interfaces) String() string {
 	if this == nil {
 		return "nil"
@@ -8477,6 +8564,7 @@ type AgentServiceService interface {
 	StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error)
 	PauseContainer(ctx context.Context, req *PauseContainerRequest) (*types.Empty, error)
 	ResumeContainer(ctx context.Context, req *ResumeContainerRequest) (*types.Empty, error)
+	RemoveStaleVirtiofsShareMounts(ctx context.Context, req *RemoveStaleVirtiofsShareMountsRequest) (*types.Empty, error)
 	WriteStdin(ctx context.Context, req *WriteStreamRequest) (*WriteStreamResponse, error)
 	ReadStdout(ctx context.Context, req *ReadStreamRequest) (*ReadStreamResponse, error)
 	ReadStderr(ctx context.Context, req *ReadStreamRequest) (*ReadStreamResponse, error)
@@ -8583,6 +8671,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag
 			}
 			return svc.ResumeContainer(ctx, &req)
 		},
+		"RemoveStaleVirtiofsShareMounts": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req RemoveStaleVirtiofsShareMountsRequest
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.RemoveStaleVirtiofsShareMounts(ctx, &req)
+		},
 		"WriteStdin": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
 			var req WriteStreamRequest
 			if err := unmarshal(&req); err != nil {
@@ -8859,6 +8954,14 @@ func (c *agentServiceClient) ResumeContainer(ctx context.Context, req *ResumeCon
 	return &resp, nil
 }
 
+func (c *agentServiceClient) RemoveStaleVirtiofsShareMounts(ctx context.Context, req *RemoveStaleVirtiofsShareMountsRequest) (*types.Empty, error) {
+	var resp types.Empty
+	if err := c.client.Call(ctx, "grpc.AgentService", "RemoveStaleVirtiofsShareMounts", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
 func (c *agentServiceClient) WriteStdin(ctx context.Context, req *WriteStreamRequest) (*WriteStreamResponse, error) {
 	var resp WriteStreamResponse
 	if err := c.client.Call(ctx, "grpc.AgentService", "WriteStdin", req, &resp); err != nil {
@@ -13724,6 +13827,57 @@ func (m *DestroySandboxRequest) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *RemoveStaleVirtiofsShareMountsRequest) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowAgent
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: RemoveStaleVirtiofsShareMountsRequest: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: RemoveStaleVirtiofsShareMountsRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		default:
+			iNdEx = preIndex
+			skippy, err := skipAgent(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthAgent
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *Interfaces) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go
index 6109d29ac..562d4a7e3 100644
--- a/src/runtime/virtcontainers/pkg/mock/mock.go
+++ b/src/runtime/virtcontainers/pkg/mock/mock.go
@@ -245,6 +245,10 @@ func (p *HybridVSockTTRPCMockImp) ResizeVolume(ctx context.Context, req *pb.Resi
 	return &gpb.Empty{}, nil
 }
 
+func (p *HybridVSockTTRPCMockImp) RemoveStaleVirtiofsShareMounts(ctx context.Context, req *pb.RemoveStaleVirtiofsShareMountsRequest) (*gpb.Empty, error) {
+	return &gpb.Empty{}, nil
+}
+
 func (p *HybridVSockTTRPCMockImp) GetIPTables(ctx context.Context, req *pb.GetIPTablesRequest) (*pb.GetIPTablesResponse, error) {
 	return &pb.GetIPTablesResponse{}, nil
 }

From 521519d7455d1def6f64ad92b509af6cfb947918 Mon Sep 17 00:00:00 2001
From: Ryan Savino <ryan.savino@amd.com>
Date: Mon, 24 Apr 2023 11:34:34 -0500
Subject: [PATCH 014/150] gha: Add the ability to test qemu-sev

With the changes proposed as part of this PR, a qemu-sev cluster will
be created but no tests will be performed.

GitHub Actions will only run the tests using the workflows that are
part of the **target** branch, instead of the using the ones coming
from the PR. No way to work around this for now.

After this commit is merged, the tests (not the yaml files for the
actions) will be altered in order for the checkout action  to help in this
case.

Fixes: #6711

Signed-off-by: Ryan Savino <ryan.savino@amd.com>
---
 tests/integration/kubernetes/run_kubernetes_tests.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
index db1e16633..daddb756a 100755
--- a/tests/integration/kubernetes/run_kubernetes_tests.sh
+++ b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -54,6 +54,10 @@ else
 	)
 fi
 
+if [ ${KATA_HYPERVISOR} == "qemu-sev" ]; then
+	exit 0
+fi
+
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then

From 17daeb9dd7ee4ca0dd5e736368feaf3f9d66da0d Mon Sep 17 00:00:00 2001
From: "alex.lyn" <alex.lyn@antgroup.com>
Date: Wed, 26 Apr 2023 19:18:49 +0800
Subject: [PATCH 015/150] warning_fix: fix warnings when build with
 cargo-1.68.0

Fixes: #6593

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 src/libs/kata-types/src/annotations/mod.rs    |  4 +-
 src/runtime-rs/crates/agent/src/types.rs      | 18 +---
 .../crates/hypervisor/src/ch/inner_device.rs  |  2 +-
 .../hypervisor/src/ch/inner_hypervisor.rs     |  6 +-
 .../resource/src/network/utils/address.rs     |  4 +-
 .../crates/resource/src/network/utils/mod.rs  |  2 +-
 .../crates/resource/src/rootfs/mod.rs         | 12 ++-
 src/runtime-rs/crates/runtimes/src/manager.rs |  5 +-
 .../src/container_manager/process.rs          |  4 +-
 .../virt_container/src/health_check.rs        |  2 +-
 .../runtimes/virt_container/src/sandbox.rs    |  2 +-
 src/tools/agent-ctl/src/main.rs               |  2 +-
 src/tools/kata-ctl/src/arch/x86_64/mod.rs     |  2 +-
 src/tools/kata-ctl/src/ops/exec_ops.rs        |  4 +-
 src/tools/kata-ctl/src/ops/volume_ops.rs      |  2 +-
 src/tools/kata-ctl/src/utils.rs               |  2 +-
 src/tools/runk/Cargo.lock                     | 83 ++++++++++++++-----
 src/tools/runk/libcontainer/src/cgroup.rs     |  2 +-
 src/tools/runk/src/main.rs                    |  4 +-
 versions.yaml                                 |  4 +-
 20 files changed, 91 insertions(+), 75 deletions(-)

diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs
index 16af5ab28..3af0563c1 100644
--- a/src/libs/kata-types/src/annotations/mod.rs
+++ b/src/libs/kata-types/src/annotations/mod.rs
@@ -417,10 +417,10 @@ impl Annotation {
         match self.get_value::<u32>(KATA_ANNO_CONTAINER_RES_SWAPPINESS) {
             Ok(r) => {
                 if r.unwrap_or_default() > 100 {
-                    return Err(io::Error::new(
+                    Err(io::Error::new(
                         io::ErrorKind::InvalidData,
                         format!("{} greater than 100", r.unwrap_or_default()),
-                    ));
+                    ))
                 } else {
                     Ok(r)
                 }
diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs
index de8822980..da6e14430 100644
--- a/src/runtime-rs/crates/agent/src/types.rs
+++ b/src/runtime-rs/crates/agent/src/types.rs
@@ -20,14 +20,9 @@ impl Empty {
     }
 }
 
-impl Default for FSGroupChangePolicy {
-    fn default() -> Self {
-        FSGroupChangePolicy::Always
-    }
-}
-
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub enum FSGroupChangePolicy {
+    #[default]
     Always = 0,
     OnRootMismatch = 1,
 }
@@ -65,18 +60,13 @@ pub struct Storage {
     pub mount_point: String,
 }
 
-#[derive(Deserialize, Clone, PartialEq, Eq, Debug, Hash)]
+#[derive(Deserialize, Default, Clone, PartialEq, Eq, Debug, Hash)]
 pub enum IPFamily {
+    #[default]
     V4 = 0,
     V6 = 1,
 }
 
-impl ::std::default::Default for IPFamily {
-    fn default() -> Self {
-        IPFamily::V4
-    }
-}
-
 #[derive(Deserialize, Debug, PartialEq, Clone, Default)]
 pub struct IPAddress {
     pub family: IPFamily,
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
index f4475f66e..c04531de4 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
@@ -42,7 +42,7 @@ impl CloudHypervisorInner {
         match device {
             Device::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
             Device::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
-            _ => return Err(anyhow!("unhandled device: {:?}", device)),
+            _ => Err(anyhow!("unhandled device: {:?}", device)),
         }
     }
 
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
index 89747d936..fa1979e83 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
@@ -228,11 +228,9 @@ impl CloudHypervisorInner {
 
         let join_handle = self.cloud_hypervisor_ping_until_ready(CH_POLL_TIME_MS);
 
-        let result = tokio::time::timeout(Duration::new(timeout_secs as u64, 0), join_handle)
+        tokio::time::timeout(Duration::new(timeout_secs as u64, 0), join_handle)
             .await
-            .context(timeout_msg)?;
-
-        result
+            .context(timeout_msg)?
     }
 
     async fn cloud_hypervisor_ensure_not_launched(&self) -> Result<()> {
diff --git a/src/runtime-rs/crates/resource/src/network/utils/address.rs b/src/runtime-rs/crates/resource/src/network/utils/address.rs
index ef3b68278..3046d3685 100644
--- a/src/runtime-rs/crates/resource/src/network/utils/address.rs
+++ b/src/runtime-rs/crates/resource/src/network/utils/address.rs
@@ -80,9 +80,7 @@ pub(crate) fn parse_ip(ip: &[u8], family: u8) -> Result<IpAddr> {
             octets.copy_from_slice(&ip[..16]);
             Ok(IpAddr::V6(Ipv6Addr::from(octets)))
         }
-        _ => {
-            return Err(anyhow!("unknown IP network family {}", family));
-        }
+        _ => Err(anyhow!("unknown IP network family {}", family)),
     }
 }
 
diff --git a/src/runtime-rs/crates/resource/src/network/utils/mod.rs b/src/runtime-rs/crates/resource/src/network/utils/mod.rs
index 74635a5d9..341038cb9 100644
--- a/src/runtime-rs/crates/resource/src/network/utils/mod.rs
+++ b/src/runtime-rs/crates/resource/src/network/utils/mod.rs
@@ -25,7 +25,7 @@ pub(crate) fn parse_mac(s: &str) -> Option<hypervisor::Address> {
 
 pub(crate) fn get_mac_addr(b: &[u8]) -> Result<String> {
     if b.len() != 6 {
-        return Err(anyhow!("invalid mac address {:?}", b));
+        Err(anyhow!("invalid mac address {:?}", b))
     } else {
         Ok(format!(
             "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs
index b18085ed8..2c9160cec 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs
@@ -79,7 +79,7 @@ impl RootFsResource {
                         .context("new share fs rootfs")?,
                     ))
                 } else {
-                    return Err(anyhow!("share fs is unavailable"));
+                    Err(anyhow!("share fs is unavailable"))
                 }
             }
             mounts_vec if is_single_layer_rootfs(mounts_vec) => {
@@ -114,12 +114,10 @@ impl RootFsResource {
                 inner.rootfs.push(Arc::clone(&rootfs));
                 Ok(rootfs)
             }
-            _ => {
-                return Err(anyhow!(
-                    "unsupported rootfs mounts count {}",
-                    rootfs_mounts.len()
-                ))
-            }
+            _ => Err(anyhow!(
+                "unsupported rootfs mounts count {}",
+                rootfs_mounts.len()
+            )),
         }
     }
 
diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs
index b32c36773..1fa7ce4a0 100644
--- a/src/runtime-rs/crates/runtimes/src/manager.rs
+++ b/src/runtime-rs/crates/runtimes/src/manager.rs
@@ -397,12 +397,11 @@ fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig>
         path
     } else if let Some(option) = option {
         // get rid of the special characters in options to get the config path
-        let path = if option.len() > 2 {
+        if option.len() > 2 {
             from_utf8(&option[2..])?.to_string()
         } else {
             String::from("")
-        };
-        path
+        }
     } else {
         String::from("")
     };
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs
index 438a817e2..32856f27c 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs
@@ -132,7 +132,7 @@ impl Process {
         info!(self.logger, "run io copy for {}", io_name);
         let io_name = io_name.to_string();
         let logger = self.logger.new(o!("io_name" => io_name));
-        let _ = tokio::spawn(async move {
+        tokio::spawn(async move {
             match tokio::io::copy(&mut reader, &mut writer).await {
                 Err(e) => {
                     warn!(logger, "run_io_copy: failed to copy stream: {}", e);
@@ -156,7 +156,7 @@ impl Process {
         let exit_notifier = self.exit_watcher_tx.take();
         let status = self.status.clone();
 
-        let _ = tokio::spawn(async move {
+        tokio::spawn(async move {
             // wait on all of the container's io stream terminated
             info!(logger, "begin wait group io");
             wg.wait().await;
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs b/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs
index 81fb3d58b..874ccb7f1 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs
@@ -47,7 +47,7 @@ impl HealthCheck {
 
         let stop_rx = self.stop_rx.clone();
         let keep_abnormal = self.keep_abnormal;
-        let _ = tokio::spawn(async move {
+        tokio::spawn(async move {
             let mut version_check_threshold_count = 0;
 
             loop {
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
index c5ec38e46..8202d854d 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs
@@ -285,7 +285,7 @@ impl Sandbox for VirtSandbox {
         let agent = self.agent.clone();
         let sender = self.msg_sender.clone();
         info!(sl!(), "oom watcher start");
-        let _ = tokio::spawn(async move {
+        tokio::spawn(async move {
             loop {
                 match agent
                     .get_oom_event(agent::Empty::new())
diff --git a/src/tools/agent-ctl/src/main.rs b/src/tools/agent-ctl/src/main.rs
index f08c1a884..a56915727 100644
--- a/src/tools/agent-ctl/src/main.rs
+++ b/src/tools/agent-ctl/src/main.rs
@@ -314,7 +314,7 @@ fn real_main() -> Result<()> {
             Ok(())
         }
         "connect" => connect(name, args),
-        _ => return Err(anyhow!(format!("invalid sub-command: {:?}", subcmd))),
+        _ => Err(anyhow!(format!("invalid sub-command: {:?}", subcmd))),
     }
 }
 
diff --git a/src/tools/kata-ctl/src/arch/x86_64/mod.rs b/src/tools/kata-ctl/src/arch/x86_64/mod.rs
index 026312624..a01f4a2cd 100644
--- a/src/tools/kata-ctl/src/arch/x86_64/mod.rs
+++ b/src/tools/kata-ctl/src/arch/x86_64/mod.rs
@@ -225,7 +225,7 @@ mod arch_specific {
                 expected_param_value
             );
 
-            return Err(anyhow!("{} {}", error_msg, action_msg));
+            Err(anyhow!("{} {}", error_msg, action_msg))
         }
     }
 
diff --git a/src/tools/kata-ctl/src/ops/exec_ops.rs b/src/tools/kata-ctl/src/ops/exec_ops.rs
index 2a6b970aa..9e1d4549a 100644
--- a/src/tools/kata-ctl/src/ops/exec_ops.rs
+++ b/src/tools/kata-ctl/src/ops/exec_ops.rs
@@ -326,9 +326,7 @@ fn setup_client(server_url: String, dbg_console_port: u32) -> anyhow::Result<Uni
             vsock.setup_sock().context("set up vsock")
         }
         // Others will be INVALID URI.
-        _ => {
-            return Err(anyhow!("invalid URI scheme: {:?}", scheme));
-        }
+        _ => Err(anyhow!("invalid URI scheme: {:?}", scheme)),
     }
 }
 
diff --git a/src/tools/kata-ctl/src/ops/volume_ops.rs b/src/tools/kata-ctl/src/ops/volume_ops.rs
index 60c753f6f..1027dcb72 100644
--- a/src/tools/kata-ctl/src/ops/volume_ops.rs
+++ b/src/tools/kata-ctl/src/ops/volume_ops.rs
@@ -162,7 +162,7 @@ pub fn get_sandbox_id_for_volume(volume_path: &str) -> Result<String> {
         return Ok(String::from(file_name));
     }
 
-    return Err(anyhow!("no sandbox found for {}", volume_path));
+    Err(anyhow!("no sandbox found for {}", volume_path))
 }
 
 #[cfg(test)]
diff --git a/src/tools/kata-ctl/src/utils.rs b/src/tools/kata-ctl/src/utils.rs
index 03c005e9f..ba5a6497d 100644
--- a/src/tools/kata-ctl/src/utils.rs
+++ b/src/tools/kata-ctl/src/utils.rs
@@ -53,7 +53,7 @@ const UNKNOWN: &str = "unknown";
 fn get_field_fn(line: &str, delimiter: &str, file_name: &str) -> Result<String> {
     let fields: Vec<&str> = line.split(delimiter).collect();
     if fields.len() < 2 {
-        return Err(anyhow!("Unexpected file contents for {}", file_name));
+        Err(anyhow!("Unexpected file contents for {}", file_name))
     } else {
         let val = fields[1].trim();
         Ok(String::from(val))
diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock
index f9ebce67b..5b66a95fa 100644
--- a/src/tools/runk/Cargo.lock
+++ b/src/tools/runk/Cargo.lock
@@ -1303,9 +1303,16 @@ name = "protobuf"
 version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
+
+[[package]]
+name = "protobuf"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b55bad9126f378a853655831eb7363b7b01b81d19f8cb1218861086ca4a1a61e"
 dependencies = [
- "serde",
- "serde_derive",
+ "once_cell",
+ "protobuf-support",
+ "thiserror",
 ]
 
 [[package]]
@@ -1314,17 +1321,47 @@ version = "2.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707"
 dependencies = [
- "protobuf",
+ "protobuf 2.27.1",
 ]
 
 [[package]]
-name = "protobuf-codegen-pure"
-version = "2.27.1"
+name = "protobuf-codegen"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6"
+checksum = "0dd418ac3c91caa4032d37cb80ff0d44e2ebe637b2fb243b6234bf89cdac4901"
 dependencies = [
- "protobuf",
- "protobuf-codegen",
+ "anyhow",
+ "once_cell",
+ "protobuf 3.2.0",
+ "protobuf-parse",
+ "regex",
+ "tempfile",
+ "thiserror",
+]
+
+[[package]]
+name = "protobuf-parse"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d39b14605eaa1f6a340aec7f320b34064feb26c93aec35d6a9a2272a8ddfa49"
+dependencies = [
+ "anyhow",
+ "indexmap",
+ "log",
+ "protobuf 3.2.0",
+ "protobuf-support",
+ "tempfile",
+ "thiserror",
+ "which",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d4d7b8601c814cfb36bcebb79f0e61e45e1e93640cf778837833bbed05c372"
+dependencies = [
+ "thiserror",
 ]
 
 [[package]]
@@ -1332,7 +1369,7 @@ name = "protocols"
 version = "0.1.0"
 dependencies = [
  "oci",
- "protobuf",
+ "protobuf 3.2.0",
  "ttrpc",
  "ttrpc-codegen",
 ]
@@ -1530,7 +1567,7 @@ dependencies = [
  "nix 0.24.2",
  "oci",
  "path-absolutize",
- "protobuf",
+ "protobuf 3.2.0",
  "protocols",
  "regex",
  "rlimit",
@@ -1911,43 +1948,43 @@ dependencies = [
 
 [[package]]
 name = "ttrpc"
-version = "0.6.1"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8"
+checksum = "a35f22a2964bea14afee161665bb260b83cb48e665e0260ca06ec0e775c8b06c"
 dependencies = [
  "byteorder",
  "libc",
  "log",
  "nix 0.23.1",
- "protobuf",
- "protobuf-codegen-pure",
+ "protobuf 3.2.0",
+ "protobuf-codegen 3.2.0",
  "thiserror",
 ]
 
 [[package]]
 name = "ttrpc-codegen"
-version = "0.2.0"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408"
+checksum = "94d7f7631d7a9ebed715a47cd4cb6072cbc7ae1d4ec01598971bbec0024340c2"
 dependencies = [
- "protobuf",
- "protobuf-codegen",
- "protobuf-codegen-pure",
+ "protobuf 2.27.1",
+ "protobuf-codegen 3.2.0",
+ "protobuf-support",
  "ttrpc-compiler",
 ]
 
 [[package]]
 name = "ttrpc-compiler"
-version = "0.4.1"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15"
+checksum = "ec3cb5dbf1f0865a34fe3f722290fe776cacb16f50428610b779467b76ddf647"
 dependencies = [
  "derive-new",
  "prost",
  "prost-build",
  "prost-types",
- "protobuf",
- "protobuf-codegen",
+ "protobuf 2.27.1",
+ "protobuf-codegen 2.27.1",
  "tempfile",
 ]
 
diff --git a/src/tools/runk/libcontainer/src/cgroup.rs b/src/tools/runk/libcontainer/src/cgroup.rs
index 586c6e894..0f4673da1 100644
--- a/src/tools/runk/libcontainer/src/cgroup.rs
+++ b/src/tools/runk/libcontainer/src/cgroup.rs
@@ -27,7 +27,7 @@ pub fn remove_cgroup_dir(cgroup: &cgroups::Cgroup) -> Result<()> {
         retries -= 1;
     }
 
-    return Err(anyhow!("failed to remove cgroups paths"));
+    Err(anyhow!("failed to remove cgroups paths"))
 }
 
 // Make sure we get a stable freezer state, so retry if the cgroup is still undergoing freezing.
diff --git a/src/tools/runk/src/main.rs b/src/tools/runk/src/main.rs
index c79746678..2e6e5220e 100644
--- a/src/tools/runk/src/main.rs
+++ b/src/tools/runk/src/main.rs
@@ -66,9 +66,7 @@ async fn cmd_run(subcmd: SubCommand, root_path: &Path, logger: &Logger) -> Resul
             CommonCmd::Ps(ps) => commands::ps::run(ps, root_path, logger),
             CommonCmd::Pause(pause) => commands::pause::run(pause, root_path, logger),
             CommonCmd::Resume(resume) => commands::resume::run(resume, root_path, logger),
-            _ => {
-                return Err(anyhow!("command is not implemented yet"));
-            }
+            _ => Err(anyhow!("command is not implemented yet")),
         },
         _ => unreachable!(),
     }
diff --git a/versions.yaml b/versions.yaml
index c222499d9..0e0b88d47 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -321,12 +321,12 @@ languages:
   rust:
     description: "Rust language"
     notes: "'version' is the default minimum version used by this project."
-    version: "1.66.0"
+    version: "1.68.0"
     meta:
       description: |
         'newest-version' is the latest version known to work when
         building Kata
-      newest-version: "1.66.0"
+      newest-version: "1.68.0"
 
   golangci-lint:
     description: "golangci-lint"

From 376884b8a428a81ce6ef5da59d472239872a990d Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Wed, 5 Apr 2023 15:54:55 -0700
Subject: [PATCH 016/150] cargo: Update version of clap to 4.1.13

This version includes macros related to using command options.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/Cargo.lock | 428 +++++++++++++++++++++++-----------
 src/tools/kata-ctl/Cargo.toml |   2 +-
 2 files changed, 293 insertions(+), 137 deletions(-)

diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock
index 6acf131f0..e31c8622d 100644
--- a/src/tools/kata-ctl/Cargo.lock
+++ b/src/tools/kata-ctl/Cargo.lock
@@ -33,6 +33,46 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-wincon",
+ "concolor-override",
+ "concolor-query",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.69"
@@ -53,18 +93,7 @@ checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
-]
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -98,7 +127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -149,41 +178,60 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "3.2.23"
+version = "4.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3"
 dependencies = [
- "atty",
- "bitflags",
+ "clap_builder",
  "clap_derive",
+ "once_cell",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "bitflags",
  "clap_lex",
- "indexmap",
  "once_cell",
  "strsim",
- "termcolor",
- "textwrap",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "3.2.18"
+version = "4.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65"
+checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
 dependencies = [
  "heck 0.4.1",
- "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.13",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.2.4"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
+
+[[package]]
+name = "concolor-override"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f"
+
+[[package]]
+name = "concolor-query"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf"
 dependencies = [
- "os_str_bytes",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -229,7 +277,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -257,6 +305,27 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "errno"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys 0.45.0",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.8.0"
@@ -358,7 +427,7 @@ checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -437,15 +506,6 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.2.6"
@@ -455,6 +515,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -587,6 +653,17 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "io-lifetimes"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -602,6 +679,18 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "io-lifetimes",
+ "rustix",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -694,6 +783,12 @@ version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f"
+
 [[package]]
 name = "lock_api"
 version = "0.4.9"
@@ -772,7 +867,7 @@ dependencies = [
  "libc",
  "log",
  "wasi",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -901,7 +996,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -923,12 +1018,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "os_str_bytes"
-version = "6.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
-
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -987,7 +1076,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1018,35 +1107,11 @@ dependencies = [
  "nix 0.26.2",
 ]
 
-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
 [[package]]
 name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
 dependencies = [
  "unicode-ident",
 ]
@@ -1089,7 +1154,7 @@ dependencies = [
  "itertools",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1181,9 +1246,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
  "proc-macro2",
 ]
@@ -1280,6 +1345,20 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.37.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b24138615de35e32031d041a09032ef3487a616d901ca4db224e7d557efae2"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "rustls"
 version = "0.20.8"
@@ -1326,7 +1405,7 @@ version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"
 dependencies = [
- "windows-sys",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -1391,7 +1470,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1436,7 +1515,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1549,7 +1628,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1563,6 +1642,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -1583,15 +1673,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "termcolor"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
-dependencies = [
- "winapi-util",
-]
-
 [[package]]
 name = "test-utils"
 version = "0.1.0"
@@ -1599,12 +1680,6 @@ dependencies = [
  "nix 0.24.3",
 ]
 
-[[package]]
-name = "textwrap"
-version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
-
 [[package]]
 name = "thiserror"
 version = "1.0.38"
@@ -1622,7 +1697,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1692,7 +1767,7 @@ dependencies = [
  "pin-project-lite",
  "socket2",
  "tokio-macros",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -1703,7 +1778,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
 ]
 
 [[package]]
@@ -1885,18 +1960,18 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
-
 [[package]]
 name = "vmm-sys-util"
 version = "0.11.1"
@@ -1954,7 +2029,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
  "wasm-bindgen-shared",
 ]
 
@@ -1988,7 +2063,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.107",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2055,15 +2130,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
-[[package]]
-name = "winapi-util"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
@@ -2076,56 +2142,146 @@ version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.0",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm 0.48.0",
+ "windows_x86_64_msvc 0.48.0",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
 [[package]]
 name = "winreg"
diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml
index 3de3a532a..d2fc5fb1e 100644
--- a/src/tools/kata-ctl/Cargo.toml
+++ b/src/tools/kata-ctl/Cargo.toml
@@ -14,7 +14,7 @@ edition = "2018"
 
 [dependencies]
 anyhow = "1.0.31"
-clap = { version = "3.2.20", features = ["derive", "cargo"] }
+clap = { version = "4.1.13", features = ["derive", "cargo"] }
 serde_json = "1.0.85"
 thiserror = "1.0.35"
 privdrop = "0.5.2"

From 572b338b3bfd21dda94dea85f3c1b3417305c609 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Wed, 5 Apr 2023 16:12:35 -0700
Subject: [PATCH 017/150] gitignore: Ignore .swp and .swo editor backup files

Ignore temporary files created by vim editor.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index fca150940..29d21ac6d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,8 @@
 **/.vscode
 **/.idea
 **/.fleet
+**/*.swp
+**/*.swo
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service

From 94a00f9346b1d39434cfb47408e69065adc10997 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Mon, 10 Apr 2023 15:52:33 -0700
Subject: [PATCH 018/150] utils: Make certain constants in utils.rs public

These would be used outside of utils.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/utils.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/tools/kata-ctl/src/utils.rs b/src/tools/kata-ctl/src/utils.rs
index 03c005e9f..524821aed 100644
--- a/src/tools/kata-ctl/src/utils.rs
+++ b/src/tools/kata-ctl/src/utils.rs
@@ -27,7 +27,7 @@ pub fn drop_privs() -> Result<()> {
     Ok(())
 }
 
-const PROC_VERSION_FILE: &str = "/proc/version";
+pub const PROC_VERSION_FILE: &str = "/proc/version";
 
 pub fn get_kernel_version(proc_version_file: &str) -> Result<String> {
     let contents = fs::read_to_string(proc_version_file)
@@ -43,10 +43,10 @@ pub fn get_kernel_version(proc_version_file: &str) -> Result<String> {
     Ok(kernel_version)
 }
 
-const OS_RELEASE: &str = "/etc/os-release";
+pub const OS_RELEASE: &str = "/etc/os-release";
 
 // Clear Linux has a different path (for stateless support)
-const OS_RELEASE_CLR: &str = "/usr/lib/os-release";
+pub const OS_RELEASE_CLR: &str = "/usr/lib/os-release";
 
 const UNKNOWN: &str = "unknown";
 
@@ -144,7 +144,7 @@ pub fn get_generic_cpu_details(cpu_info_file: &str) -> Result<(String, String)>
     Ok((vendor, model))
 }
 
-const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock";
+pub const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock";
 pub fn supports_vsocks(vsock_path: &str) -> Result<bool> {
     let metadata = fs::metadata(vsock_path)?;
     Ok(metadata.is_file())

From 7565b3356831e5a07dae2b6b63235db326eef0b8 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Mon, 10 Apr 2023 16:35:39 -0700
Subject: [PATCH 019/150] kata-ctl: Implement Display trait for GuestProtection
 enum

Implement Display for enum to display in env output.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/check.rs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs
index dfb9a3b7b..bd5406bb8 100644
--- a/src/tools/kata-ctl/src/check.rs
+++ b/src/tools/kata-ctl/src/check.rs
@@ -8,6 +8,7 @@
 use anyhow::{anyhow, Result};
 use reqwest::header::{CONTENT_TYPE, USER_AGENT};
 use serde::{Deserialize, Serialize};
+use std::fmt;
 use thiserror::Error;
 
 #[cfg(any(target_arch = "x86_64"))]
@@ -147,6 +148,19 @@ pub enum GuestProtection {
     Se,
 }
 
+impl fmt::Display for GuestProtection {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            GuestProtection::Tdx => write!(f, "tdx"),
+            GuestProtection::Sev => write!(f, "sev"),
+            GuestProtection::Snp => write!(f, "snp"),
+            GuestProtection::Pef => write!(f, "pef"),
+            GuestProtection::Se => write!(f, "se"),
+            GuestProtection::NoProtection => write!(f, "none"),
+        }
+    }
+}
+
 #[allow(dead_code)]
 #[derive(Error, Debug)]
 pub enum ProtectionError {

From 68f63577319bb0f78d9d798a14d5859ffb5eec31 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 13 Apr 2023 23:13:45 -0700
Subject: [PATCH 020/150] config: Make function to get the default conf file
 public

This will be used by the kata-env command.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/libs/kata-types/src/config/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs
index 173026921..2a696922a 100644
--- a/src/libs/kata-types/src/config/mod.rs
+++ b/src/libs/kata-types/src/config/mod.rs
@@ -196,7 +196,7 @@ impl TomlConfig {
     }
 
     /// Probe configuration file according to the default configuration file list.
-    fn get_default_config_file() -> Result<PathBuf> {
+    pub fn get_default_config_file() -> Result<PathBuf> {
         for f in default::DEFAULT_RUNTIME_CONFIGURATIONS.iter() {
             if let Ok(path) = fs::canonicalize(f) {
                 return Ok(path);

From 07f7d17db59b8b8cd1bdb2fd83dacb88db47c231 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 13 Apr 2023 23:16:04 -0700
Subject: [PATCH 021/150] config: Make the pipe_size field optional

Add the serde default attribute to the field so that parsing
can continue if this field is not present.
The agent assumes a default value for this, so it is not required
by the user to provide a value here.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/libs/kata-types/src/config/agent.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs
index 98ec142f4..f30ab9a23 100644
--- a/src/libs/kata-types/src/config/agent.rs
+++ b/src/libs/kata-types/src/config/agent.rs
@@ -80,6 +80,7 @@ pub struct Agent {
     pub kernel_modules: Vec<String>,
 
     /// container pipe size
+    #[serde(default)]
     pub container_pipe_size: u32,
 }
 

From 2f81f48dae39bceac8087b7ddd7287fcbf0b0fdf Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 13 Apr 2023 23:18:39 -0700
Subject: [PATCH 022/150] config: Add file under /opt as another location to
 look for the config

Most of kata installation tools use this path for installation, so
add this to the paths to look for the configuration.toml file.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/libs/kata-types/src/config/default.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs
index 1d5e2b4d3..6b5495c5a 100644
--- a/src/libs/kata-types/src/config/default.rs
+++ b/src/libs/kata-types/src/config/default.rs
@@ -16,6 +16,7 @@ lazy_static! {
     pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![
         "/etc/kata-containers/configuration.toml",
         "/usr/share/defaults/kata-containers/configuration.toml",
+        "/opt/kata/share/defaults/kata-containers/configuration.toml",
     ];
 }
 

From 9a94f1f149a3bb35e7e73b410949814c477fa7e6 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 13 Apr 2023 23:24:03 -0700
Subject: [PATCH 023/150] make: Export VERSION and COMMIT

These will be consumed by kata-ctl, so export these so that
they can be used to replace variables available to the rust binary.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/Makefile              | 8 +++++---
 src/tools/kata-ctl/src/ops/version.rs.in | 2 ++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/tools/kata-ctl/Makefile b/src/tools/kata-ctl/Makefile
index 60abae34b..fbbd5c112 100644
--- a/src/tools/kata-ctl/Makefile
+++ b/src/tools/kata-ctl/Makefile
@@ -12,10 +12,10 @@ PROJECT_COMPONENT = kata-ctl
 TARGET = $(PROJECT_COMPONENT)
 
 VERSION_FILE := ./VERSION
-VERSION := $(shell grep -v ^\# $(VERSION_FILE))
+export VERSION := $(shell grep -v ^\# $(VERSION_FILE))
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
 COMMIT_NO_SHORT := $(shell git rev-parse --short HEAD 2>/dev/null || true)
-COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO})
+export COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO})
 
 # Exported to allow cargo to see it
 export KATA_CTL_VERSION := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
@@ -23,7 +23,9 @@ export KATA_CTL_VERSION := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 GENERATED_CODE = src/ops/version.rs
 
 GENERATED_REPLACEMENTS= \
-    KATA_CTL_VERSION
+    KATA_CTL_VERSION \
+    VERSION \
+    COMMIT
 
 GENERATED_FILES := $(GENERATED_CODE)
 
diff --git a/src/tools/kata-ctl/src/ops/version.rs.in b/src/tools/kata-ctl/src/ops/version.rs.in
index 052eccf16..e4a5db3c9 100644
--- a/src/tools/kata-ctl/src/ops/version.rs.in
+++ b/src/tools/kata-ctl/src/ops/version.rs.in
@@ -10,6 +10,8 @@
 use clap::crate_version;
 
 const KATA_CTL_VERSION: &str = "@KATA_CTL_VERSION@";
+pub const VERSION: &str = "@VERSION@";
+pub const COMMIT: &str = "@COMMIT@";
 
 pub fn get() -> Result<String, String> {
     if KATA_CTL_VERSION.trim().is_empty() {

From f2ebdd81c2a10e85a211bfefdc7097a99f8362f4 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 13 Apr 2023 23:26:05 -0700
Subject: [PATCH 024/150] utils: Get rid of spurious print statement left
 behind.

The print was used for debugging, get ris of it.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/utils.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/tools/kata-ctl/src/utils.rs b/src/tools/kata-ctl/src/utils.rs
index 524821aed..f05f3bb10 100644
--- a/src/tools/kata-ctl/src/utils.rs
+++ b/src/tools/kata-ctl/src/utils.rs
@@ -110,7 +110,6 @@ pub fn get_distro_details(os_release: &str, os_release_clr: &str) -> Result<(Str
 pub fn get_generic_cpu_details(cpu_info_file: &str) -> Result<(String, String)> {
     let cpu_info = get_single_cpu_info(cpu_info_file, "\n\n")?;
     let lines = cpu_info.lines();
-    println!("Single cpu info: {}", cpu_info);
     let mut vendor = String::new();
     let mut model = String::new();
 

From 0fec2e6986f1e532f8e31f32b0ca90f8cef95322 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 27 Apr 2023 09:20:30 +0000
Subject: [PATCH 025/150] gpu: Add cold-plug test

Cold plug setting is now correctly decoded in toml

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/cmd/kata-runtime/kata-env_test.go  | 18 ++++++++++++------
 .../pkg/hypervisors/hypervisor_state.go        | 14 ++++++++++++++
 src/runtime/pkg/katatestutils/utils.go         |  2 +-
 .../virtcontainers/persist/api/config.go       |  2 +-
 src/runtime/virtcontainers/qemu.go             |  3 +--
 5 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go
index 321bc507b..3760104d0 100644
--- a/src/runtime/cmd/kata-runtime/kata-env_test.go
+++ b/src/runtime/cmd/kata-runtime/kata-env_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/BurntSushi/toml"
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -74,6 +75,7 @@ func createConfig(configPath string, fileData string) error {
 }
 
 func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) {
+	var coldPlugVFIO hv.PCIePort
 	const logPath = "/log/path"
 	hypervisorPath := filepath.Join(prefixDir, "hypervisor")
 	kernelPath := filepath.Join(prefixDir, "kernel")
@@ -86,6 +88,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
+	coldPlugVFIO = hv.NoPort
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := filepath.Join(prefixDir, "virtiofsd")
@@ -129,6 +132,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		ColdPlugVFIO:         coldPlugVFIO,
 		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs,
@@ -191,12 +195,13 @@ func genericGetExpectedHostDetails(tmpdir string, expectedVendor string, expecte
 
 	expectedSupportVSocks, _ := vcUtils.SupportsVsocks()
 	expectedHostDetails := HostInfo{
-		Kernel:             expectedKernelVersion,
-		Architecture:       expectedArch,
-		Distro:             expectedDistro,
-		CPU:                expectedCPU,
-		VMContainerCapable: expectedVMContainerCapable,
-		SupportVSocks:      expectedSupportVSocks,
+		AvailableGuestProtections: vc.AvailableGuestProtections(),
+		Kernel:                    expectedKernelVersion,
+		Architecture:              expectedArch,
+		Distro:                    expectedDistro,
+		CPU:                       expectedCPU,
+		VMContainerCapable:        expectedVMContainerCapable,
+		SupportVSocks:             expectedSupportVSocks,
 	}
 
 	testProcCPUInfo := filepath.Join(tmpdir, "cpuinfo")
@@ -273,6 +278,7 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
+		ColdPlugVFIO:         config.HypervisorConfig.ColdPlugVFIO,
 	}
 
 	if os.Geteuid() == 0 {
diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go
index 6a9fd7af2..c241dd675 100644
--- a/src/runtime/pkg/hypervisors/hypervisor_state.go
+++ b/src/runtime/pkg/hypervisors/hypervisor_state.go
@@ -5,6 +5,8 @@
 
 package hypervisors
 
+import "fmt"
+
 // Bridge is a bridge where devices can be hot plugged
 type Bridge struct {
 	// DeviceAddr contains information about devices plugged and its address in the bridge
@@ -40,6 +42,18 @@ const (
 	NoPort = "no-port"
 )
 
+func (p PCIePort) String() string {
+	switch p {
+	case RootPort:
+		return "root-port"
+	case SwitchPort:
+		return "switch-port"
+	case NoPort:
+		return "no-port"
+	}
+	return fmt.Sprintf("unknown PCIePort: %s", string(p))
+}
+
 type HypervisorState struct {
 	BlockIndexMap map[int]struct{}
 
diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go
index bec0ed70d..4c8257a40 100644
--- a/src/runtime/pkg/katatestutils/utils.go
+++ b/src/runtime/pkg/katatestutils/utils.go
@@ -319,7 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
 	hotplug_vfio_on_root_bus =  ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
 	pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `
-	cold_plug_vfio =  "` + config.ColdPlugVFIO + `"
+	cold_plug_vfio =  "` + config.ColdPlugVFIO.String() + `"
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"
diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go
index 71533d651..e4facc6b9 100644
--- a/src/runtime/virtcontainers/persist/api/config.go
+++ b/src/runtime/virtcontainers/persist/api/config.go
@@ -199,7 +199,7 @@ type HypervisorConfig struct {
 	// root bus instead of a bridge.
 	HotplugVFIOOnRootBus bool
 
-	// ColdPlugVFIO is used to indicate if devices need to be coldlugged on the
+	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root port or a switch or no-port
 	ColdPlugVFIO hv.PCIePort
 
diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go
index 49b0d6abd..3a8c55cd3 100644
--- a/src/runtime/virtcontainers/qemu.go
+++ b/src/runtime/virtcontainers/qemu.go
@@ -710,8 +710,6 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort, memSize32bit, memSize64bit)
 	}
 
-	q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath)
-
 	// The default OVMF MMIO aperture is too small for some PCIe devices
 	// with huge BARs so we need to increase it.
 	// memSize64bit is in bytes, convert to MB, OVMF expects MB as a string
@@ -726,6 +724,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 
 	q.qemuConfig = qemuConfig
 
+	q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath)
 	return err
 }
 

From f7ad75cb12cfd3491b7dd8518607396235fa22c1 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 27 Apr 2023 09:35:05 +0000
Subject: [PATCH 026/150] gpu: Cold-plug extend the api.md

Make the hypervisorconfig consistent in code and api.md

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/virtcontainers/documentation/api/1.0/api.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md
index 9255a96ca..d3071a86f 100644
--- a/src/runtime/virtcontainers/documentation/api/1.0/api.md
+++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md
@@ -292,6 +292,10 @@ type HypervisorConfig struct {
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
+	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
+	// root port, switch or no port
+	ColdPlugVFIO hv.PCIePort
+
 	// BootToBeTemplate used to indicate if the VM is created to be a template VM
 	BootToBeTemplate bool
 

From defb64334694cfb0649b7c121592e2553d482976 Mon Sep 17 00:00:00 2001
From: Amulyam24 <amulmek1@in.ibm.com>
Date: Thu, 27 Apr 2023 14:56:11 +0530
Subject: [PATCH 027/150] runtime: remove overriding ARCH value by default for
 ppc64le

Currently, ARCH value is being set to powerpc64le by default.
powerpc64le is only right in context of rust and any operation
which might use this variable for a different purpose would fail on ppc64le.

Fixes: #6741

Signed-off-by: Amulyam24 <amulmek1@in.ibm.com>
---
 src/agent/Makefile                 | 8 ++++++--
 src/runtime-rs/Makefile            | 4 ++++
 src/tools/agent-ctl/Makefile       | 4 ++++
 src/tools/runk/Makefile            | 4 ++++
 src/tools/trace-forwarder/Makefile | 4 ++++
 utils.mk                           | 1 -
 6 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/agent/Makefile b/src/agent/Makefile
index 51da58d3a..69423edda 100644
--- a/src/agent/Makefile
+++ b/src/agent/Makefile
@@ -33,6 +33,12 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+include ../../utils.mk
+
+ifeq ($(ARCH), ppc64le)
+    override ARCH = powerpc64le
+endif
+
 ##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
 STANDARD_OCI_RUNTIME := no
 
@@ -45,8 +51,6 @@ ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif
 
-include ../../utils.mk
-
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile
index 599949bb0..afe974bf9 100644
--- a/src/runtime-rs/Makefile
+++ b/src/runtime-rs/Makefile
@@ -17,6 +17,10 @@ CONTAINERD_RUNTIME_NAME = io.containerd.kata.v2
 
 include ../../utils.mk
 
+ifeq ($(ARCH), ppc64le)
+    override ARCH = powerpc64le
+endif
+
 ARCH_DIR = arch
 ARCH_FILE_SUFFIX = -options.mk
 ARCH_FILE = $(ARCH_DIR)/$(ARCH)$(ARCH_FILE_SUFFIX)
diff --git a/src/tools/agent-ctl/Makefile b/src/tools/agent-ctl/Makefile
index 980e47ccb..fe56fa098 100644
--- a/src/tools/agent-ctl/Makefile
+++ b/src/tools/agent-ctl/Makefile
@@ -5,6 +5,10 @@
 
 include ../../../utils.mk
 
+ifeq ($(ARCH), ppc64le)
+     override ARCH = powerpc64le
+ endif
+
 .DEFAULT_GOAL := default
 default: build
 
diff --git a/src/tools/runk/Makefile b/src/tools/runk/Makefile
index d025b6a89..cd9a24a8b 100644
--- a/src/tools/runk/Makefile
+++ b/src/tools/runk/Makefile
@@ -8,6 +8,10 @@ LIBC ?= gnu
 
 include ../../../utils.mk
 
+ifeq ($(ARCH), ppc64le)
+     override ARCH = powerpc64le
+ endif
+
 TARGET = runk
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 AGENT_SOURCE_PATH = ../../agent
diff --git a/src/tools/trace-forwarder/Makefile b/src/tools/trace-forwarder/Makefile
index 5a529e211..3bdd5d53a 100644
--- a/src/tools/trace-forwarder/Makefile
+++ b/src/tools/trace-forwarder/Makefile
@@ -5,6 +5,10 @@
 
 include ../../../utils.mk
 
+ifeq ($(ARCH), ppc64le)
+     override ARCH = powerpc64le
+ endif
+
 .DEFAULT_GOAL := default
 default: build
 
diff --git a/utils.mk b/utils.mk
index 27768809c..f382990dc 100644
--- a/utils.mk
+++ b/utils.mk
@@ -146,7 +146,6 @@ ifneq ($(LIBC),musl)
 endif
 
 ifeq ($(ARCH), ppc64le)
-    override ARCH = powerpc64le
     override LIBC = gnu
     $(warning "WARNING: powerpc64le-unknown-linux-musl target is unavailable")
 endif

From 138ada049c7a4d70b6e8a02f90ed489b73d40491 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 27 Apr 2023 09:59:47 +0000
Subject: [PATCH 028/150] gpu: Cold Plug VFIO toml setting

Added the cold_plug_vfio setting to the qemu-toml.in with some
epxlanation

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 src/runtime/config/configuration-qemu.toml.in |  5 +++++
 src/runtime/pkg/katautils/config_test.go      | 11 +++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in
index 6446b0d0d..f58f87141 100644
--- a/src/runtime/config/configuration-qemu.toml.in
+++ b/src/runtime/config/configuration-qemu.toml.in
@@ -352,6 +352,11 @@ pflashes = []
 # Default false
 #hotplug_vfio_on_root_bus = true
 
+# In a confidential compute environment hot-plugging can compromise
+# security. Enable cold-plugging of VFIO devices to a root-port.
+# The default setting is  "no-port", which means disabled. 
+#cold_plug_vfio = "root-port" 
+
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go
index 80268f911..171f011b8 100644
--- a/src/runtime/pkg/katautils/config_test.go
+++ b/src/runtime/pkg/katautils/config_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@@ -70,7 +71,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	if hypervisor == "" {
 		return config, fmt.Errorf("BUG: need hypervisor")
 	}
-
+	var coldPlugVFIO hv.PCIePort
 	hypervisorPath := path.Join(dir, "hypervisor")
 	kernelPath := path.Join(dir, "kernel")
 	kernelParams := "foo=bar xyz"
@@ -85,6 +86,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
+	coldPlugVFIO = hv.RootPort
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
@@ -107,6 +109,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		PCIeRootPort:         pcieRootPort,
+		ColdPlugVFIO:         coldPlugVFIO,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     defaultVCPUCount,
 		DefaultMaxVCPUCount:  defaultMaxVCPUCount,
@@ -170,6 +173,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		EnableIOThreads:       enableIOThreads,
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		PCIeRootPort:          pcieRootPort,
+		ColdPlugVFIO:          coldPlugVFIO,
 		Msize9p:               defaultMsize9p,
 		MemSlots:              defaultMemSlots,
 		EntropySource:         defaultEntropySource,
@@ -564,6 +568,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		VirtioFSCache:         defaultVirtioFSCacheMode,
 		BlockDeviceAIO:        defaultBlockDeviceAIO,
 		DisableGuestSeLinux:   defaultDisableGuestSeLinux,
+		ColdPlugVFIO:          defaultColdPlugVFIO,
 	}
 
 	expectedAgentConfig := vc.KataAgentConfig{
@@ -597,7 +602,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 
 func TestNewQemuHypervisorConfig(t *testing.T) {
 	dir := t.TempDir()
-
+	var coldPlugVFIO hv.PCIePort
 	hypervisorPath := path.Join(dir, "hypervisor")
 	kernelPath := path.Join(dir, "kernel")
 	imagePath := path.Join(dir, "image")
@@ -606,6 +611,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
+	coldPlugVFIO = hv.RootPort
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
 	blockDeviceAIO := "io_uring"
 	defer func() {
@@ -625,6 +631,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		EnableIOThreads:       enableIOThreads,
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		PCIeRootPort:          pcieRootPort,
+		ColdPlugVFIO:          coldPlugVFIO,
 		RxRateLimiterMaxRate:  rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:  txRateLimiterMaxRate,
 		SharedFS:              "virtio-fs",

From 0d49ceee0b8936252a1cf413e49c52469fcdc2f2 Mon Sep 17 00:00:00 2001
From: Tamas K Lengyel <tamas.lengyel@intel.com>
Date: Thu, 27 Apr 2023 18:36:17 +0000
Subject: [PATCH 029/150] gha: Fix snap creation workflow warnings

Fix recurring issues of failing to install dependencies due to stale apt cache.
Uprev actions/checkout to v3 to resolve issue "Node.js 12 actions are deprecated."

Fixes: #5659
Signed-off-by: Tamas K Lengyel <tamas.lengyel@intel.com>
---
 .github/workflows/snap-release.yaml | 3 ++-
 .github/workflows/snap.yaml         | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/snap-release.yaml b/.github/workflows/snap-release.yaml
index 0d41197a0..4a12ce8d7 100644
--- a/.github/workflows/snap-release.yaml
+++ b/.github/workflows/snap-release.yaml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
@@ -32,6 +32,7 @@ jobs:
         run: |
           # Removing man-db, workflow kept failing, fixes: #4480
           sudo apt -y remove --purge man-db
+          sudo apt-get update
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
           latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml
index 2f1495824..ac163fa2e 100644
--- a/.github/workflows/snap.yaml
+++ b/.github/workflows/snap.yaml
@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Check out
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 

From 6bf1fc6051600befbb98f1ae42a970da96ae8d7d Mon Sep 17 00:00:00 2001
From: Eduardo Berrocal <eduardo.berrocal@intel.com>
Date: Thu, 27 Apr 2023 13:08:35 -0700
Subject: [PATCH 030/150] virtcontainers/factory: Improved test coverage

Expanded tests on factory_test.go to cover more lines of code. Coverage went from 34% to 41.5% in the case of user-mode run tests,
and from 77.7% to 84% in the case of priviledge-mode run tests.

Fixes: #260

Signed-off-by: Eduardo Berrocal <eduardo.berrocal@intel.com>
---
 .../virtcontainers/factory/factory_test.go    | 124 ++++++++++++++++++
 1 file changed, 124 insertions(+)

diff --git a/src/runtime/virtcontainers/factory/factory_test.go b/src/runtime/virtcontainers/factory/factory_test.go
index 80cff101d..e7c40046b 100644
--- a/src/runtime/virtcontainers/factory/factory_test.go
+++ b/src/runtime/virtcontainers/factory/factory_test.go
@@ -8,6 +8,7 @@ package factory
 import (
 	"context"
 	"os"
+	"strings"
 	"testing"
 
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@@ -331,3 +332,126 @@ func TestDeepCompare(t *testing.T) {
 	assert.Nil(err)
 	assert.False(utils.DeepCompare(f1, f2))
 }
+
+func TestFactoryConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	// Valid config
+	var config Config
+	config.VMConfig.HypervisorConfig = vc.HypervisorConfig{
+		KernelPath: "foo",
+		ImagePath:  "bar",
+	}
+	ctx := context.Background()
+	vf, err := NewFactory(ctx, config, false)
+	assert.Nil(err)
+
+	f, ok := vf.(*factory)
+	assert.True(ok)
+
+	vmc := f.Config()
+
+	assert.Equal(config.VMConfig.HypervisorConfig.KernelPath, vmc.HypervisorConfig.KernelPath)
+	assert.Equal(config.VMConfig.HypervisorConfig.ImagePath, vmc.HypervisorConfig.ImagePath)
+}
+
+func TestFactoryGetBaseVM(t *testing.T) {
+	assert := assert.New(t)
+
+	// Set configs
+	var config Config
+	testDir := t.TempDir()
+
+	hyperConfig := vc.HypervisorConfig{
+		KernelPath: testDir,
+		ImagePath:  testDir,
+	}
+	vmConfig := vc.VMConfig{
+		HypervisorType:   vc.MockHypervisor,
+		HypervisorConfig: hyperConfig,
+	}
+	config.VMConfig = vmConfig
+	config.TemplatePath = testDir
+
+	err := vmConfig.Valid()
+	assert.Nil(err)
+
+	ctx := context.Background()
+
+	url, err := mock.GenerateKataMockHybridVSock()
+	assert.NoError(err)
+	defer mock.RemoveKataMockHybridVSock(url)
+	vc.MockHybridVSockPath = url
+
+	hybridVSockTTRPCMock := mock.HybridVSockTTRPCMock{}
+	err = hybridVSockTTRPCMock.Start(url)
+	assert.NoError(err)
+	defer hybridVSockTTRPCMock.Stop()
+
+	// New factory
+	vf, err := NewFactory(ctx, config, false)
+	assert.Nil(err)
+
+	f, ok := vf.(*factory)
+	assert.True(ok)
+
+	// Check VM Config
+	assert.Equal(f.Config(), vmConfig)
+
+	// GetBaseVM
+	vm, err := f.GetBaseVM(ctx, vmConfig)
+	assert.Nil(err)
+
+	// Get VM Status
+	defer func() {
+		r := recover()
+		assert.NotNil(r)
+
+		// Close
+		err = vm.Stop(ctx)
+		assert.Nil(err)
+	}()
+	vmStatus := f.GetVMStatus()
+	assert.NotNil(vmStatus) // line of code to make golang happy. This is never executed.
+}
+
+func TestNewFactoryWithCache(t *testing.T) {
+	assert := assert.New(t)
+
+	// Config
+	var config Config
+	config.VMConfig.HypervisorConfig = vc.HypervisorConfig{
+		KernelPath: "foo",
+		ImagePath:  "bar",
+	}
+	ctx := context.Background()
+
+	// cache>0 and fetch only should throw error
+	config.Cache = 1
+	vf, err := NewFactory(ctx, config, true)
+
+	assert.Nil(vf)
+	assert.Error(err)
+	b := err.Error()
+	assert.True(strings.Contains(b, "cache factory does not support fetch"))
+}
+
+func TestNewFactoryWrongCacheEndpoint(t *testing.T) {
+	assert := assert.New(t)
+
+	// Config
+	var config Config
+	config.VMConfig.HypervisorConfig = vc.HypervisorConfig{
+		KernelPath: "foo",
+		ImagePath:  "bar",
+	}
+	ctx := context.Background()
+
+	config.VMCache = true
+	vf, err := NewFactory(ctx, config, false)
+
+	assert.Nil(vf)
+	assert.Error(err)
+	b := err.Error()
+	assert.True(strings.Contains(b, "rpc error")) // sanity check
+}

From c849bdb0a5454e48135ad3e8a111c15b381cdd7b Mon Sep 17 00:00:00 2001
From: Ryan Savino <ryan.savino@amd.com>
Date: Mon, 24 Apr 2023 11:36:35 -0500
Subject: [PATCH 031/150] gha: Also run k8s tests on qemu-sev

Added the k8s tests for qemu-sev

Fixes: #6711

Signed-Off-By: Ryan Savino <ryan.savino@amd.com>
---
 .github/workflows/ci-on-push.yaml           |  8 +++
 .github/workflows/run-k8s-tests-on-sev.yaml | 65 +++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 .github/workflows/run-k8s-tests-on-sev.yaml

diff --git a/.github/workflows/ci-on-push.yaml b/.github/workflows/ci-on-push.yaml
index 6db1cda72..8b1b6ef36 100644
--- a/.github/workflows/ci-on-push.yaml
+++ b/.github/workflows/ci-on-push.yaml
@@ -29,6 +29,14 @@ jobs:
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
     secrets: inherit
 
+  run-k8s-tests-on-sev:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+
   run-k8s-tests-on-tdx:
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml
new file mode 100644
index 000000000..043b21cc1
--- /dev/null
+++ b/.github/workflows/run-k8s-tests-on-sev.yaml
@@ -0,0 +1,65 @@
+name: CI | Run kubernetes tests on SEV
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      KUBECONFIG: /home/kata/.kube/config
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Deploy kata-deploy
+        run: |
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+
+          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: |
+          pushd tests/integration/kubernetes
+          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
+          bash run_kubernetes_tests.sh
+          popd
+        env:
+          KATA_HYPERVISOR: ${{ matrix.vmm }}
+
+      - name: Delete kata-deploy
+        if: always()
+        run: |
+          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          sleep 180s
+
+          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

From f2b2621dec931921a5d4dfb5f860a09018b88fb7 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Thu, 13 Apr 2023 23:34:32 -0700
Subject: [PATCH 032/150] kata-env: Implement the kata-env command.

Command implements functionality to get user environment settings.

Fixes: #5339

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/Cargo.lock           |  12 +
 src/tools/kata-ctl/Cargo.toml           |   2 +
 src/tools/kata-ctl/src/args.rs          |   8 +-
 src/tools/kata-ctl/src/main.rs          |   5 +-
 src/tools/kata-ctl/src/ops.rs           |   1 +
 src/tools/kata-ctl/src/ops/check_ops.rs |   4 -
 src/tools/kata-ctl/src/ops/env_ops.rs   | 431 ++++++++++++++++++++++++
 src/tools/kata-ctl/src/utils.rs         |   7 +-
 8 files changed, 462 insertions(+), 8 deletions(-)
 create mode 100644 src/tools/kata-ctl/src/ops/env_ops.rs

diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock
index e31c8622d..333470cc5 100644
--- a/src/tools/kata-ctl/Cargo.lock
+++ b/src/tools/kata-ctl/Cargo.lock
@@ -742,10 +742,12 @@ dependencies = [
  "slog-scope",
  "strum",
  "strum_macros",
+ "sys-info",
  "tempfile",
  "test-utils",
  "thiserror",
  "tokio",
+ "toml",
  "url",
  "vmm-sys-util",
 ]
@@ -1653,6 +1655,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "sys-info"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml
index d2fc5fb1e..642c9a8a8 100644
--- a/src/tools/kata-ctl/Cargo.toml
+++ b/src/tools/kata-ctl/Cargo.toml
@@ -25,6 +25,8 @@ serde = { version = "1.0.149", features = ["derive"] }
 url = "2.3.1"
 futures = "0.3.24"
 base64 = "0.13.0"
+toml = "0.5.8"
+sys-info = "0.9.1"
 
 shim-interface = { path = "../../libs/shim-interface"}
 kata-types = { path = "../../libs/kata-types" }
diff --git a/src/tools/kata-ctl/src/args.rs b/src/tools/kata-ctl/src/args.rs
index ff489f82b..84fac00a0 100644
--- a/src/tools/kata-ctl/src/args.rs
+++ b/src/tools/kata-ctl/src/args.rs
@@ -23,7 +23,7 @@ pub enum Commands {
     DirectVolume(DirectVolumeCommand),
 
     /// Display settings
-    Env,
+    Env(EnvArgument),
 
     /// Enter into guest VM by debug console
     Exec(ExecArguments),
@@ -69,6 +69,12 @@ pub enum CheckSubCommand {
     List,
 }
 
+#[derive(Debug, Args)]
+pub struct EnvArgument {
+    /// Format output as JSON
+    #[arg(long)]
+    pub json: bool,
+}
 #[derive(Debug, Args)]
 pub struct MetricsCommand {
     #[clap(subcommand)]
diff --git a/src/tools/kata-ctl/src/main.rs b/src/tools/kata-ctl/src/main.rs
index 980a94771..fe64f10e1 100644
--- a/src/tools/kata-ctl/src/main.rs
+++ b/src/tools/kata-ctl/src/main.rs
@@ -17,8 +17,9 @@ use std::process::exit;
 use args::{Commands, KataCtlCli};
 
 use ops::check_ops::{
-    handle_check, handle_env, handle_factory, handle_iptables, handle_metrics, handle_version,
+    handle_check, handle_factory, handle_iptables, handle_metrics, handle_version,
 };
+use ops::env_ops::handle_env;
 use ops::exec_ops::handle_exec;
 use ops::volume_ops::handle_direct_volume;
 
@@ -29,7 +30,7 @@ fn real_main() -> Result<()> {
         Commands::Check(args) => handle_check(args),
         Commands::DirectVolume(args) => handle_direct_volume(args),
         Commands::Exec(args) => handle_exec(args),
-        Commands::Env => handle_env(),
+        Commands::Env(args) => handle_env(args),
         Commands::Factory => handle_factory(),
         Commands::Iptables(args) => handle_iptables(args),
         Commands::Metrics(args) => handle_metrics(args),
diff --git a/src/tools/kata-ctl/src/ops.rs b/src/tools/kata-ctl/src/ops.rs
index d5d4fe162..f90f55cb2 100644
--- a/src/tools/kata-ctl/src/ops.rs
+++ b/src/tools/kata-ctl/src/ops.rs
@@ -4,6 +4,7 @@
 //
 
 pub mod check_ops;
+pub mod env_ops;
 pub mod exec_ops;
 pub mod version;
 pub mod volume_ops;
diff --git a/src/tools/kata-ctl/src/ops/check_ops.rs b/src/tools/kata-ctl/src/ops/check_ops.rs
index f2dbea702..298e214c5 100644
--- a/src/tools/kata-ctl/src/ops/check_ops.rs
+++ b/src/tools/kata-ctl/src/ops/check_ops.rs
@@ -107,10 +107,6 @@ pub fn handle_check(checkcmd: CheckArgument) -> Result<()> {
     Ok(())
 }
 
-pub fn handle_env() -> Result<()> {
-    Ok(())
-}
-
 pub fn handle_factory() -> Result<()> {
     Ok(())
 }
diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs
new file mode 100644
index 000000000..894cc7020
--- /dev/null
+++ b/src/tools/kata-ctl/src/ops/env_ops.rs
@@ -0,0 +1,431 @@
+// Copyright (c) 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Contains checks that are not architecture-specific
+
+use crate::arch::arch_specific;
+use crate::args::EnvArgument;
+use crate::check;
+use crate::ops::version;
+use crate::utils;
+use kata_types::config::TomlConfig;
+
+use anyhow::{anyhow, Context, Result};
+use serde::{Deserialize, Serialize};
+use std::process::Command;
+use sys_info;
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct HostInfo {
+    #[serde(default)]
+    available_guest_protection: String,
+    #[serde(default)]
+    kernel: String,
+    #[serde(default)]
+    architecture: String,
+    #[serde(default)]
+    vm_container_capable: bool,
+    #[serde(default)]
+    support_vsocks: bool,
+    #[serde(default)]
+    distro: DistroInfo,
+    #[serde(default)]
+    cpu: CPUInfo,
+    #[serde(default)]
+    memory: MemoryInfo,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct DistroInfo {
+    #[serde(default)]
+    name: String,
+    #[serde(default)]
+    version: String,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct CPUInfo {
+    #[serde(default)]
+    vendor: String,
+    #[serde(default)]
+    model: String,
+    #[serde(default)]
+    cpus: usize,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct MemoryInfo {
+    #[serde(default)]
+    total: u64,
+    #[serde(default)]
+    available: u64,
+    #[serde(default)]
+    free: u64,
+}
+
+// Semantic version for the output of the command.
+//
+// XXX: Increment for every change to the output format
+// (meaning any change to the EnvInfo type).
+const FORMAT_VERSION: &str = "0.0.1-kata-ctl";
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct MetaInfo {
+    #[serde(default)]
+    version: String,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct VersionInfo {
+    #[serde(default)]
+    semver: String,
+    #[serde(default)]
+    commit: String,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct RuntimeConfigInfo {
+    #[serde(default)]
+    path: String,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct RuntimeInfo {
+    #[serde(default)]
+    path: String,
+    #[serde(default)]
+    guest_selinux_label: String,
+    #[serde(default)]
+    pub experimental: Vec<String>,
+    #[serde(default)]
+    debug: bool,
+    #[serde(default)]
+    trace: bool,
+    #[serde(default)]
+    disable_guest_seccomp: bool,
+    #[serde(default)]
+    disable_new_net_ns: bool,
+    #[serde(default)]
+    sandbox_cgroup_only: bool,
+    #[serde(default)]
+    static_sandbox_resource_mgmt: bool,
+    #[serde(default)]
+    config: RuntimeConfigInfo,
+    #[serde(default)]
+    version: VersionInfo,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct AgentInfo {
+    #[serde(default)]
+    debug: bool,
+    #[serde(default)]
+    trace: bool,
+}
+// KernelInfo stores kernel details
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct KernelInfo {
+    #[serde(default)]
+    path: String,
+    #[serde(default)]
+    parameters: String,
+}
+
+// InitrdInfo stores initrd image details
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct InitrdInfo {
+    #[serde(default)]
+    path: String,
+}
+
+// ImageInfo stores root filesystem image details
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct ImageInfo {
+    #[serde(default)]
+    path: String,
+}
+
+// HypervisorInfo stores hypervisor details
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct HypervisorInfo {
+    #[serde(default)]
+    machine_type: String,
+    #[serde(default)]
+    machine_accelerators: String,
+    #[serde(default)]
+    version: String,
+    #[serde(default)]
+    path: String,
+    #[serde(default)]
+    block_device_driver: String,
+    #[serde(default)]
+    entropy_source: String,
+    #[serde(default)]
+    shared_fs: String,
+    #[serde(default)]
+    virtio_fs_daemon: String,
+    #[serde(default)]
+    msize_9p: u32,
+    #[serde(default)]
+    memory_slots: u32,
+    #[serde(default)]
+    pcie_root_port: u32,
+    #[serde(default)]
+    hotplug_vfio_on_rootbus: bool,
+    #[serde(default)]
+    debug: bool,
+    #[serde(default)]
+    enable_iommu: bool,
+    #[serde(default)]
+    enable_iommu_platform: bool,
+    #[serde(default)]
+    default_vcpus: i32,
+    #[serde(default)]
+    cpu_features: String,
+}
+
+// EnvInfo collects all information that will be displayed by the
+// env command.
+//
+// XXX: Any changes must be coupled with a change to formatVersion.
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct EnvInfo {
+    #[serde(default)]
+    kernel: KernelInfo,
+    #[serde(default)]
+    meta: MetaInfo,
+    #[serde(default)]
+    image: ImageInfo,
+    #[serde(default)]
+    initrd: InitrdInfo,
+    #[serde(default)]
+    hypervisor: HypervisorInfo,
+    #[serde(default)]
+    runtime: RuntimeInfo,
+    #[serde(default)]
+    host: HostInfo,
+    #[serde(default)]
+    agent: AgentInfo,
+}
+
+pub fn get_meta_info() -> MetaInfo {
+    MetaInfo {
+        version: String::from(FORMAT_VERSION),
+    }
+}
+
+pub fn get_memory_info() -> Result<MemoryInfo> {
+    let mem_info = sys_info::mem_info().context("get host memory information")?;
+    Ok(MemoryInfo {
+        total: mem_info.total,
+        available: mem_info.avail,
+        free: mem_info.free,
+    })
+}
+
+fn get_host_info() -> Result<HostInfo> {
+    let host_kernel_version = utils::get_kernel_version(utils::PROC_VERSION_FILE)?;
+    let (host_distro_name, host_distro_version) =
+        utils::get_distro_details(utils::OS_RELEASE, utils::OS_RELEASE_CLR)?;
+    let (cpu_vendor, cpu_model) = utils::get_generic_cpu_details(check::PROC_CPUINFO)?;
+
+    let host_distro = DistroInfo {
+        name: host_distro_name,
+        version: host_distro_version,
+    };
+
+    let cores: usize = std::thread::available_parallelism()
+        .context("get available parallelism")?
+        .into();
+
+    let host_cpu = CPUInfo {
+        vendor: cpu_vendor,
+        model: cpu_model,
+        cpus: cores,
+    };
+
+    let memory_info = get_memory_info()?;
+
+    let guest_protection =
+        arch_specific::available_guest_protection().map_err(|e| anyhow!(e.to_string()))?;
+
+    let guest_protection = guest_protection.to_string();
+
+    let support_vsocks = utils::supports_vsocks(utils::VHOST_VSOCK_DEVICE)?;
+
+    Ok(HostInfo {
+        kernel: host_kernel_version,
+        architecture: String::from(std::env::consts::ARCH),
+        distro: host_distro,
+        cpu: host_cpu,
+        memory: memory_info,
+        available_guest_protection: guest_protection,
+        // TODO: See https://github.com/kata-containers/kata-containers/issues/6727
+        vm_container_capable: true,
+        support_vsocks,
+    })
+}
+
+pub fn get_runtime_info(toml_config: &TomlConfig) -> Result<RuntimeInfo> {
+    let version = VersionInfo {
+        semver: String::from(version::VERSION),
+        commit: String::from(version::COMMIT),
+    };
+
+    let config_path = TomlConfig::get_default_config_file();
+    let mut toml_path = String::new();
+    if config_path.is_ok() {
+        let p = config_path?;
+        let path_str = p.to_str();
+        toml_path = match path_str {
+            Some(s) => String::from(s),
+            None => String::new(),
+        };
+    }
+
+    Ok(RuntimeInfo {
+        // TODO: Needs to be implemented: https://github.com/kata-containers/kata-containers/issues/6518
+        path: String::from("not implemented yet. See: https://github.com/kata-containers/kata-containers/issues/6518"),
+        version,
+        experimental: toml_config.runtime.experimental.clone(),
+        // TODO: See https://github.com/kata-containers/kata-containers/issues/6667
+        guest_selinux_label: String::from("not implemented yet: See https://github.com/kata-containers/kata-containers/issues/6667"),
+        debug: toml_config.runtime.debug,
+        trace: toml_config.runtime.enable_tracing,
+        disable_guest_seccomp: toml_config.runtime.disable_guest_seccomp,
+        disable_new_net_ns: toml_config.runtime.disable_new_netns,
+        sandbox_cgroup_only: toml_config.runtime.sandbox_cgroup_only,
+        static_sandbox_resource_mgmt: toml_config.runtime.static_sandbox_resource_mgmt,
+        config: RuntimeConfigInfo { path: toml_path },
+    }
+}
+
+pub fn get_agent_info(toml_config: &TomlConfig) -> Result<AgentInfo> {
+    let agent_config = toml_config
+        .agent
+        .get(&toml_config.runtime.agent_name)
+        .ok_or("could not find agent config in configuration")
+        .map_err(|e| anyhow!(e))?;
+
+    Ok(AgentInfo {
+        debug: agent_config.debug,
+        trace: agent_config.enable_tracing,
+    })
+}
+
+pub fn get_command_version(cmd: &str) -> Result<String> {
+    // Path is empty in case of dragonball hypervisor
+    if cmd.is_empty() {
+        return Ok("unknown".to_string());
+    }
+    let output = Command::new(cmd)
+        .arg("--version")
+        .output()
+        .map_err(|e| anyhow!(e))?;
+
+    let version = String::from_utf8(output.stdout).map_err(|e| anyhow!(e))?;
+
+    Ok(version)
+}
+
+pub fn get_hypervisor_info(
+    toml_config: &TomlConfig,
+) -> Result<(HypervisorInfo, ImageInfo, KernelInfo, InitrdInfo)> {
+    let hypervisor_config = toml_config
+        .hypervisor
+        .get(&toml_config.runtime.hypervisor_name)
+        .ok_or("could not find hypervisor config in configuration")
+        .map_err(|e| anyhow!(e))?;
+
+    let version =
+        get_command_version(&hypervisor_config.path).context("error getting hypervisor version")?;
+
+    let hypervisor_info = HypervisorInfo {
+        machine_type: hypervisor_config.machine_info.machine_type.to_string(),
+        machine_accelerators: hypervisor_config
+            .machine_info
+            .machine_accelerators
+            .to_string(),
+        version,
+        path: hypervisor_config.path.to_string(),
+        block_device_driver: hypervisor_config
+            .blockdev_info
+            .block_device_driver
+            .to_string(),
+        entropy_source: hypervisor_config.machine_info.entropy_source.to_string(),
+        shared_fs: hypervisor_config
+            .shared_fs
+            .shared_fs
+            .clone()
+            .unwrap_or_else(|| String::from("none")),
+        virtio_fs_daemon: hypervisor_config.shared_fs.virtio_fs_daemon.to_string(),
+        msize_9p: hypervisor_config.shared_fs.msize_9p,
+        memory_slots: hypervisor_config.memory_info.memory_slots,
+        pcie_root_port: hypervisor_config.device_info.pcie_root_port,
+        hotplug_vfio_on_rootbus: hypervisor_config.device_info.hotplug_vfio_on_root_bus,
+        debug: hypervisor_config.debug_info.enable_debug,
+        enable_iommu: hypervisor_config.device_info.enable_iommu,
+        enable_iommu_platform: hypervisor_config.device_info.enable_iommu_platform,
+        default_vcpus: hypervisor_config.cpu_info.default_vcpus,
+        cpu_features: hypervisor_config.cpu_info.cpu_features.to_string(),
+    };
+
+    let image_info = ImageInfo {
+        path: hypervisor_config.boot_info.image.clone(),
+    };
+
+    let kernel_info = KernelInfo {
+        path: hypervisor_config.boot_info.kernel.to_string(),
+        parameters: hypervisor_config.boot_info.kernel_params.to_string(),
+    };
+
+    let initrd_info = InitrdInfo {
+        path: hypervisor_config.boot_info.initrd.to_string(),
+    };
+
+    Ok((hypervisor_info, image_info, kernel_info, initrd_info))
+}
+
+pub fn get_env_info(toml_config: &TomlConfig) -> Result<EnvInfo> {
+    let metainfo = get_meta_info();
+
+    let runtime_info = get_runtime_info(toml_config).context("get runtime info")?;
+
+    let agent_info = get_agent_info(toml_config).context("get agent configuration")?;
+
+    let host_info = get_host_info().context("get host information")?;
+
+    let (hypervisor_info, _image_info, kernel_info, initrd_info) =
+        get_hypervisor_info(toml_config).context("get hypervisor configuration")?;
+
+    let env_info = EnvInfo {
+        meta: metainfo,
+        runtime: runtime_info,
+        kernel: kernel_info,
+        image: _image_info,
+        initrd: initrd_info,
+        hypervisor: hypervisor_info,
+        host: host_info,
+        agent: agent_info,
+    };
+
+    Ok(env_info)
+}
+
+pub fn handle_env(env_args: EnvArgument) -> Result<()> {
+    let (toml_config, _) = TomlConfig::load_raw_from_file("").context("load toml config")?;
+
+    let env_info = get_env_info(&toml_config)?;
+
+    if env_args.json {
+        let serialized_json = serde_json::to_string_pretty(&env_info)?;
+        println!("{}", serialized_json);
+    } else {
+        let toml = toml::to_string(&env_info)?;
+        println!("{}", toml);
+    }
+
+    Ok(())
+}
diff --git a/src/tools/kata-ctl/src/utils.rs b/src/tools/kata-ctl/src/utils.rs
index f05f3bb10..371b5165d 100644
--- a/src/tools/kata-ctl/src/utils.rs
+++ b/src/tools/kata-ctl/src/utils.rs
@@ -145,7 +145,12 @@ pub fn get_generic_cpu_details(cpu_info_file: &str) -> Result<(String, String)>
 
 pub const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock";
 pub fn supports_vsocks(vsock_path: &str) -> Result<bool> {
-    let metadata = fs::metadata(vsock_path)?;
+    let metadata = fs::metadata(vsock_path).map_err(|err| {
+        anyhow!(
+            "Host system does not support vhost-vsock (try running (`sudo modprobe vhost_vsock`) : {}",
+            err.to_string()
+        )
+    })?;
     Ok(metadata.is_file())
 }
 

From b1920198bea28e397992750c925273e59683a694 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Sat, 15 Apr 2023 00:46:53 -0700
Subject: [PATCH 033/150] config: Workaround the way agent and hypervisor
 configs are fetched

This is essentially a workaround for the issue:
https://github.com/kata-containers/kata-containers/issues/5954

runtime-rs chnages the Kata config format adding agent_name and
hypervisor_name which are then used as keys to fetch the agent and
hypervisor configs. This will not work for older configs.
So use the first entry in the hashmaps to fetch the configs as a
workaround while the config change issue is resolved.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/ops/env_ops.rs | 46 ++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs
index 894cc7020..e44892efd 100644
--- a/src/tools/kata-ctl/src/ops/env_ops.rs
+++ b/src/tools/kata-ctl/src/ops/env_ops.rs
@@ -299,15 +299,27 @@ pub fn get_runtime_info(toml_config: &TomlConfig) -> Result<RuntimeInfo> {
         sandbox_cgroup_only: toml_config.runtime.sandbox_cgroup_only,
         static_sandbox_resource_mgmt: toml_config.runtime.static_sandbox_resource_mgmt,
         config: RuntimeConfigInfo { path: toml_path },
-    }
+    })
 }
 
 pub fn get_agent_info(toml_config: &TomlConfig) -> Result<AgentInfo> {
-    let agent_config = toml_config
-        .agent
-        .get(&toml_config.runtime.agent_name)
-        .ok_or("could not find agent config in configuration")
-        .map_err(|e| anyhow!(e))?;
+    // Assign the first entry to the agent config, to make this
+    // work for configs where agent_name is absent.
+    // This is a workaround for https://github.com/kata-containers/kata-containers/issues/5954
+    let key_val = toml_config.agent.iter().next();
+    let mut agent_config = match key_val {
+        Some(x) => Ok(x.1),
+        None => Err(anyhow!("Missing agent config")),
+    }?;
+
+    // If the agent_name config is present, use that
+    if !&toml_config.runtime.agent_name.is_empty() {
+        agent_config = toml_config
+            .agent
+            .get(&toml_config.runtime.agent_name)
+            .ok_or("could not find agent config in configuration")
+            .map_err(|e| anyhow!(e))?;
+    }
 
     Ok(AgentInfo {
         debug: agent_config.debug,
@@ -333,11 +345,23 @@ pub fn get_command_version(cmd: &str) -> Result<String> {
 pub fn get_hypervisor_info(
     toml_config: &TomlConfig,
 ) -> Result<(HypervisorInfo, ImageInfo, KernelInfo, InitrdInfo)> {
-    let hypervisor_config = toml_config
-        .hypervisor
-        .get(&toml_config.runtime.hypervisor_name)
-        .ok_or("could not find hypervisor config in configuration")
-        .map_err(|e| anyhow!(e))?;
+    // Assign the first entry in the hashmap to the hypervisor config, to make this
+    // work for configs where hypervisor_name is absent.
+    // This is a workaround for https://github.com/kata-containers/kata-containers/issues/5954
+    let key_val = toml_config.hypervisor.iter().next();
+    let mut hypervisor_config = match key_val {
+        Some(x) => Ok(x.1),
+        None => Err(anyhow!("Missing hypervisor config")),
+    }?;
+
+    // If hypervisor_name config is present, use that
+    if !&toml_config.runtime.hypervisor_name.is_empty() {
+        hypervisor_config = toml_config
+            .hypervisor
+            .get(&toml_config.runtime.hypervisor_name)
+            .ok_or("could not find hypervisor config in configuration")
+            .map_err(|e| anyhow!(e))?;
+    }
 
     let version =
         get_command_version(&hypervisor_config.path).context("error getting hypervisor version")?;

From b908a780a02b26a70bf9d7aa8b409b3672a74d93 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Tue, 25 Apr 2023 14:21:00 -0700
Subject: [PATCH 034/150] kata-env: Pass cmd option for file path

Add ability to write the environment information to a file
or stdout if file path is absent.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/args.rs        |  3 +++
 src/tools/kata-ctl/src/ops/env_ops.rs | 14 ++++++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/tools/kata-ctl/src/args.rs b/src/tools/kata-ctl/src/args.rs
index 84fac00a0..cc2dc0513 100644
--- a/src/tools/kata-ctl/src/args.rs
+++ b/src/tools/kata-ctl/src/args.rs
@@ -74,6 +74,9 @@ pub struct EnvArgument {
     /// Format output as JSON
     #[arg(long)]
     pub json: bool,
+    /// File to write env output to
+    #[arg(short = 'f', long = "file")]
+    pub file: Option<String>,
 }
 #[derive(Debug, Args)]
 pub struct MetricsCommand {
diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs
index e44892efd..7c6327d47 100644
--- a/src/tools/kata-ctl/src/ops/env_ops.rs
+++ b/src/tools/kata-ctl/src/ops/env_ops.rs
@@ -14,6 +14,8 @@ use kata_types::config::TomlConfig;
 
 use anyhow::{anyhow, Context, Result};
 use serde::{Deserialize, Serialize};
+use std::fs::File;
+use std::io::{self, Write};
 use std::process::Command;
 use sys_info;
 
@@ -439,16 +441,24 @@ pub fn get_env_info(toml_config: &TomlConfig) -> Result<EnvInfo> {
 }
 
 pub fn handle_env(env_args: EnvArgument) -> Result<()> {
+    let mut file: Box<dyn Write> = if let Some(path) = env_args.file {
+        Box::new(
+            File::create(path.as_str()).with_context(|| format!("Error creating file {}", path))?,
+        )
+    } else {
+        Box::new(io::stdout())
+    };
+
     let (toml_config, _) = TomlConfig::load_raw_from_file("").context("load toml config")?;
 
     let env_info = get_env_info(&toml_config)?;
 
     if env_args.json {
         let serialized_json = serde_json::to_string_pretty(&env_info)?;
-        println!("{}", serialized_json);
+        write!(file, "{}", serialized_json)?;
     } else {
         let toml = toml::to_string(&env_info)?;
-        println!("{}", toml);
+        write!(file, "{}", toml)?;
     }
 
     Ok(())

From 1016bc17b74e2efceacd4889b5086ea98c94f420 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Tue, 25 Apr 2023 15:14:52 -0700
Subject: [PATCH 035/150] config: Add api to fetch config from default config
 path

Add api to fetch config from default config path and use that in
kata-ctl tool.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/libs/kata-types/src/config/mod.rs | 8 ++++++++
 src/tools/kata-ctl/src/ops/env_ops.rs | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs
index 2a696922a..e73f8b84e 100644
--- a/src/libs/kata-types/src/config/mod.rs
+++ b/src/libs/kata-types/src/config/mod.rs
@@ -127,6 +127,14 @@ impl TomlConfig {
         result
     }
 
+    /// Load raw Kata configuration information from default configuration file.
+    ///
+    /// Configuration file is probed according to the default configuration file list
+    /// default::DEFAULT_RUNTIME_CONFIGURATIONS.
+    pub fn load_from_default() -> Result<(TomlConfig, PathBuf)> {
+        Self::load_raw_from_file("")
+    }
+
     /// Load raw Kata configuration information from configuration files.
     ///
     /// If `config_file` is valid, it will used, otherwise a built-in default path list will be
diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs
index 7c6327d47..a0332af85 100644
--- a/src/tools/kata-ctl/src/ops/env_ops.rs
+++ b/src/tools/kata-ctl/src/ops/env_ops.rs
@@ -449,7 +449,7 @@ pub fn handle_env(env_args: EnvArgument) -> Result<()> {
         Box::new(io::stdout())
     };
 
-    let (toml_config, _) = TomlConfig::load_raw_from_file("").context("load toml config")?;
+    let (toml_config, _) = TomlConfig::load_from_default().context("load toml config")?;
 
     let env_info = get_env_info(&toml_config)?;
 

From fb40c71a2156f91bc13049e35a7fcbcf793541ed Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Wed, 26 Apr 2023 11:43:58 -0700
Subject: [PATCH 036/150] env: Check for root privileges

Check for root privileges early on.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/ops/env_ops.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs
index a0332af85..de8970ad0 100644
--- a/src/tools/kata-ctl/src/ops/env_ops.rs
+++ b/src/tools/kata-ctl/src/ops/env_ops.rs
@@ -13,6 +13,7 @@ use crate::utils;
 use kata_types::config::TomlConfig;
 
 use anyhow::{anyhow, Context, Result};
+use nix::unistd::Uid;
 use serde::{Deserialize, Serialize};
 use std::fs::File;
 use std::io::{self, Write};
@@ -441,6 +442,10 @@ pub fn get_env_info(toml_config: &TomlConfig) -> Result<EnvInfo> {
 }
 
 pub fn handle_env(env_args: EnvArgument) -> Result<()> {
+    if !Uid::effective().is_root() {
+        return Err(anyhow!("kata-ctl env command requires root privileges to get host information. Please run as root or use sudo"));
+    }
+
     let mut file: Box<dyn Write> = if let Some(path) = env_args.file {
         Box::new(
             File::create(path.as_str()).with_context(|| format!("Error creating file {}", path))?,

From 40641928967a95f4fb846d7ff4c508ccb2d4e92e Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Wed, 26 Apr 2023 22:01:03 -0700
Subject: [PATCH 037/150] env: Utilize arch specific functionality to get cpu
 details

Have kata-env call architecture specific function to get cpu details
instead of generic function to get cpu details that works only for
certain architectures. The functionality for cpu details has been fully
implemented for x86_64 and arm architectures, but needs to be
implemented for s390 and powerpc.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/src/arch/aarch64/mod.rs    | 29 +++++++++++++++++++
 .../kata-ctl/src/arch/powerpc64le/mod.rs      |  9 ++++++
 src/tools/kata-ctl/src/arch/s390x/mod.rs      |  9 ++++++
 src/tools/kata-ctl/src/arch/x86_64/mod.rs     |  5 ++++
 src/tools/kata-ctl/src/check.rs               |  2 --
 src/tools/kata-ctl/src/ops/env_ops.rs         |  3 +-
 src/tools/kata-ctl/src/utils.rs               |  4 +--
 7 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/src/tools/kata-ctl/src/arch/aarch64/mod.rs b/src/tools/kata-ctl/src/arch/aarch64/mod.rs
index a6137856f..7eaf3ccb4 100644
--- a/src/tools/kata-ctl/src/arch/aarch64/mod.rs
+++ b/src/tools/kata-ctl/src/arch/aarch64/mod.rs
@@ -9,6 +9,7 @@ pub use arch_specific::*;
 mod arch_specific {
     use crate::check;
     use crate::types::*;
+    use crate::utils;
     use anyhow::Result;
     use std::path::Path;
 
@@ -37,6 +38,34 @@ mod arch_specific {
         Ok(())
     }
 
+    fn normalize_vendor(vendor: &str) -> String {
+        match vendor {
+            "0x41" => String::from("ARM Limited"),
+            _ => String::from("3rd Party Limited"),
+        }
+    }
+
+    fn normalize_model(model: &str) -> String {
+        match model {
+            "8" => String::from("v8"),
+            "7" | "7M" | "?(12)" | "?(13)" | "?(14)" | "?(15)" | "?(16)" | "?(17)" => {
+                String::from("v7")
+            }
+            "6" | "6TEJ" => String::from("v6"),
+            "5" | "5T" | "5TE" | "5TEJ" => String::from("v5"),
+            "4" | "4T" => String::from("v4"),
+            "3" => String::from("v3"),
+            _ => String::from("unknown"),
+        }
+    }
+
+    pub fn get_cpu_details() -> Result<(String, String)> {
+        let (vendor, model) = utils::get_generic_cpu_details(check::PROC_CPUINFO)?;
+        let norm_vendor = normalize_vendor(&vendor);
+        let norm_model = normalize_model(&model);
+        Ok((norm_vendor, norm_model))
+    }
+
     pub fn get_checks() -> Option<&'static [CheckItem<'static>]> {
         Some(CHECK_LIST)
     }
diff --git a/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs b/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs
index 8290dbb13..fc849c631 100644
--- a/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs
+++ b/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs
@@ -9,6 +9,7 @@ pub use arch_specific::*;
 
 mod arch_specific {
     use crate::check;
+    use crate::utils;
     use anyhow::Result;
 
     pub const ARCH_CPU_VENDOR_FIELD: &str = "";
@@ -24,6 +25,14 @@ mod arch_specific {
 
     const PEF_SYS_FIRMWARE_DIR: &str = "/sys/firmware/ultravisor/";
 
+    pub fn get_cpu_details() -> Result<(String, String)> {
+        utils::get_generic_cpu_details(check::PROC_CPUINFO)
+
+        // TODO: In case of error from get_generic_cpu_details, implement functionality
+        // to get cpu details specific to powerpc architecture similar
+        // to the goloang implementation of function getCPUDetails()
+    }
+
     pub fn available_guest_protection() -> Result<check::GuestProtection, check::ProtectionError> {
         if !Uid::effective().is_root() {
             return Err(check::ProtectionError::NoPerms);
diff --git a/src/tools/kata-ctl/src/arch/s390x/mod.rs b/src/tools/kata-ctl/src/arch/s390x/mod.rs
index 7a9940dcf..929e79c92 100644
--- a/src/tools/kata-ctl/src/arch/s390x/mod.rs
+++ b/src/tools/kata-ctl/src/arch/s390x/mod.rs
@@ -10,6 +10,7 @@ pub use arch_specific::*;
 mod arch_specific {
     use crate::check;
     use crate::types::*;
+    use crate::utils;
     use anyhow::{anyhow, Result};
     use nix::unistd::Uid;
     use std::collections::HashMap;
@@ -144,6 +145,14 @@ mod arch_specific {
         Ok(false)
     }
 
+    pub fn get_cpu_details() -> Result<(String, String)> {
+        utils::get_generic_cpu_details(check::PROC_CPUINFO)
+
+        // TODO: In case of error from get_generic_cpu_details, implement functionality
+        // to get cpu details specific to s390x architecture similar
+        // to the goloang implementation of function getS390xCPUDetails()
+    }
+
     #[allow(dead_code)]
     // Guest protection is not supported on ARM64.
     pub fn available_guest_protection() -> Result<check::GuestProtection, check::ProtectionError> {
diff --git a/src/tools/kata-ctl/src/arch/x86_64/mod.rs b/src/tools/kata-ctl/src/arch/x86_64/mod.rs
index 026312624..deafff94f 100644
--- a/src/tools/kata-ctl/src/arch/x86_64/mod.rs
+++ b/src/tools/kata-ctl/src/arch/x86_64/mod.rs
@@ -12,6 +12,7 @@ mod arch_specific {
     use crate::check;
     use crate::check::{GuestProtection, ProtectionError};
     use crate::types::*;
+    use crate::utils;
     use anyhow::{anyhow, Result};
     use nix::unistd::Uid;
     use std::fs;
@@ -109,6 +110,10 @@ mod arch_specific {
         Ok(cpu_flags)
     }
 
+    pub fn get_cpu_details() -> Result<(String, String)> {
+        utils::get_generic_cpu_details(check::PROC_CPUINFO)
+    }
+
     pub const TDX_SYS_FIRMWARE_DIR: &str = "/sys/firmware/tdx_seam/";
     pub const TDX_CPU_FLAG: &str = "tdx";
     pub const SEV_KVM_PARAMETER_PATH: &str = "/sys/module/kvm_amd/parameters/sev";
diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs
index bd5406bb8..8f21f6fab 100644
--- a/src/tools/kata-ctl/src/check.rs
+++ b/src/tools/kata-ctl/src/check.rs
@@ -47,7 +47,6 @@ pub const GENERIC_CPU_MODEL_FIELD: &str = "model name";
 #[allow(dead_code)]
 pub const PROC_CPUINFO: &str = "/proc/cpuinfo";
 
-#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))]
 fn read_file_contents(file_path: &str) -> Result<String> {
     let contents = std::fs::read_to_string(file_path)?;
     Ok(contents)
@@ -55,7 +54,6 @@ fn read_file_contents(file_path: &str) -> Result<String> {
 
 // get_single_cpu_info returns the contents of the first cpu from
 // the specified cpuinfo file by parsing based on a specified delimiter
-#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))]
 pub fn get_single_cpu_info(cpu_info_file: &str, substring: &str) -> Result<String> {
     let contents = read_file_contents(cpu_info_file)?;
 
diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs
index de8970ad0..05602e479 100644
--- a/src/tools/kata-ctl/src/ops/env_ops.rs
+++ b/src/tools/kata-ctl/src/ops/env_ops.rs
@@ -7,7 +7,6 @@
 
 use crate::arch::arch_specific;
 use crate::args::EnvArgument;
-use crate::check;
 use crate::ops::version;
 use crate::utils;
 use kata_types::config::TomlConfig;
@@ -232,7 +231,7 @@ fn get_host_info() -> Result<HostInfo> {
     let host_kernel_version = utils::get_kernel_version(utils::PROC_VERSION_FILE)?;
     let (host_distro_name, host_distro_version) =
         utils::get_distro_details(utils::OS_RELEASE, utils::OS_RELEASE_CLR)?;
-    let (cpu_vendor, cpu_model) = utils::get_generic_cpu_details(check::PROC_CPUINFO)?;
+    let (cpu_vendor, cpu_model) = arch_specific::get_cpu_details()?;
 
     let host_distro = DistroInfo {
         name: host_distro_name,
diff --git a/src/tools/kata-ctl/src/utils.rs b/src/tools/kata-ctl/src/utils.rs
index 371b5165d..a5ab39d08 100644
--- a/src/tools/kata-ctl/src/utils.rs
+++ b/src/tools/kata-ctl/src/utils.rs
@@ -5,9 +5,7 @@
 
 #![allow(dead_code)]
 
-#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))]
 use crate::arch::arch_specific;
-#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))]
 use crate::check::get_single_cpu_info;
 
 use anyhow::{anyhow, Context, Result};
@@ -106,7 +104,7 @@ pub fn get_distro_details(os_release: &str, os_release_clr: &str) -> Result<(Str
     Ok((name, version))
 }
 
-#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))]
+#[cfg(any(target_arch = "s390x", target_arch = "x86_64", target_arch = "aarch64"))]
 pub fn get_generic_cpu_details(cpu_info_file: &str) -> Result<(String, String)> {
     let cpu_info = get_single_cpu_info(cpu_info_file, "\n\n")?;
     let lines = cpu_info.lines();

From c57a44436cd58bc974386611773fb2eee7d4a77a Mon Sep 17 00:00:00 2001
From: Ryan Savino <ryan.savino@amd.com>
Date: Mon, 24 Apr 2023 11:34:34 -0500
Subject: [PATCH 038/150] gha: Add the ability to test qemu-snp

With the changes proposed as part of this PR, a qemu-snp cluster
will be created but no tests will be performed.

GitHub Actions will only run the tests using the workflows that are
part of the **target** branch, instead of the using the ones coming
from the PR. No way to work around this for now.

After this commit is merged, the tests (not the yaml files for the
actions) will be altered in order for the checkout action  to help in
this case.

Fixes: #6722

Signed-off-by: Ryan Savino <ryan.savino@amd.com>
---
 tests/integration/kubernetes/run_kubernetes_tests.sh | 4 ++++
 1 file changed, 4 insertions(+)
 mode change 100755 => 100644 tests/integration/kubernetes/run_kubernetes_tests.sh

diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
old mode 100755
new mode 100644
index daddb756a..281d1d878
--- a/tests/integration/kubernetes/run_kubernetes_tests.sh
+++ b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -58,6 +58,10 @@ if [ ${KATA_HYPERVISOR} == "qemu-sev" ]; then
 	exit 0
 fi
 
+if [ ${KATA_HYPERVISOR} == "qemu-snp" ]; then
+	exit 0
+fi
+
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then

From 5c9246db19bf8c6cf5567df42535fcbae1795e0b Mon Sep 17 00:00:00 2001
From: Ryan Savino <ryan.savino@amd.com>
Date: Mon, 24 Apr 2023 11:36:35 -0500
Subject: [PATCH 039/150] gha: Also run k8s tests on qemu-snp

Added the k8s tests for qemu-snp

Fixes: #6722

Signed-Off-By: Ryan Savino <ryan.savino@amd.com>
---
 .github/workflows/ci-on-push.yaml           |  8 +++
 .github/workflows/run-k8s-tests-on-snp.yaml | 65 +++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 .github/workflows/run-k8s-tests-on-snp.yaml

diff --git a/.github/workflows/ci-on-push.yaml b/.github/workflows/ci-on-push.yaml
index 8b1b6ef36..3951bba08 100644
--- a/.github/workflows/ci-on-push.yaml
+++ b/.github/workflows/ci-on-push.yaml
@@ -37,6 +37,14 @@ jobs:
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
 
+  run-k8s-tests-on-snp:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+
   run-k8s-tests-on-tdx:
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml
new file mode 100644
index 000000000..5bc4aea50
--- /dev/null
+++ b/.github/workflows/run-k8s-tests-on-snp.yaml
@@ -0,0 +1,65 @@
+name: CI | Run kubernetes tests on SEV-SNP
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      KUBECONFIG: /home/kata/.kube/config
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Deploy kata-deploy
+        run: |
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+
+          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: |
+          pushd tests/integration/kubernetes
+          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
+          bash run_kubernetes_tests.sh
+          popd
+        env:
+          KATA_HYPERVISOR: ${{ matrix.vmm }}
+
+      - name: Delete kata-deploy
+        if: always()
+        run: |
+          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+
+          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
+          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          sleep 180s
+
+          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

From 9e2b7ff17733a657670c64eadbeb1f58cbb0a4cc Mon Sep 17 00:00:00 2001
From: Ryan Savino <ryan.savino@amd.com>
Date: Fri, 28 Apr 2023 14:54:36 -0500
Subject: [PATCH 040/150] gha: sev: fix for kata-deploy error

kubectl commands need a '-f' instead of a '-k'

Fixes: #6758

Signed-Off-By: Ryan Savino <ryan.savino@amd.com>
---
 .github/workflows/run-k8s-tests-on-sev.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml
index 043b21cc1..aeafa242e 100644
--- a/.github/workflows/run-k8s-tests-on-sev.yaml
+++ b/.github/workflows/run-k8s-tests-on-sev.yaml
@@ -34,7 +34,7 @@ jobs:
           cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
 
           kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
           kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 
@@ -51,7 +51,7 @@ jobs:
       - name: Delete kata-deploy
         if: always()
         run: |
-          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
 
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml

From 03a8cd69c26119715d22bfbff643236a25d2014a Mon Sep 17 00:00:00 2001
From: Eduardo Berrocal <eduardo.berrocal@intel.com>
Date: Fri, 28 Apr 2023 15:40:45 -0700
Subject: [PATCH 041/150] virtcontainers: Improved test coverage for fc.go from
 4.6% to 18.5%

Expanded tests on fc_test.go to cover more lines of code. Coverage went from 4.6% to 18.5%.

Fixes: #266

Signed-off-by: Eduardo Berrocal <eduardo.berrocal@intel.com>
---
 src/runtime/virtcontainers/fc_test.go | 175 ++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)

diff --git a/src/runtime/virtcontainers/fc_test.go b/src/runtime/virtcontainers/fc_test.go
index 64bde2232..361ca0962 100644
--- a/src/runtime/virtcontainers/fc_test.go
+++ b/src/runtime/virtcontainers/fc_test.go
@@ -8,6 +8,7 @@
 package virtcontainers
 
 import (
+	"context"
 	"strings"
 	"testing"
 
@@ -54,11 +55,185 @@ func TestFCParseVersion(t *testing.T) {
 
 	fc := firecracker{}
 
+	// correct versions
 	for rawVersion, v := range map[string]string{"Firecracker v0.23.1": "0.23.1", "Firecracker v0.25.0\nSupported snapshot data format versions: 0.23.0": "0.25.0"} {
 		parsedVersion, err := fc.parseVersion(rawVersion)
 		assert.NoError(err)
 		assert.Equal(parsedVersion, v)
 	}
+
+	// wrong version str
+	rawVersion := "Firecracker_v0.23.0"
+	parsedVersion, err := fc.parseVersion(rawVersion)
+	assert.Error(err)
+	assert.Equal(parsedVersion, "")
+}
+
+func TestFCCheckVersion(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+
+	// correct version
+	v := "0.23.0"
+	err := fc.checkVersion(v)
+	assert.NoError(err)
+
+	// version too low
+	v = "0.1.1"
+	err = fc.checkVersion(v)
+	assert.Error(err)
+	b := err.Error()
+	assert.True(strings.Contains(b, "version 0.1.1 is not supported")) // sanity
+
+	// version is malformed
+	v = "Firecracker v0.23.0"
+	err = fc.checkVersion(v)
+	assert.Error(err)
+	b = err.Error()
+	assert.True(strings.Contains(b, "Malformed firecracker version:")) // sanity
+}
+
+func TestFCGetVersionNumber(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	_, err := fc.getVersionNumber()
+	assert.Error(err)
+}
+
+func TestFCDriveIndexToID(t *testing.T) {
+	assert := assert.New(t)
+
+	d := fcDriveIndexToID(5)
+	assert.Equal(d, "drive_5")
+}
+
+func TestFCPauseVM(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+	err := fc.PauseVM(ctx)
+	assert.NoError(err)
+}
+
+func TestFCSaveVM(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	err := fc.SaveVM()
+	assert.NoError(err)
+}
+
+func TestFCResumeVM(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+	err := fc.ResumeVM(ctx)
+	assert.NoError(err)
+}
+
+func TestFCGetVirtioFsPid(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	pid := fc.GetVirtioFsPid()
+	assert.Nil(pid)
+}
+
+func TestFCIsRateLimiterBuiltin(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	rl := fc.IsRateLimiterBuiltin()
+	assert.True(rl)
+}
+
+func TestFCCheck(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	err := fc.Check()
+	assert.NoError(err)
+}
+
+func TestFCGetPids(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	pids := fc.GetPids()
+	assert.Equal(len(pids), 1)
+}
+
+func TestFCCleanup(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+	err := fc.Cleanup(ctx)
+	assert.NoError(err)
+}
+
+func TestFCToGrpc(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+	_, err := fc.toGrpc(ctx)
+	assert.Error(err)
+}
+
+func TestFCHypervisorConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	config := fc.HypervisorConfig()
+	assert.Equal(fc.config, config)
+}
+
+func TestFCGetTotalMemoryMB(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+
+	var initialMemSize uint32
+	initialMemSize = 1024
+
+	fc.config.MemorySize = 1024
+	memSize := fc.GetTotalMemoryMB(ctx)
+	assert.Equal(memSize, initialMemSize)
+}
+
+func TestFCClient(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+
+	conn := fc.client(ctx)
+	assert.Equal(conn, fc.connection)
+}
+
+func TestFCVmRunning(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+	ctx := context.Background()
+	sr := fc.vmRunning(ctx)
+	assert.False(sr)
+}
+
+func TestFCCreateJailedDrive(t *testing.T) {
+	assert := assert.New(t)
+
+	fc := firecracker{}
+
+	driveID := fcDriveIndexToID(0)
+	_, err := fc.createJailedDrive(driveID)
+	assert.NoError(err)
 }
 
 func TestFcSetConfig(t *testing.T) {

From 6594a9329d0af06e49f512d8ee8b9cd885ef0593 Mon Sep 17 00:00:00 2001
From: Gabe Venberg <gabevenberg@gmail.com>
Date: Fri, 24 Feb 2023 14:26:58 -0600
Subject: [PATCH 042/150] tools: made log-parser-rs

Eventual replacement of kata-log-parser, but for now replicates its
functionaility for the new runtime-rs syntax. Takes in log files,
parses, sorts by timestamp, spits them out in json, csv, xml, toml, and
a few others.

Fixes #5350

Signed-off-by: Gabe Venberg <gabevenberg@gmail.com>
---
 Makefile                                      |   1 +
 README.md                                     |   1 +
 src/tools/log-parser-rs/Cargo.lock            | 899 ++++++++++++++++++
 src/tools/log-parser-rs/Cargo.toml            |  21 +
 src/tools/log-parser-rs/Makefile              |  35 +
 src/tools/log-parser-rs/README.md             |  67 ++
 src/tools/log-parser-rs/src/args.rs           |  58 ++
 src/tools/log-parser-rs/src/log_message.rs    | 467 +++++++++
 .../log-parser-rs/src/log_parser_error.rs     |  58 ++
 src/tools/log-parser-rs/src/main.rs           |  68 ++
 src/tools/log-parser-rs/src/output_file.rs    | 149 +++
 src/tools/log-parser-rs/src/parse_file.rs     |  85 ++
 src/tools/log-parser-rs/src/process_logs.rs   |  90 ++
 13 files changed, 1999 insertions(+)
 create mode 100644 src/tools/log-parser-rs/Cargo.lock
 create mode 100644 src/tools/log-parser-rs/Cargo.toml
 create mode 100644 src/tools/log-parser-rs/Makefile
 create mode 100644 src/tools/log-parser-rs/README.md
 create mode 100644 src/tools/log-parser-rs/src/args.rs
 create mode 100644 src/tools/log-parser-rs/src/log_message.rs
 create mode 100644 src/tools/log-parser-rs/src/log_parser_error.rs
 create mode 100644 src/tools/log-parser-rs/src/main.rs
 create mode 100644 src/tools/log-parser-rs/src/output_file.rs
 create mode 100644 src/tools/log-parser-rs/src/parse_file.rs
 create mode 100644 src/tools/log-parser-rs/src/process_logs.rs

diff --git a/Makefile b/Makefile
index 8e856138c..e70af93e4 100644
--- a/Makefile
+++ b/Makefile
@@ -18,6 +18,7 @@ TOOLS =
 TOOLS += agent-ctl
 TOOLS += kata-ctl
 TOOLS += log-parser
+TOOLS += log-parser-rs
 TOOLS += runk
 TOOLS += trace-forwarder
 
diff --git a/README.md b/README.md
index 6972ed278..662899ea6 100644
--- a/README.md
+++ b/README.md
@@ -134,6 +134,7 @@ The table below lists the remaining parts of the project:
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
+| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
diff --git a/src/tools/log-parser-rs/Cargo.lock b/src/tools/log-parser-rs/Cargo.lock
new file mode 100644
index 000000000..c1a19ec4e
--- /dev/null
+++ b/src/tools/log-parser-rs/Cargo.lock
@@ -0,0 +1,899 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1"
+
+[[package]]
+name = "bumpalo"
+version = "3.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
+
+[[package]]
+name = "cc"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "chrono"
+version = "0.4.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
+dependencies = [
+ "iana-time-zone",
+ "js-sys",
+ "num-integer",
+ "num-traits",
+ "serde",
+ "time 0.1.45",
+ "wasm-bindgen",
+ "winapi",
+]
+
+[[package]]
+name = "clap"
+version = "4.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098"
+dependencies = [
+ "bitflags 2.0.2",
+ "clap_derive",
+ "clap_lex",
+ "is-terminal",
+ "once_cell",
+ "strsim",
+ "termcolor",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646"
+dependencies = [
+ "os_str_bytes",
+]
+
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
+[[package]]
+name = "csv"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "cxx"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038"
+dependencies = [
+ "cc",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn 2.0.5",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.5",
+]
+
+[[package]]
+name = "darling"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "errno"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.54"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+dependencies = [
+ "cxx",
+ "cxx-build",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "indexmap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+ "serde",
+]
+
+[[package]]
+name = "io-lifetimes"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "is-terminal"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e"
+dependencies = [
+ "hermit-abi",
+ "io-lifetimes",
+ "rustix",
+ "windows-sys",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
+
+[[package]]
+name = "js-sys"
+version = "0.3.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.140"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
+
+[[package]]
+name = "link-cplusplus"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "log-parser"
+version = "0.0.1"
+dependencies = [
+ "chrono",
+ "clap",
+ "csv",
+ "quick-xml",
+ "ron",
+ "serde",
+ "serde_json",
+ "serde_with",
+ "serde_yaml",
+ "slog",
+ "thiserror",
+ "toml",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "num-integer"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
+dependencies = [
+ "autocfg",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+
+[[package]]
+name = "os_str_bytes"
+version = "6.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267"
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quick-xml"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "ron"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300a51053b1cb55c80b7a9fde4120726ddf25ca241a1cbb926626f62fb136bff"
+dependencies = [
+ "base64",
+ "bitflags 1.3.2",
+ "serde",
+]
+
+[[package]]
+name = "rustix"
+version = "0.36.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e"
+dependencies = [
+ "bitflags 1.3.2",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
+
+[[package]]
+name = "scratch"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
+
+[[package]]
+name = "serde"
+version = "1.0.158"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.158"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.5",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.94"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_with"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf"
+dependencies = [
+ "base64",
+ "chrono",
+ "hex",
+ "indexmap",
+ "serde",
+ "serde_json",
+ "serde_with_macros",
+ "time 0.3.20",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "serde_yaml"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
+[[package]]
+name = "slog"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06"
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89c2d1c76a26822187a1fbb5964e3fff108bc208f02e820ab9dac1234f6b388a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.5",
+]
+
+[[package]]
+name = "time"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
+dependencies = [
+ "libc",
+ "wasi",
+ "winapi",
+]
+
+[[package]]
+name = "time"
+version = "0.3.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
+dependencies = [
+ "itoa",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+
+[[package]]
+name = "time-macros"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
+dependencies = [
+ "time-core",
+]
+
+[[package]]
+name = "toml"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "wasi"
+version = "0.10.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
diff --git a/src/tools/log-parser-rs/Cargo.toml b/src/tools/log-parser-rs/Cargo.toml
new file mode 100644
index 000000000..bcd98246b
--- /dev/null
+++ b/src/tools/log-parser-rs/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "log-parser"
+version = "0.0.1"
+edition = "2021"
+authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serde_yaml = "0.9"
+toml = "0.4"
+ron = "0.8"
+quick-xml = { version = "0.28", features = ["serialize"]}
+csv = "1.2"
+serde_with = "2.3"
+clap = { version = "4.1", features = ["derive", "cargo"] }
+thiserror = "1.0"
+chrono = { version = "0.4", features = ["serde"]}
+slog = "2.7"
diff --git a/src/tools/log-parser-rs/Makefile b/src/tools/log-parser-rs/Makefile
new file mode 100644
index 000000000..5bca7cc57
--- /dev/null
+++ b/src/tools/log-parser-rs/Makefile
@@ -0,0 +1,35 @@
+# Copyright (c) 2023 Gabe Venberg
+#
+# SPDX-License-Identifier: Apache-2.0
+
+include ../../../utils.mk
+
+.DEFAULT_GOAL := default
+default: build
+
+build:
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE)
+
+static-checks-build:
+	@echo "INFO: static-checks-build do nothing.."
+
+clean:
+	cargo clean
+
+vendor:
+	cargo vendor
+
+test:
+
+install:
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path .
+
+check: standard_rust_check
+
+.PHONY: \
+	build \
+	check \
+	clean \
+	install \
+	test \
+	vendor
diff --git a/src/tools/log-parser-rs/README.md b/src/tools/log-parser-rs/README.md
new file mode 100644
index 000000000..a641b0030
--- /dev/null
+++ b/src/tools/log-parser-rs/README.md
@@ -0,0 +1,67 @@
+# `kata-log-parser`
+
+## Introduction
+
+`log-parser-rs` is a tool that combines logfiles generated by the various
+system components, sorts them by time stamp, and re-displays the log entries.
+
+The tool is also able to check the validity of all log records, can re-format the
+logs, and output them in a different format.
+
+For more information on the `kata-log-parser` tool, use the help command:
+
+```
+$ kata-log-parser --help
+```
+
+> **Note** this is a rewrite of the go-based `kata-log-parser` tool, and will eventually replace it.
+
+## Log Format
+
+Kata's `runtime-rs` logs are JSON objects in the following format:
+
+```json
+{"msg":"message","level":"INFO","ts":"1970-01-01T00:00:00.000000000Z","name":"kata-runtime","version":"0.1.0","pid":"0","source":"source","subsystem":"subsystem"}
+```
+
+However, if `--ignore-missing-fields` is set, a log missing one or more of the following fields may be omitted: 
+
+- `level`
+- `name`
+- `version`
+- `pid`
+- `source`
+- `subsystem`
+
+> **Note** a log entry must be on one single line, and a line must contain only one log entry.
+
+## Command line opts
+
+The most valuable command line options are listed below:
+
+- `-o, --output-file <OUTPUT_FILE>` File to output to. If not set, sends to stdout.
+- `--output-format <OUTPUT_FORMAT>` Sets the format of the output. Defaults to `json`, and can be set to `csv`, `json`, `ron`, `text`, `toml`, `xml`, and `yaml`.
+- `-q, --quiet` Will not print invalid log entry errors to stderr.
+- `-s, --strict` Any invalid log entry will halt the program.
+
+For a comprehensive (and guaranteed up to date) list, please run `log-parser-rs --help`.
+
+## Usage
+
+1. Make sure containerd is in [debug mode](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#enabling-full-containerd-debug)
+1. Make sure you are running runtime-rs:
+   ```
+   $ containerd-shim-kata-v2 --version|grep -qi rust && echo rust || echo golang
+   ```
+1. Collect the logs (alternatively to journal clearing you may consider constraining collected logs by adding `--since=<container creation time>`).
+   ```
+   $ sudo journalctl -q -o cat -a -t kata | grep "^{" > ./kata.log ./kata.log
+   ```
+1. Ensure the logs are readable:
+   ```
+   $ sudo chown $USER *.log
+   ```
+1. Process the logs:
+   ```
+   $ log-parser-rs kata.log -o out.log
+   ```
diff --git a/src/tools/log-parser-rs/src/args.rs b/src/tools/log-parser-rs/src/args.rs
new file mode 100644
index 000000000..1980fcc35
--- /dev/null
+++ b/src/tools/log-parser-rs/src/args.rs
@@ -0,0 +1,58 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use clap::{Parser, ValueEnum};
+use std::path::PathBuf;
+
+#[derive(Parser, Debug)]
+#[command(name="kata-log-parser", author, version, about, long_about = None)] // Read from `Cargo.toml`
+pub struct Cli {
+    pub input_file: Vec<PathBuf>,
+
+    #[arg(short, long)]
+    pub output_file: Option<PathBuf>,
+
+    #[arg(short, long, help = "check log files and only display output on error")]
+    pub check_only: bool,
+
+    #[arg(long, help = "error if any files are empty")]
+    pub error_if_file_empty: bool,
+
+    #[arg(long, help = "error if all logfiles are empty")]
+    pub error_if_no_records: bool,
+
+    #[arg(
+        long,
+        help = "do not make an error for lines with no pid, source, name, or level"
+    )]
+    pub ignore_missing_fields: bool,
+
+    #[arg(
+        short,
+        long,
+        help = "suppress warning messages that would otherwise go to stderr."
+    )]
+    pub quiet: bool,
+
+    #[arg(
+        short,
+        long,
+        help = "do not tolerate misformed agent messages (may be caused by non-Kata Containers log lines)"
+    )]
+    pub strict: bool,
+
+    #[arg(long, value_enum, default_value_t = OutputFormat::Json, help="set the output format")]
+    pub output_format: OutputFormat,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
+pub enum OutputFormat {
+    Csv,
+    Json,
+    Ron,
+    Text,
+    Toml,
+    Xml,
+    Yaml,
+}
diff --git a/src/tools/log-parser-rs/src/log_message.rs b/src/tools/log-parser-rs/src/log_message.rs
new file mode 100644
index 000000000..9caf849c6
--- /dev/null
+++ b/src/tools/log-parser-rs/src/log_message.rs
@@ -0,0 +1,467 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{fmt::Debug, fmt::Display, str::FromStr};
+
+use chrono::{DateTime, Utc};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use serde_with::{
+    serde_as, skip_serializing_none, DeserializeFromStr, DisplayFromStr, SerializeDisplay,
+};
+use thiserror::Error;
+
+pub trait AnyLogMessage: Serialize + DeserializeOwned + Debug {
+    fn get_timestamp(&self) -> DateTime<Utc>;
+}
+
+#[serde_as]
+#[skip_serializing_none]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default)]
+pub struct LogMessage {
+    pub level: Option<LogLevel>,
+
+    #[serde(rename = "msg")]
+    pub message: String,
+
+    pub name: Option<String>,
+
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub pid: Option<usize>,
+
+    pub source: Option<String>,
+
+    pub subsystem: Option<String>,
+
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(rename = "ts")]
+    pub timestamp: DateTime<Utc>,
+}
+
+impl AnyLogMessage for LogMessage {
+    fn get_timestamp(&self) -> DateTime<Utc> {
+        self.timestamp
+    }
+}
+
+//totally abusing serde to easily display this.
+impl Display for LogMessage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            serde_json::to_string_pretty(&self).map_err(|_| std::fmt::Error)?
+        )
+    }
+}
+
+#[serde_as]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default)]
+pub struct StrictLogMessage {
+    pub level: LogLevel,
+
+    #[serde(rename = "msg")]
+    pub message: String,
+
+    pub name: String,
+
+    #[serde_as(as = "DisplayFromStr")]
+    pub pid: usize,
+
+    pub source: String,
+
+    pub subsystem: String,
+
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(rename = "ts")]
+    pub timestamp: DateTime<Utc>,
+}
+
+impl AnyLogMessage for StrictLogMessage {
+    fn get_timestamp(&self) -> DateTime<Utc> {
+        self.timestamp
+    }
+}
+
+//totally abusing serde to easily display this.
+impl Display for StrictLogMessage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            serde_json::to_string_pretty(&self).map_err(|_| std::fmt::Error)?
+        )
+    }
+}
+
+// A newtype for slog::Level, as it does not implement Serialize and Deserialize.
+#[derive(Debug, SerializeDisplay, DeserializeFromStr, PartialEq, Eq)]
+pub struct LogLevel(slog::Level);
+
+#[derive(Debug, Error)]
+pub enum LevelError {
+    #[error("invalid slog level: {0}")]
+    InvalidLevel(String),
+}
+
+impl Display for LogLevel {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("{}", self.0.as_str()))
+    }
+}
+
+impl Default for LogLevel {
+    fn default() -> Self {
+        LogLevel(slog::Level::Info)
+    }
+}
+
+impl FromStr for LogLevel {
+    type Err = LevelError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let level = match s.to_lowercase().as_str() {
+            //need to accept both the short and long string versions.
+            "critical" | "crit" => slog::Level::Critical,
+            "error" | "erro" => slog::Level::Error,
+            "warning" | "warn" => slog::Level::Warning,
+            "info" => slog::Level::Info,
+            "debug" | "debg" => slog::Level::Debug,
+            "trace" | "trce" => slog::Level::Trace,
+            _ => return Err(LevelError::InvalidLevel(s.to_string())),
+        };
+
+        Ok(LogLevel(level))
+    }
+}
+
+//TODO: add tests for serialization.
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::log_parser_error::LogParserError;
+
+    #[test]
+    fn parse_string() {
+        let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#;
+        let result = Ok(LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: Some("kata-runtime".to_string()),
+            pid: Some(3327263),
+            source: Some("foo".to_string()),
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        });
+        assert_eq!(
+            serde_json::from_str(log)
+                .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())),
+            result
+        )
+    }
+
+    #[test]
+    fn parse_string_with_missing_fields() {
+        let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","version":"0.1.0","subsystem":"hypervisor","source":"foo"}"#;
+        let result = Ok(LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: None,
+            source: Some("foo".to_string()),
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        });
+        assert_eq!(
+            serde_json::from_str(log)
+                .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())),
+            result
+        )
+    }
+
+    #[test]
+    #[should_panic]
+    fn parse_error() {
+        let log = "random non-kata log message";
+        serde_json::from_str::<LogMessage>(log).unwrap();
+    }
+
+    #[test]
+    fn parse_string_strict() {
+        let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#;
+        let result = Ok(StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        });
+        assert_eq!(
+            serde_json::from_str(log)
+                .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())),
+            result
+        )
+    }
+
+    #[test]
+    #[should_panic]
+    fn parse_string_with_missing_fields_strict() {
+        let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","version":"0.1.0","subsystem":"hypervisor","source":"foo"}"#;
+        println!(
+            "{:?}",
+            serde_json::from_str::<StrictLogMessage>(log).unwrap()
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn parse_error_strict() {
+        let log = "random non-kata log message";
+        serde_json::from_str::<StrictLogMessage>(log).unwrap();
+    }
+
+    #[test]
+    fn serialize_json_strict() {
+        let result = r#"{"level":"DEBUG","msg":"vmm-master thread is uninitialized or has exited.","name":"kata-runtime","pid":"3327263","source":"foo","subsystem":"hypervisor","ts":"2023-03-15 14:17:02.526992506 UTC"}"#;
+        let log = StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, serde_json::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_json() {
+        let result = r#"{"level":"DEBUG","msg":"vmm-master thread is uninitialized or has exited.","pid":"3327263","subsystem":"hypervisor","ts":"2023-03-15 14:17:02.526992506 UTC"}"#;
+        let log = LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: Some(3327263),
+            source: None,
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, serde_json::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_csv_strict() {
+        let result = r#"level,msg,name,pid,source,subsystem,ts
+DEBUG,vmm-master thread is uninitialized or has exited.,kata-runtime,3327263,foo,hypervisor,2023-03-15 14:17:02.526992506 UTC
+"#;
+        let log = StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        let mut csv_writer = csv::Writer::from_writer(vec![]);
+        csv_writer.serialize(&log).unwrap();
+        let output = String::from_utf8(csv_writer.into_inner().unwrap()).unwrap();
+        assert_eq!(result, output);
+    }
+
+    #[test]
+    fn serialize_csv() {
+        let result = r#"level,msg,pid,subsystem,ts
+DEBUG,vmm-master thread is uninitialized or has exited.,3327263,hypervisor,2023-03-15 14:17:02.526992506 UTC
+"#;
+        let log = LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: Some(3327263),
+            source: None,
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        let mut csv_writer = csv::Writer::from_writer(vec![]);
+        csv_writer.serialize(&log).unwrap();
+        let output = String::from_utf8(csv_writer.into_inner().unwrap()).unwrap();
+        assert_eq!(result, output);
+    }
+
+    #[test]
+    fn serialize_ron_strict() {
+        let result = r#"(level:"DEBUG",msg:"vmm-master thread is uninitialized or has exited.",name:"kata-runtime",pid:"3327263",source:"foo",subsystem:"hypervisor",ts:"2023-03-15 14:17:02.526992506 UTC")"#;
+        let log = StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, ron::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_ron() {
+        let result = r#"(level:Some("DEBUG"),msg:"vmm-master thread is uninitialized or has exited.",pid:Some("3327263"),subsystem:Some("hypervisor"),ts:"2023-03-15 14:17:02.526992506 UTC")"#;
+        let log = LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: Some(3327263),
+            source: None,
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, ron::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_toml_strict() {
+        let result = r#"level = "DEBUG"
+msg = "vmm-master thread is uninitialized or has exited."
+name = "kata-runtime"
+pid = "3327263"
+source = "foo"
+subsystem = "hypervisor"
+ts = "2023-03-15 14:17:02.526992506 UTC"
+"#;
+        let log = StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, toml::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_toml() {
+        let result = r#"level = "DEBUG"
+msg = "vmm-master thread is uninitialized or has exited."
+pid = "3327263"
+subsystem = "hypervisor"
+ts = "2023-03-15 14:17:02.526992506 UTC"
+"#;
+        let log = LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: Some(3327263),
+            source: None,
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, toml::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_xml_strict() {
+        let result = r#"<StrictLogMessage><level>DEBUG</level><msg>vmm-master thread is uninitialized or has exited.</msg><name>kata-runtime</name><pid>3327263</pid><source>foo</source><subsystem>hypervisor</subsystem><ts>2023-03-15 14:17:02.526992506 UTC</ts></StrictLogMessage>"#;
+        let log = StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, quick_xml::se::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_xml() {
+        let result = r#"<LogMessage><level>DEBUG</level><msg>vmm-master thread is uninitialized or has exited.</msg><pid>3327263</pid><subsystem>hypervisor</subsystem><ts>2023-03-15 14:17:02.526992506 UTC</ts></LogMessage>"#;
+        let log = LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: Some(3327263),
+            source: None,
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, quick_xml::se::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_yaml_strict() {
+        let result = r#"level: DEBUG
+msg: vmm-master thread is uninitialized or has exited.
+name: kata-runtime
+pid: '3327263'
+source: foo
+subsystem: hypervisor
+ts: 2023-03-15 14:17:02.526992506 UTC
+"#;
+        let log = StrictLogMessage {
+            level: LogLevel(slog::Level::Debug),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: "kata-runtime".to_string(),
+            pid: 3327263,
+            source: "foo".to_string(),
+            subsystem: "hypervisor".to_string(),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, serde_yaml::to_string(&log).unwrap());
+    }
+
+    #[test]
+    fn serialize_yaml() {
+        let result = r#"level: DEBUG
+msg: vmm-master thread is uninitialized or has exited.
+pid: '3327263'
+subsystem: hypervisor
+ts: 2023-03-15 14:17:02.526992506 UTC
+"#;
+        let log = LogMessage {
+            level: Some(LogLevel(slog::Level::Debug)),
+            message: "vmm-master thread is uninitialized or has exited.".to_string(),
+            name: None,
+            pid: Some(3327263),
+            source: None,
+            subsystem: Some("hypervisor".to_string()),
+            timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z")
+                .unwrap()
+                .into(),
+        };
+        assert_eq!(result, serde_yaml::to_string(&log).unwrap());
+    }
+}
diff --git a/src/tools/log-parser-rs/src/log_parser_error.rs b/src/tools/log-parser-rs/src/log_parser_error.rs
new file mode 100644
index 000000000..4af197d27
--- /dev/null
+++ b/src/tools/log-parser-rs/src/log_parser_error.rs
@@ -0,0 +1,58 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{error::Error, path::PathBuf};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum LogParserError {
+    #[error("Unknown Error")]
+    Unknown(Box<dyn Error>),
+
+    #[error("Input file '{0}' cannot be found")]
+    InputFileNotFound(PathBuf),
+
+    #[error("Input file '{0}' does not contain any valid logs")]
+    FileEmpty(PathBuf),
+
+    #[error("No permission to open '{0}'")]
+    InputFilePermissionError(PathBuf),
+
+    #[error("No permission to write to '{0}'")]
+    OutputFilePermissionError(PathBuf),
+
+    #[error("Log parsing error: {0} with string {1}")]
+    ParsingError(serde_json::Error, String),
+
+    #[error("Error serializing {0}: {1}")]
+    SerializationError(String, Box<dyn Error>),
+
+    #[error("No logs in any file")]
+    NoRecordsError(),
+}
+
+impl PartialEq for LogParserError {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::Unknown(l0), Self::Unknown(r0)) => l0.to_string() == r0.to_string(),
+            (Self::InputFileNotFound(l0), Self::InputFileNotFound(r0)) => l0 == r0,
+            (Self::FileEmpty(l0), Self::FileEmpty(r0)) => l0 == r0,
+            (Self::InputFilePermissionError(l0), Self::InputFilePermissionError(r0)) => l0 == r0,
+            (Self::OutputFilePermissionError(l0), Self::OutputFilePermissionError(r0)) => l0 == r0,
+            //serde_json::Error does not impl partialeq, but for testing cases a quick and dirty
+            //string comparison works well enough.
+            (Self::ParsingError(l0, l1), Self::ParsingError(r0, r1)) => {
+                l0.to_string() == r0.to_string() && l1 == r1
+            }
+            //this catch all returns whether the two enums are the same variant type. eg,
+            //core::mem:discriminant(LogParserError::Unkown)==core:mem:discriminant(LogParserError::Unkown)
+            //is true, but it would not be true, for say, Unkown and InputFilePermissionError.
+            //Note that it only compares the variants, not the contents of the variants, hence the
+            //need for the above branches.
+            _ => core::mem::discriminant(self) == core::mem::discriminant(other),
+        }
+    }
+}
+
+impl Eq for LogParserError {}
diff --git a/src/tools/log-parser-rs/src/main.rs b/src/tools/log-parser-rs/src/main.rs
new file mode 100644
index 000000000..72c406a59
--- /dev/null
+++ b/src/tools/log-parser-rs/src/main.rs
@@ -0,0 +1,68 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#![warn(unused_crate_dependencies)]
+#![warn(missing_debug_implementations)]
+
+mod args;
+mod log_message;
+mod log_parser_error;
+mod output_file;
+mod parse_file;
+mod process_logs;
+
+use crate::args::Cli;
+use crate::log_message::StrictLogMessage;
+use crate::log_parser_error::LogParserError;
+use crate::output_file::*;
+use crate::parse_file::*;
+use crate::process_logs::*;
+use clap::Parser;
+use log_message::AnyLogMessage;
+use log_message::LogMessage;
+use std::process::exit;
+
+fn handle_logs<T: AnyLogMessage>(cli: Cli) -> Result<(), LogParserError> {
+    let mut logs = Vec::new();
+
+    for file in &cli.input_file {
+        let in_file = open_file_into_memory(file)?;
+        let file_logs = filter_errors(parse_log::<T>(in_file), &cli)?;
+
+        if cli.error_if_file_empty && file_logs.is_empty() {
+            return Err(LogParserError::FileEmpty(file.to_path_buf()));
+        }
+
+        logs.extend(file_logs)
+    }
+
+    if cli.error_if_no_records && logs.is_empty() {
+        return Err(LogParserError::NoRecordsError());
+    }
+    if cli.check_only {
+        return Ok(());
+    }
+
+    sort_logs(&mut logs);
+    output_file(logs, &cli)?;
+    Ok(())
+}
+
+//needed another layer of function call in order to genericize over both LogMessage and
+//StrictLogMessage.
+fn real_main() -> std::result::Result<(), LogParserError> {
+    let cli = Cli::parse();
+    if cli.ignore_missing_fields {
+        handle_logs::<LogMessage>(cli)
+    } else {
+        handle_logs::<StrictLogMessage>(cli)
+    }
+}
+
+fn main() {
+    if let Err(e) = real_main() {
+        eprintln!("ERROR: {:#}", e);
+        exit(1);
+    }
+}
diff --git a/src/tools/log-parser-rs/src/output_file.rs b/src/tools/log-parser-rs/src/output_file.rs
new file mode 100644
index 000000000..eb6ddf146
--- /dev/null
+++ b/src/tools/log-parser-rs/src/output_file.rs
@@ -0,0 +1,149 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{
+    fs::File,
+    io::{ErrorKind, Write},
+    path::Path,
+};
+
+use crate::{args::Cli, log_message::AnyLogMessage, log_parser_error::LogParserError};
+
+/// a simple dispatcher method that outputs the deserialized result of the parsed logs according to
+/// the CLI arguments.
+///
+/// # Errors
+///
+/// If outputting to a file, may return a LogParserError having to do with opening and writing to
+/// the output file.
+pub(crate) fn output_file<O: AnyLogMessage>(
+    contents: Vec<O>,
+    options: &Cli,
+) -> Result<(), LogParserError> {
+    let serializer = choose_formatting(options);
+    if let Some(out_file) = &options.output_file {
+        write_logs_to_file(out_file, contents, serializer)?;
+    } else {
+        print_logs(contents, serializer)?;
+    };
+    Ok(())
+}
+
+fn choose_formatting<O: AnyLogMessage>(
+    options: &Cli,
+) -> fn(Vec<O>) -> Result<String, LogParserError> {
+    match options.output_format {
+        crate::args::OutputFormat::Csv => serialize_csv,
+        crate::args::OutputFormat::Json => serialize_json,
+        crate::args::OutputFormat::Ron => serialize_ron,
+        crate::args::OutputFormat::Text => serialize_text,
+        crate::args::OutputFormat::Toml => serialize_toml,
+        crate::args::OutputFormat::Xml => serialize_xml,
+        crate::args::OutputFormat::Yaml => serialize_yaml,
+    }
+}
+
+fn serialize_text<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    Ok(input
+        .iter()
+        .map(|r| Ok(format!("{:?}", r)))
+        .collect::<Result<Vec<String>, LogParserError>>()?
+        .join("\n"))
+}
+
+fn serialize_json<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    Ok(input
+        .iter()
+        .map(|r| {
+            serde_json::to_string(r)
+                .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e)))
+        })
+        .collect::<Result<Vec<String>, LogParserError>>()?
+        .join("\n"))
+}
+
+fn serialize_toml<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    Ok(input
+        .iter()
+        .map(|r| {
+            toml::to_string(r)
+                .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e)))
+        })
+        .collect::<Result<Vec<String>, LogParserError>>()?
+        .join("\n"))
+}
+
+fn serialize_yaml<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    Ok(input
+        .iter()
+        .map(|r| {
+            serde_yaml::to_string(r)
+                .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e)))
+        })
+        .collect::<Result<Vec<String>, LogParserError>>()?
+        .join("\n"))
+}
+
+fn serialize_xml<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    Ok(input
+        .iter()
+        .map(|r| {
+            quick_xml::se::to_string(r)
+                .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e)))
+        })
+        .collect::<Result<Vec<String>, LogParserError>>()?
+        .join("\n"))
+}
+
+fn serialize_ron<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    Ok(input
+        .iter()
+        .map(|r| {
+            ron::to_string(r)
+                .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e)))
+        })
+        .collect::<Result<Vec<String>, LogParserError>>()?
+        .join("\n"))
+}
+
+fn serialize_csv<O: AnyLogMessage>(input: Vec<O>) -> Result<String, LogParserError> {
+    let mut csv_writer = csv::Writer::from_writer(vec![]);
+    for record in input {
+        csv_writer.serialize(&record).map_err(|e| {
+            LogParserError::SerializationError(format!("{:?}", record), Box::new(e))
+        })?;
+    }
+    String::from_utf8(
+        csv_writer
+            .into_inner()
+            .map_err(|e| LogParserError::Unknown(Box::new(e)))?,
+    )
+    .map_err(|e| LogParserError::Unknown(Box::new(e)))
+}
+
+fn write_logs_to_file<O: AnyLogMessage>(
+    out_file: &Path,
+    contents: Vec<O>,
+    serializer: fn(Vec<O>) -> Result<String, LogParserError>,
+) -> Result<(), LogParserError> {
+    let mut file = File::create(out_file).map_err(|e| map_out_io_err(e, out_file))?;
+    file.write_all(serializer(contents)?.as_bytes())
+        .map_err(|e| LogParserError::Unknown(Box::new(e)))?;
+    Ok(())
+}
+
+fn print_logs<O: AnyLogMessage>(
+    contents: Vec<O>,
+    serializer: fn(Vec<O>) -> Result<String, LogParserError>,
+) -> Result<(), LogParserError> {
+    print!("{}", serializer(contents)?);
+    Ok(())
+}
+
+fn map_out_io_err(e: std::io::Error, out_file: &Path) -> LogParserError {
+    match e.kind() {
+        ErrorKind::PermissionDenied => LogParserError::OutputFilePermissionError(out_file.into()),
+        _ => LogParserError::Unknown(Box::new(e)),
+    }
+}
diff --git a/src/tools/log-parser-rs/src/parse_file.rs b/src/tools/log-parser-rs/src/parse_file.rs
new file mode 100644
index 000000000..3304aebe2
--- /dev/null
+++ b/src/tools/log-parser-rs/src/parse_file.rs
@@ -0,0 +1,85 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{log_message::AnyLogMessage, log_parser_error::LogParserError};
+use std::{fs, io::ErrorKind, path::Path};
+
+/// Reads the entire file into memory as a string.
+///
+/// # Errors
+///
+/// This function will return an error if opening the file returns an error.
+/// FileNotFound and FilePermissionError *should* be the only error types that can happen, and they
+/// are wrapped in the appropriate LogParserError types. Any other error will panic.
+///
+/// # Panic
+///
+/// Will panic if File::open returns an error other than NotFound or PermissionDenied.
+/// This *should* not happen.
+///
+/// # Gochas
+///
+/// Memory use can be unexpectedly high, as entire file is read into memory.
+pub(crate) fn open_file_into_memory(inputfile: &Path) -> Result<String, LogParserError> {
+    match fs::read_to_string(inputfile) {
+        Ok(s) => Ok(s),
+        Err(e) => match e.kind() {
+            ErrorKind::NotFound => Err(LogParserError::InputFileNotFound(inputfile.to_path_buf())),
+            ErrorKind::PermissionDenied => Err(LogParserError::InputFilePermissionError(
+                inputfile.to_path_buf(),
+            )),
+            _ => Err(LogParserError::Unknown(Box::new(e))),
+        },
+    }
+}
+
+/// Parses a series of logs from a string, returning an array of log messages and parsing errors.
+///
+/// # Errors
+///
+/// Will pass any errors from serde in to the output array
+pub(crate) fn parse_log<O>(in_file: String) -> Vec<Result<O, LogParserError>>
+where
+    O: AnyLogMessage,
+{
+    let mut output = Vec::new();
+    for line in in_file.lines() {
+        let logentry = serde_json::from_str::<O>(line)
+            .map_err(|e| LogParserError::ParsingError(e, line.to_string()));
+        output.push(logentry);
+    }
+    output
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::log_message::LogMessage;
+
+    #[test]
+    fn parse_strings() {
+        let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}
+{"msg":"resource clean up","level":"INFO","ts":"2023-03-15T14:17:02.527047136Z","subsystem":"virt-container","name":"kata-runtime","pid":"3327263","version":"0.1.0","source":"foo"}"#;
+        let result = vec![
+            serde_json::from_str(r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())),
+            serde_json::from_str(r#"{"msg":"resource clean up","level":"INFO","ts":"2023-03-15T14:17:02.527047136Z","subsystem":"virt-container","name":"kata-runtime","pid":"3327263","version":"0.1.0","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())),
+            ];
+        assert_eq!(parse_log::<LogMessage>(log.into()), result)
+    }
+    #[test]
+    fn parse_mixed() {
+        let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}
+Random Kernel Message"#;
+        let result = vec![
+            serde_json::from_str(r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())),
+
+                Err(LogParserError::ParsingError(
+                    serde_json::from_str::<LogMessage>("Random Kernel Message")
+                        .err()
+                        .unwrap()
+                , "Random Kernel Message".to_string())),
+            ];
+        assert_eq!(parse_log::<LogMessage>(log.into()), result)
+    }
+}
diff --git a/src/tools/log-parser-rs/src/process_logs.rs b/src/tools/log-parser-rs/src/process_logs.rs
new file mode 100644
index 000000000..966124fa0
--- /dev/null
+++ b/src/tools/log-parser-rs/src/process_logs.rs
@@ -0,0 +1,90 @@
+// Copyright (c) 2023 Gabe Venberg
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{args::Cli, log_message::AnyLogMessage, log_parser_error::LogParserError};
+
+/// Calls functions to either check for errors, or to filter them out, discarding them or printing
+/// them to stderr.
+///
+/// # Errors
+///
+///  If cli.strict is true, will return the first error it finds in input (if any)
+pub(crate) fn filter_errors<O: AnyLogMessage>(
+    input: Vec<Result<O, LogParserError>>,
+    cli: &Cli,
+) -> Result<Vec<O>, LogParserError> {
+    if cli.strict {
+        find_errors(input)
+    } else if cli.quiet {
+        Ok(filter_errors_quiet(input))
+    } else {
+        Ok(filter_errors_to_stderr(input))
+    }
+}
+
+/// checks if any errors are in the passed log vector. If there are none, returns a vec of O
+/// (stripping the containing Result). If there are some, returns the first found error.
+fn find_errors<O: AnyLogMessage>(
+    input: Vec<Result<O, LogParserError>>,
+) -> Result<Vec<O>, LogParserError> {
+    // yes, this is only the one line, but the behavior of collect() here is non-obvious. In short,
+    // as Result implements IntoIter, you can go from a `Vec<Result<T, E>` to a `Result<Vec<T>, E>`
+    // with just collect()
+    input.into_iter().collect()
+}
+
+/// removes all LogParserErrors, returning a vec of just O
+fn filter_errors_quiet<O: AnyLogMessage>(input: Vec<Result<O, LogParserError>>) -> Vec<O> {
+    input.into_iter().filter_map(|l| l.ok()).collect()
+}
+
+/// removes all LogParserErrors, returning a vec of just O, and prints any errors to stderr.
+fn filter_errors_to_stderr<O: AnyLogMessage>(input: Vec<Result<O, LogParserError>>) -> Vec<O> {
+    input
+        .into_iter()
+        .filter_map(|l| match l {
+            Ok(log) => Some(log),
+            Err(e) => {
+                eprintln!("{}", e);
+                None
+            }
+        })
+        .collect()
+}
+
+/// does what it says on the tin. Sorts a vec of logs in place by their timestamp.
+pub(crate) fn sort_logs<O: AnyLogMessage>(input: &mut [O]) {
+    input.sort_by_key(|l| l.get_timestamp());
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::log_message::LogMessage;
+    #[test]
+    fn error_filter() {
+        let unclean_logs = vec![
+            Ok(LogMessage::default()),
+            Err(LogParserError::SerializationError(
+                "test error".to_string(),
+                Box::new(slog::Error::Fmt(std::fmt::Error)),
+            )),
+        ];
+        let logs = vec![LogMessage::default()];
+        assert_eq!(filter_errors_quiet(unclean_logs), logs)
+    }
+
+    #[test]
+    fn error_filter_to_stderr() {
+        let unclean_logs = vec![
+            Ok(LogMessage::default()),
+            Err(LogParserError::SerializationError(
+                "test error".to_string(),
+                Box::new(slog::Error::Fmt(std::fmt::Error)),
+            )),
+        ];
+        let logs = vec![LogMessage::default()];
+        assert_eq!(filter_errors_to_stderr(unclean_logs), logs)
+    }
+}

From 13d7f39c71e45ae18d9c12cdf5dfdc15fcbbb349 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Tue, 2 May 2023 08:12:18 +0000
Subject: [PATCH 043/150] gpu: Check for VFIO port assignments

Bailing out early if the port is wrong, allowed port settings are
no-port, root-port, switch-port

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
 .../pkg/containerd-shim-v2/create_test.go     |  4 +++
 .../pkg/hypervisors/hypervisor_state.go       |  4 ++-
 .../pkg/katautils/config-settings.go.in       |  6 +++-
 src/runtime/pkg/katautils/config.go           | 34 +++++++++++++++++--
 src/runtime/virtcontainers/sandbox.go         |  2 +-
 5 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go
index ccad5ceea..e3e8e9369 100644
--- a/src/runtime/pkg/containerd-shim-v2/create_test.go
+++ b/src/runtime/pkg/containerd-shim-v2/create_test.go
@@ -20,6 +20,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
@@ -308,6 +309,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 }
 
 func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) {
+	var coldPlugVFIO hv.PCIePort
 	if dir == "" {
 		return "", fmt.Errorf("BUG: need directory")
 	}
@@ -332,6 +334,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
+	coldPlugVFIO = hv.RootPort
 
 	configFileOptions := ktu.RuntimeConfigOptions{
 		Hypervisor:           "qemu",
@@ -350,6 +353,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 		DisableNewNetNs:      disableNewNetNs,
 		SharedFS:             sharedFS,
 		VirtioFSDaemon:       virtioFSdaemon,
+		ColdPlugVFIO:         coldPlugVFIO,
 	}
 
 	runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions)
diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go
index c241dd675..482b7e9e2 100644
--- a/src/runtime/pkg/hypervisors/hypervisor_state.go
+++ b/src/runtime/pkg/hypervisors/hypervisor_state.go
@@ -48,10 +48,12 @@ func (p PCIePort) String() string {
 		return "root-port"
 	case SwitchPort:
 		return "switch-port"
+	case BridgePort:
+		return "bridge-port"
 	case NoPort:
 		return "no-port"
 	}
-	return fmt.Sprintf("unknown PCIePort: %s", string(p))
+	return fmt.Sprintf("<unknown PCIePort: %s>", string(p))
 }
 
 type HypervisorState struct {
diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in
index 14a2b0b58..139d54826 100644
--- a/src/runtime/pkg/katautils/config-settings.go.in
+++ b/src/runtime/pkg/katautils/config-settings.go.in
@@ -9,6 +9,10 @@
 
 package katautils
 
+import (
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+)
+
 // name is the name of the runtime
 var NAME = "@RUNTIME_NAME@"
 
@@ -104,4 +108,4 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
 
-const defaultColdPlugVFIO = "no-port"
+const defaultColdPlugVFIO = hv.NoPort
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
index 06b217a9a..6b08f4afe 100644
--- a/src/runtime/pkg/katautils/config.go
+++ b/src/runtime/pkg/katautils/config.go
@@ -287,6 +287,13 @@ func (h hypervisor) firmware() (string, error) {
 	return ResolvePath(p)
 }
 
+func (h hypervisor) coldPlugVFIO() hv.PCIePort {
+	if h.ColdPlugVFIO == "" {
+		return defaultColdPlugVFIO
+	}
+	return h.ColdPlugVFIO
+}
+
 func (h hypervisor) firmwareVolume() (string, error) {
 	p := h.FirmwareVolume
 
@@ -856,7 +863,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Msize9p:                 h.msize9p(),
 		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
-		ColdPlugVFIO:            h.ColdPlugVFIO,
+		ColdPlugVFIO:            h.coldPlugVFIO(),
 		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         h.DisableVhostNet,
 		EnableVhostUserStore:    h.EnableVhostUserStore,
@@ -1051,7 +1058,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:                h.EnableIOThreads,
 		Msize9p:                        h.msize9p(),
 		HotplugVFIOOnRootBus:           h.HotplugVFIOOnRootBus,
-		ColdPlugVFIO:                   h.ColdPlugVFIO,
+		ColdPlugVFIO:                   h.coldPlugVFIO(),
 		PCIeRootPort:                   h.PCIeRootPort,
 		DisableVhostNet:                true,
 		GuestHookPath:                  h.guestHookPath(),
@@ -1655,9 +1662,32 @@ func checkConfig(config oci.RuntimeConfig) error {
 		return err
 	}
 
+	coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO
+	machineType := config.HypervisorConfig.HypervisorMachineType
+	if err := checkPCIeConfig(coldPlugVFIO, machineType); err != nil {
+		return err
+	}
+
 	return nil
 }
 
+// checkPCIeConfig ensures the PCIe configuration is valid.
+// Only allow one of the following settings for cold-plug:
+// no-port, root-port, switch-port
+func checkPCIeConfig(vfioPort hv.PCIePort, machineType string) error {
+	// Currently only QEMU q35 supports advanced PCIe topologies
+	// firecracker, dragonball do not have right now any PCIe support
+	if machineType != "q35" {
+		return nil
+	}
+	if vfioPort == hv.NoPort || vfioPort == hv.RootPort || vfioPort == hv.SwitchPort {
+		return nil
+	}
+
+	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s",
+		vfioPort, hv.NoPort, hv.RootPort, hv.SwitchPort)
+}
+
 // checkNetNsConfig performs sanity checks on disable_new_netns config.
 // Because it is an expert option and conflicts with some other common configs.
 func checkNetNsConfig(config oci.RuntimeConfig) error {
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index 644904f75..0eb866bb0 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -623,7 +623,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 
 	// If we have a confidential guest we need to cold-plug the PCIe VFIO devices
 	// until we have TDISP/IDE PCIe support.
-	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO == hv.RootPort)
+	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
 	var devs []config.DeviceInfo
 	for cnt, containers := range sandboxConfig.Containers {
 		for dev, device := range containers.DeviceInfos {

From 8495f830b77f610f5cfea1325c256a3730a71f23 Mon Sep 17 00:00:00 2001
From: Archana Shinde <archana.m.shinde@intel.com>
Date: Fri, 28 Apr 2023 00:35:18 -0700
Subject: [PATCH 044/150] cross-compile: Include documentation and
 configuration for cross-compile

`cross` is an open source tool that provides zero-setup cross compile
for rust binaries. Add documentation on this tool for compiling
kata-ctl tool and Cross.toml file that provides required configuration
for installing dependencies for various targets.
This is pretty useful for a developer to make sure code compiles and
passes checks for various architectures.

Fixes: #6765

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
---
 src/tools/kata-ctl/Cross.toml | 12 ++++++++++++
 src/tools/kata-ctl/README.md  | 31 +++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 src/tools/kata-ctl/Cross.toml

diff --git a/src/tools/kata-ctl/Cross.toml b/src/tools/kata-ctl/Cross.toml
new file mode 100644
index 000000000..88f18e96e
--- /dev/null
+++ b/src/tools/kata-ctl/Cross.toml
@@ -0,0 +1,12 @@
+[target.s390x-unknown-linux-gnu]
+pre-build = ["dpkg --add-architecture s390x && apt-get update && apt-get install -y libssl-dev:s390x"]
+
+[target.aarch64-unknown-linux-musl]
+pre-build = ["dpkg --add-architecture arm64 && apt-get update && apt-get install -y libssl-dev:arm64"]
+
+[target.x86-64-unknown-linux-musl]
+pre-build = ["dpkg --add-architecture amd64 && apt-get update && apt-get install -y libssl-dev:amd64"]
+
+# Powerpc compile seems to be broken, due to `ring` crate not being supported on powerpc.
+[target.powerpc64le-unknown-linux-gnu]
+pre-build = ["dpkg --add-architecture ppc64le && apt-get update && apt-get install -y libssl-dev:ppc64le"]
diff --git a/src/tools/kata-ctl/README.md b/src/tools/kata-ctl/README.md
index bf908f60d..181e66db7 100644
--- a/src/tools/kata-ctl/README.md
+++ b/src/tools/kata-ctl/README.md
@@ -47,3 +47,34 @@ For a usage statement, run:
 ```bash
 $ kata-ctl --help
 ```
+
+## Cross-builds
+
+For developers that want to build and test the `kata-ctl` tool on various architectures,
+the makefile included does have support for that. This would however, require installing 
+the cross compile toolchain for the target architecture on the host along with required libraries.
+
+[Cross](https://github.com/cross-rs/cross) is an open source tool that offers zero setup
+cross compile and requires no changes to the system installation for cross-compiling
+rust binaries. It makes use of docker containers for cross-compilation.
+
+You can install cross with:
+```
+cargo install -f cross
+```
+
+`cross` relies on `docker` or `podman`. For dependencies take a look at: https://github.com/cross-rs/cross#dependencies
+
+There is an included `cross` configuration file [Cross.yaml](./Cross.toml) that can be used
+to compile `kata-ctl` for various targets. This configuration helps install required
+dependencies inside a docker container.
+
+For example, to compile for target `s390x-unknown-linux-gnu` included in `Cross.yaml` simple run:
+```
+cross build --target=s390x-unknown-linux-gnu
+```
+
+You may also need to add the target on your host system prior to the above step as:
+```
+rustup target add s390x-unknown-linux-gnu
+``` 

From a4c0303d899361ff2fffaffa620877d0ff2c8fcf Mon Sep 17 00:00:00 2001
From: Eduardo Berrocal <eduardo.berrocal@intel.com>
Date: Sun, 7 May 2023 00:17:36 -0700
Subject: [PATCH 045/150] virtcontainers: Fixed static checks for improved test
 coverage for fc.go

Expanded tests on fc_test.go to cover more lines of code. Coverage went from 4.6% to 18.5%.
Fixed very simple static check fail on line 202.

Fixes: #266

Signed-off-by: Eduardo Berrocal <eduardo.berrocal@intel.com>
---
 src/runtime/virtcontainers/fc_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/runtime/virtcontainers/fc_test.go b/src/runtime/virtcontainers/fc_test.go
index 361ca0962..5550b6895 100644
--- a/src/runtime/virtcontainers/fc_test.go
+++ b/src/runtime/virtcontainers/fc_test.go
@@ -199,8 +199,7 @@ func TestFCGetTotalMemoryMB(t *testing.T) {
 	fc := firecracker{}
 	ctx := context.Background()
 
-	var initialMemSize uint32
-	initialMemSize = 1024
+	var initialMemSize uint32 = 1024
 
 	fc.config.MemorySize = 1024
 	memSize := fc.GetTotalMemoryMB(ctx)

From b0e6a094be16b58fe3c04639955ede973a826e38 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Mon, 17 Apr 2023 23:06:17 +0000
Subject: [PATCH 046/150] packaging: Add sev kernel build capability

Adding code that builds sev kernel.

Fixes: #6572

Signed-off-by: Unmesh Deodhar <udeodhar@amd.com>
---
 .../kata-deploy/local-build/Makefile          |  3 +
 .../local-build/kata-deploy-binaries.sh       | 70 ++++++++++++++++---
 2 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index 82356f1fa..88de4399a 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -72,6 +72,9 @@ kernel-experimental-tarball:
 kernel-tdx-experimental-tarball:
 	${MAKE} $@-build
 
+kernel-sev-tarball:
+	${MAKE} $@-build
+
 nydus-tarball:
 	${MAKE} $@-build
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 902b067c9..7b0227442 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -82,6 +82,7 @@ options:
 	kernel-dragonball-experimental
 	kernel-experimental
 	kernel-tdx-experimental
+	kernel-sev-tarball
 	kernel-gpu
 	kernel-gpu-snp
 	kernel-gpu-tdx-experimental
@@ -175,7 +176,45 @@ install_initrd() {
 		&& return 0
 
 	info "Create initrd"
-	"${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}"
+	"${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
+}
+
+#Install guest initrd for sev
+install_initrd_sev() {
+	install_initrd "initrd-sev" "sev"
+}
+
+#Install kernel component helper
+install_cached_kernel_tarball_component() {
+	local kernel_name=${1}
+
+	install_cached_tarball_component \
+		"${kernel_name}" \
+		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}-${kernel_kata_config_version}" \
+		"$(get_kernel_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		|| return 1
+	
+	if [[ "${kernel_name}" != "kernel-sev" ]]; then
+		return 0
+	fi
+
+	# SEV specific code path
+	install_cached_tarball_component \
+		"${kernel_name}" \
+		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}-${kernel_kata_config_version}" \
+		"$(get_kernel_image_name)" \
+		"kata-static-kernel-sev-modules.tar.xz" \
+		"${workdir}/kata-static-kernel-sev-modules.tar.xz" \
+		|| return 1
+
+	mkdir -p "${module_dir}"
+	tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C  "${module_dir}" && return 0
+
+	return 1
 }
 
 #Install kernel asset
@@ -185,16 +224,14 @@ install_kernel_helper() {
 	local extra_cmd=${3}
 
 	export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})"
-	local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
+	export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
+	local module_dir=""
 
-	install_cached_tarball_component \
-		"${kernel_name}" \
-		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
-		"${kernel_version}-${kernel_kata_config_version}" \
-		"$(get_kernel_image_name)" \
-		"${final_tarball_name}" \
-		"${final_tarball_path}" \
-		&& return 0
+	if [[ "${kernel_name}" == "kernel-sev" ]]; then
+		job_name="kata-containers-main-${kernel_name}-sev-$(uname -m)"
+	fi
+
+	install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0
 
 	info "build ${kernel_name}"
 	info "Kernel version ${kernel_version}"
@@ -264,6 +301,17 @@ install_kernel_tdx_experimental() {
 		"-x tdx -u ${kernel_url}"
 }
 
+#Install sev kernel asset
+install_kernel_sev() {
+	info "build sev kernel"
+	local kernel_url="$(get_from_kata_deps assets.kernel.sev.url)"
+
+	install_kernel_helper \
+		"assets.kernel.sev.version" \
+		"kernel-sev" \
+		"-x sev -u ${kernel_url}"
+}
+
 install_qemu_helper() {
 	local qemu_repo_yaml_path="${1}"
 	local qemu_version_yaml_path="${2}"
@@ -481,6 +529,8 @@ handle_build() {
 
 	kernel-tdx-experimental) install_kernel_tdx_experimental ;;
 
+	kernel-sev) install_kernel_sev ;;
+
 	kernel-gpu) install_kernel_gpu ;;
 
 	kernel-gpu-snp) install_kernel_gpu_snp;;

From 022a33de9269345156e3266d6f8284849567014b Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Wed, 3 May 2023 16:16:57 +0200
Subject: [PATCH 047/150] agent: Add context to errors when AgentConfig file is
 missing

When the agent config file is missing, the panic message says "no such file or
directory" but doesn't inform the user about which file was missing. Add
context to the parsing (with filename) and to the from_config_file() calls
(with information where the path is coming from).

Fixes: #6771
Depends-on: github.com/kata-containers/tests#5627
Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 src/agent/src/config.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs
index 15bef0dc8..b503dc849 100644
--- a/src/agent/src/config.rs
+++ b/src/agent/src/config.rs
@@ -200,7 +200,7 @@ impl AgentConfig {
         let config_position = args.iter().position(|a| a == "--config" || a == "-c");
         if let Some(config_position) = config_position {
             if let Some(config_file) = args.get(config_position + 1) {
-                return AgentConfig::from_config_file(config_file);
+                return AgentConfig::from_config_file(config_file).context("AgentConfig from args");
             } else {
                 panic!("The config argument wasn't formed properly: {:?}", args);
             }
@@ -216,7 +216,8 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                return AgentConfig::from_config_file(&config_file);
+                return AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline");
             }
 
             // parse cmdline flags
@@ -304,7 +305,8 @@ impl AgentConfig {
 
     #[instrument]
     pub fn from_config_file(file: &str) -> Result<AgentConfig> {
-        let config = fs::read_to_string(file)?;
+        let config = fs::read_to_string(file)
+            .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
 

From e1f3b871cdef1ae560982e710de3835672cbf80c Mon Sep 17 00:00:00 2001
From: "James O. D. Hunt" <james.o.hunt@intel.com>
Date: Tue, 9 May 2023 09:23:31 +0100
Subject: [PATCH 048/150] docs: Mark snap installation method as unmaintained

The snap package is no longer being maintained so update the docs to
warn readers.

We'll remove the snap installation docs in a few weeks.

See: #6769.
Fixes: #6793.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
---
 docs/install/README.md                  | 21 ++++++++++++++---
 docs/install/snap-installation-guide.md | 30 +++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/docs/install/README.md b/docs/install/README.md
index 0ed42d87f..8a86bc7c8 100644
--- a/docs/install/README.md
+++ b/docs/install/README.md
@@ -19,7 +19,7 @@ Packaged installation methods uses your distribution's native package format (su
 |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|
 | [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | 
 | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users.                                                                   |
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 |
+| ~~[Using snap](#snap-installation)~~                     | ~~Easy to install~~                                                                              | ~~yes~~               | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~                                                 |
 | [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 |
 | [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      |
 | [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.                                                              |
@@ -44,9 +44,24 @@ Kata packages are provided by official distribution repositories for:
 
 ### Snap Installation
 
-The snap installation is available for all distributions which support `snapd`.
+> **WARNING:**
+>
+> The Snap package method is **unmaintained** and only provides an old
+> version of Kata Containers:
+> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
+> provides Kata Containers
+> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
+> but the latest stable Kata Containers release at the time of writing is
+> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
+>
+> We recommend strongly that you switch to an alternative Kata Containers installation method.
+>
+> See: https://github.com/kata-containers/kata-containers/issues/6769
+> for further details.
 
-[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io.
+~~The snap installation is available for all distributions which support `snapd`.~~
+
+~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~
 
 ### Automatic Installation
 
diff --git a/docs/install/snap-installation-guide.md b/docs/install/snap-installation-guide.md
index 0f22c2211..acf2e637e 100644
--- a/docs/install/snap-installation-guide.md
+++ b/docs/install/snap-installation-guide.md
@@ -1,5 +1,20 @@
 # Kata Containers snap package
 
+> **WARNING:**
+>
+> The Snap package method is **unmaintained** and only provides an old
+> version of Kata Containers:
+> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
+> provides Kata Containers
+> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
+> but the latest stable Kata Containers release at the time of writing is
+> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
+>
+> We recommend strongly that you switch to an alternative Kata Containers installation method.
+>
+> See: https://github.com/kata-containers/kata-containers/issues/6769
+> for further details.
+
 ## Install Kata Containers
 
 Kata Containers can be installed in any Linux distribution that supports
@@ -7,6 +22,21 @@ Kata Containers can be installed in any Linux distribution that supports
 
 Run the following command to install **Kata Containers**:
 
+> **WARNING:**
+>
+> The Snap package method is **unmaintained** and only provides an old
+> version of Kata Containers:
+> The [latest Kata Containers snap](https://snapcraft.io/kata-containers)
+> provides Kata Containers
+> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2)
+> but the latest stable Kata Containers release at the time of writing is
+> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0).
+>
+> We recommend strongly that you switch to an alternative Kata Containers installation method.
+>
+> See: https://github.com/kata-containers/kata-containers/issues/6769
+> for further details.
+
 ```sh
 $ sudo snap install kata-containers --stable --classic
 ```

From b87820ee8c38ba0ca442e981b025142b3d73f86d Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Mon, 17 Apr 2023 23:19:25 +0000
Subject: [PATCH 049/150] packaging: add support to build initrd for sev

We need special initrd for SEV. The work on SEV initrd is based on
Ubuntu. Thus, adding another entry in versions.yaml
This binary will have '-sev' suffix to distinguish it from the generic
binary.

Fixes: #6572

Signed-Off-By: Unmesh Deodhar <udeodhar@amd.com>
---
 tools/packaging/guest-image/build_image.sh    | 15 ++++++-
 .../kata-deploy/local-build/Makefile          |  4 ++
 .../local-build/kata-deploy-binaries.sh       | 45 ++++---------------
 versions.yaml                                 |  3 ++
 4 files changed, 30 insertions(+), 37 deletions(-)

diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh
index 914b72a79..d602b85c0 100755
--- a/tools/packaging/guest-image/build_image.sh
+++ b/tools/packaging/guest-image/build_image.sh
@@ -22,6 +22,8 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
 export GOPATH=${GOPATH:-${HOME}/go}
 
 arch_target="$(uname -m)"
+final_initrd_name="kata-containers-initrd"
+image_initrd_extension=".img"
 
 build_initrd() {
 	info "Build initrd"
@@ -37,7 +39,7 @@ build_initrd() {
 	mv "kata-containers-initrd.img" "${install_dir}/${initrd_name}"
 	(
 		cd "${install_dir}"
-		ln -sf "${initrd_name}" kata-containers-initrd.img
+		ln -sf "${initrd_name}" "${final_initrd_name}${image_initrd_extension}"
 	)
 }
 
@@ -71,6 +73,7 @@ Options:
  --imagetype=${image_type}
  --prefix=${prefix}
  --destdir=${destdir}
+ --image_initrd_suffix=${image_initrd_suffix}
 EOF
 
 	exit "${return_code}"
@@ -80,6 +83,7 @@ main() {
 	image_type=image
 	destdir="$PWD"
 	prefix="/opt/kata"
+	image_initrd_suffix=""
 	builddir="${PWD}"
 	while getopts "h-:" opt; do
 		case "$opt" in
@@ -99,6 +103,15 @@ main() {
 				initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version")
 				initrd_name="kata-${initrd_distro}-${initrd_os_version}.${image_type}"
 				;;
+			image_initrd_suffix=*)
+				image_initrd_suffix=${OPTARG#*=}
+				if [ "${image_initrd_suffix}" == "sev" ]; then
+					initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name")
+					initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version")
+					initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}"
+					final_initrd_name="${final_initrd_name}-${image_initrd_suffix}"
+				fi
+				;;
 			prefix=*)
 				prefix=${OPTARG#*=}
 				;;
diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index 88de4399a..5e20f9108 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -39,6 +39,7 @@ all: serial-targets \
 serial-targets:
 	${MAKE} -f $(MK_PATH) -j 1 V= \
 	rootfs-image-tarball \
+	rootfs-initrd-sev-tarball \
 	rootfs-initrd-tarball \
 	cloud-hypervisor-tarball
 
@@ -87,6 +88,9 @@ qemu-tdx-experimental-tarball:
 rootfs-image-tarball:
 	${MAKE} $@-build
 
+rootfs-initrd-sev-tarball: kernel-sev-tarball
+	${MAKE} $@-build
+
 rootfs-initrd-tarball:
 	${MAKE} $@-build
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 7b0227442..f551fc8d9 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -91,6 +91,7 @@ options:
 	qemu-tdx-experimental
 	rootfs-image
 	rootfs-initrd
+	rootfs-initrd-sev
 	shim-v2
 	tdvf
 	virtiofsd
@@ -155,8 +156,10 @@ install_image() {
 
 #Install guest initrd
 install_initrd() {
-	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-initrd-$(uname -m)/${cached_artifacts_path}"
-	local component="rootfs-initrd"
+	local initrd_type="${1:-""}"
+	local initrd_suffix="${2:-""}"
+	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}"
+	local component="rootfs-${initrd_type}"
 
 	local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")"
 	local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")"
@@ -169,7 +172,7 @@ install_initrd() {
 	install_cached_tarball_component \
 		"${component}" \
 		"${jenkins}" \
-		"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-initrd" \
+		"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${initrd_type}" \
 		"" \
 		"${final_tarball_name}" \
 		"${final_tarball_path}" \
@@ -184,39 +187,6 @@ install_initrd_sev() {
 	install_initrd "initrd-sev" "sev"
 }
 
-#Install kernel component helper
-install_cached_kernel_tarball_component() {
-	local kernel_name=${1}
-
-	install_cached_tarball_component \
-		"${kernel_name}" \
-		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
-		"${kernel_version}-${kernel_kata_config_version}" \
-		"$(get_kernel_image_name)" \
-		"${final_tarball_name}" \
-		"${final_tarball_path}" \
-		|| return 1
-	
-	if [[ "${kernel_name}" != "kernel-sev" ]]; then
-		return 0
-	fi
-
-	# SEV specific code path
-	install_cached_tarball_component \
-		"${kernel_name}" \
-		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
-		"${kernel_version}-${kernel_kata_config_version}" \
-		"$(get_kernel_image_name)" \
-		"kata-static-kernel-sev-modules.tar.xz" \
-		"${workdir}/kata-static-kernel-sev-modules.tar.xz" \
-		|| return 1
-
-	mkdir -p "${module_dir}"
-	tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C  "${module_dir}" && return 0
-
-	return 1
-}
-
 #Install kernel asset
 install_kernel_helper() {
 	local kernel_version_yaml_path="${1}"
@@ -504,6 +474,7 @@ handle_build() {
 		install_firecracker
 		install_image
 		install_initrd
+		install_initrd_sev
 		install_kernel
 		install_kernel_dragonball_experimental
 		install_kernel_tdx_experimental
@@ -545,6 +516,8 @@ handle_build() {
 
 	rootfs-initrd) install_initrd ;;
 
+	rootfs-initrd-sev) install_initrd_sev ;;
+	
 	shim-v2) install_shimv2 ;;
 
 	tdvf) install_tdvf ;;
diff --git a/versions.yaml b/versions.yaml
index c222499d9..d95946749 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -156,6 +156,9 @@ assets:
       x86_64:
         name: *default-initrd-name
         version: *default-initrd-version
+        sev:
+          name: *glibc-initrd-name
+          version: *glibc-initrd-version
 
   kernel:
     description: "Linux kernel optimised for virtual machines"

From 12c5ef9020cdcf6dbb265b93e3fe1d714536e6ea Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Fri, 21 Apr 2023 02:34:23 +0000
Subject: [PATCH 050/150] packaging: add support to build OVMF for SEV

SEV requires special OVMF to work with kernel hashes.
Thus, adding changes that builds this custom OVMF for SEV.

Fixes: #6572

Signed-Off-By: Unmesh Deodhar <udeodhar@amd.com>
---
 src/runtime/Makefile                                     | 2 ++
 tools/packaging/kata-deploy/local-build/Makefile         | 4 ++++
 .../kata-deploy/local-build/kata-deploy-binaries.sh      | 9 +++++++++
 tools/packaging/static-build/ovmf/build-ovmf.sh          | 6 +++++-
 versions.yaml                                            | 2 +-
 5 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index aa03ce852..f2389c73c 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -126,6 +126,8 @@ FIRMWAREVOLUMEPATH :=
 FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd
 FIRMWARETDVFVOLUMEPATH :=
 
+FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
+
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
 
diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index 5e20f9108..91d76dab0 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -30,6 +30,7 @@ all: serial-targets \
 	kernel-gpu-snp-tarball \
 	kernel-gpu-tdx-experimental-tarball \
 	nydus-tarball \
+	ovmf-sev-tarball \
 	qemu-tarball \
 	qemu-tdx-experimental-tarball \
 	shim-v2-tarball \
@@ -79,6 +80,9 @@ kernel-sev-tarball:
 nydus-tarball:
 	${MAKE} $@-build
 
+ovmf-sev-tarball:
+	${MAKE} $@-build
+
 qemu-tarball:
 	${MAKE} $@-build
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index f551fc8d9..25e05da55 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -87,6 +87,7 @@ options:
 	kernel-gpu-snp
 	kernel-gpu-tdx-experimental
 	nydus
+	ovmf-sev
 	qemu
 	qemu-tdx-experimental
 	rootfs-image
@@ -453,6 +454,11 @@ install_tdvf() {
 	install_ovmf "tdx" "edk2-tdx.tar.gz"
 }
 
+# Install OVMF SEV
+install_ovmf_sev() {
+	install_ovmf "sev" "edk2-sev.tar.gz"
+}
+
 get_kata_version() {
 	local v
 	v=$(cat "${version_file}")
@@ -479,6 +485,7 @@ handle_build() {
 		install_kernel_dragonball_experimental
 		install_kernel_tdx_experimental
 		install_nydus
+		install_ovmf_sev
 		install_qemu
 		install_qemu_tdx_experimental
 		install_shimv2
@@ -502,6 +509,8 @@ handle_build() {
 
 	kernel-sev) install_kernel_sev ;;
 
+	ovmf-sev) install_ovmf_sev ;;
+
 	kernel-gpu) install_kernel_gpu ;;
 
 	kernel-gpu-snp) install_kernel_gpu_snp;;
diff --git a/tools/packaging/static-build/ovmf/build-ovmf.sh b/tools/packaging/static-build/ovmf/build-ovmf.sh
index ebdf0669f..c0e7d26e2 100755
--- a/tools/packaging/static-build/ovmf/build-ovmf.sh
+++ b/tools/packaging/static-build/ovmf/build-ovmf.sh
@@ -82,7 +82,11 @@ if [ "${ovmf_build}" == "tdx" ]; then
 fi
 
 mkdir -p "${install_dir}"
-install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF.fd "${install_dir}"
+if [ "${ovmf_build}" == "sev" ]; then
+	install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF.fd "${install_dir}/AMDSEV.fd"
+else
+	install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF.fd "${install_dir}"
+fi
 if [ "${ovmf_build}" == "tdx" ]; then
 	install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF_CODE.fd ${install_dir}
 	install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF_VARS.fd ${install_dir}
diff --git a/versions.yaml b/versions.yaml
index d95946749..4f6e81e4b 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -278,7 +278,7 @@ externals:
       package_output_dir: "OvmfX64"
     sev:
       description: "AmdSev build needed for SEV measured direct boot."
-      version: "edk2-stable202202"
+      version: "edk2-stable202302"
       package: "OvmfPkg/AmdSev/AmdSevX64.dsc"
       package_output_dir: "AmdSev"
     tdx:

From 76ae7a3abec5a7fb4864141e735a0a29dfb4d920 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Mon, 24 Apr 2023 17:04:39 +0000
Subject: [PATCH 051/150] packaging: adding caching capability for kernel

The SEV initrd build requires kernel modules.
So, for SEV case, we need to cache kernel modules tarball in
addition to kernel tarball.

Fixes: #6572

Signed-Off-By: Unmesh Deodhar <udeodhar@amd.com>
---
 .../local-build/kata-deploy-binaries.sh       | 37 ++++++++++++++++++-
 .../static-build/cache_components_main.sh     | 17 ++++++++-
 2 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 25e05da55..81784379a 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -188,6 +188,39 @@ install_initrd_sev() {
 	install_initrd "initrd-sev" "sev"
 }
 
+#Install kernel component helper
+install_cached_kernel_tarball_component() {
+	local kernel_name=${1}
+
+	install_cached_tarball_component \
+		"${kernel_name}" \
+		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}-${kernel_kata_config_version}" \
+		"$(get_kernel_image_name)" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		|| return 1
+	
+	if [[ "${kernel_name}" != "kernel-sev" ]]; then
+		return 0
+	fi
+
+	# SEV specific code path
+	install_cached_tarball_component \
+		"${kernel_name}" \
+		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
+		"${kernel_version}-${kernel_kata_config_version}" \
+		"$(get_kernel_image_name)" \
+		"kata-static-kernel-sev-modules.tar.xz" \
+		"${workdir}/kata-static-kernel-sev-modules.tar.xz" \
+		|| return 1
+
+	mkdir -p "${module_dir}"
+	tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C  "${module_dir}" && return 0
+
+	return 1
+}
+
 #Install kernel asset
 install_kernel_helper() {
 	local kernel_version_yaml_path="${1}"
@@ -199,7 +232,9 @@ install_kernel_helper() {
 	local module_dir=""
 
 	if [[ "${kernel_name}" == "kernel-sev" ]]; then
-		job_name="kata-containers-main-${kernel_name}-sev-$(uname -m)"
+		kernel_version="$(get_from_kata_deps assets.kernel.sev.version)"
+		default_patches_dir="${repo_root_dir}/tools/packaging/kernel/patches"
+		module_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kernel-sev/builddir/kata-linux-${kernel_version#v}-${kernel_kata_config_version}/lib/modules/${kernel_version#v}"
 	fi
 
 	install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0
diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index 0e8a0120f..fc0019c7f 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -33,8 +33,21 @@ cache_kernel_artifacts() {
 	local kernel_tarball_name="kata-static-${KERNEL_FLAVOUR}.tar.xz"
 	local current_kernel_image="$(get_kernel_image_name)"
 	local current_kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
-	local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")-${current_kernel_kata_config_version}"
-	create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}"
+	local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")"
+	local kernel_modules_tarball_path="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel-sev-modules.tar.xz"
+
+	if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then
+		current_kernel_version="$(get_from_kata_deps "assets.kernel.sev.version")"
+	fi
+
+	create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}"
+	if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then
+		module_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kernel-sev/builddir/kata-linux-${current_kernel_version#v}-${current_kernel_kata_config_version}/lib/modules/${current_kernel_version#v}"
+		if [ ! -f "${kernel_modules_tarball_path}" ]; then
+			tar cvfJ "${kernel_modules_tarball_path}" "${module_dir}/kernel/drivers/virt/coco/efi_secret/"
+		fi
+		create_cache_asset "kata-static-kernel-sev-modules.tar.xz" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}"
+	fi
 }
 
 cache_nydus_artifacts() {

From af18806a8d45932f2a99e3a87f9ccdb67469e8e3 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Mon, 24 Apr 2023 21:13:11 +0000
Subject: [PATCH 052/150] static-build: Add caching support to sev ovmf

SEV requires special OVMF.
Now that we have ability to build this custom OVMF, let's optimize
it by caching so that we don't have to build it for every run.

Fixes: sev: #6572

Signed-Off-By: Unmesh Deodhar <udeodhar@amd.com>
---
 tools/packaging/static-build/cache_components_main.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index fc0019c7f..1c46c1e01 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -59,6 +59,7 @@ cache_nydus_artifacts() {
 cache_ovmf_artifacts() {
 	local current_ovmf_version="$(get_from_kata_deps "externals.ovmf.${OVMF_FLAVOUR}.version")"
 	[ "${OVMF_FLAVOUR}" == "tdx" ] && OVMF_FLAVOUR="tdvf"
+	[ "${OVMF_FLAVOUR}" == "sev" ] && OVMF_FLAVOUR="ovmf-sev"
 	local ovmf_tarball_name="kata-static-${OVMF_FLAVOUR}.tar.xz"
 	local current_ovmf_image="$(get_ovmf_image_name)"
 	create_cache_asset "${ovmf_tarball_name}" "${current_ovmf_version}" "${current_ovmf_image}"

From 813e4c576fbd1f174b7353ecb93abeba3447b9b7 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Thu, 27 Apr 2023 20:53:55 +0000
Subject: [PATCH 053/150] runtimeClasses: add sev runtime class

Adding kata-qemu-sev runtime class.

Fixes: #6572

Signed-off-by: Unmesh Deodhar <udeodhar@amd.com>
---
 .../runtimeclasses/kata-runtimeClasses.yaml         | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
index f65ec6b0a..00197ed3b 100644
--- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
@@ -14,6 +14,19 @@ scheduling:
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-sev
+handler: kata-qemu-sev
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+    nodeSelector:
+        katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
 metadata:
     name: kata-qemu-tdx
 handler: kata-qemu-tdx

From fb9c1fc36e56602702d22b7cfb06fe1a9639e5f7 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Fri, 28 Apr 2023 14:20:55 -0500
Subject: [PATCH 054/150] runtime: Add qemu-sev config

Adding config file that can be used with qemu-sev runtime class.
Since SEV has limited hotplug support, increase
the pod overhead to account for fixed resource usage.

Fixes: #6572

Signed-off-by: Unmesh Deodhar <udeodhar@amd.com>
---
 src/runtime/Makefile                          |  33 +
 .../config/configuration-qemu-sev.toml.in     | 650 ++++++++++++++++++
 2 files changed, 683 insertions(+)
 create mode 100644 src/runtime/config/configuration-qemu-sev.toml.in

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index f2389c73c..ee3896c77 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -57,6 +57,7 @@ BIN_PREFIX = $(PROJECT_TYPE)
 PROJECT_DIR = $(PROJECT_TAG)
 IMAGENAME = $(PROJECT_TAG).img
 INITRDNAME = $(PROJECT_TAG)-initrd.img
+INITRDSEVNAME = $(PROJECT_TAG)-initrd-sev.img
 
 TARGET = $(BIN_PREFIX)-runtime
 RUNTIME_OUTPUT = $(CURDIR)/$(TARGET)
@@ -95,6 +96,7 @@ GENERATED_VARS = \
 		VERSION \
 		CONFIG_ACRN_IN \
 		CONFIG_QEMU_IN \
+		CONFIG_QEMU_SEV_IN \
 		CONFIG_QEMU_TDX_IN \
 		CONFIG_QEMU_GPU_IN \
 		CONFIG_CLH_IN \
@@ -114,6 +116,7 @@ KERNELDIR := $(PKGDATADIR)
 
 IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
 INITRDPATH := $(PKGDATADIR)/$(INITRDNAME)
+INITRDSEVPATH := $(PKGDATADIR)/$(INITRDSEVNAME)
 
 ROOTFSTYPE_EXT4 := \"ext4\"
 ROOTFSTYPE_XFS := \"xfs\"
@@ -206,6 +209,7 @@ DEFDISABLEBLOCK := false
 DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p
+DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
 ifeq ($(ARCH),ppc64le)
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/qemu/virtiofsd
@@ -238,6 +242,9 @@ DEFSTATICRESOURCEMGMT ?= false
 
 DEFBINDMOUNTS := []
 
+# Image Service Offload
+DEFSERVICEOFFLOAD ?= false
+
 SED = sed
 
 CLI_DIR = cmd
@@ -288,6 +295,18 @@ ifneq (,$(QEMUCMD))
 
     CONFIGS += $(CONFIG_QEMU_TDX)
 
+    CONFIG_FILE_QEMU_SEV = configuration-qemu-sev.toml
+    CONFIG_QEMU_SEV = config/$(CONFIG_FILE_QEMU_SEV)
+    CONFIG_QEMU_SEV_IN = $(CONFIG_QEMU_SEV).in
+
+    CONFIG_PATH_QEMU_SEV = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SEV))
+    CONFIG_PATHS += $(CONFIG_PATH_QEMU_SEV)
+
+    SYSCONFIG_QEMU_SEV = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SEV))
+    SYSCONFIG_PATHS_SEV += $(SYSCONFIG_QEMU_SEV)
+
+    CONFIGS += $(CONFIG_QEMU_SEV)
+
     CONFIG_FILE_QEMU_GPU = configuration-qemu-gpu.toml
     CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU)
     CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in
@@ -303,6 +322,10 @@ ifneq (,$(QEMUCMD))
     KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE))
     KERNELPATH = $(KERNELDIR)/$(KERNELNAME)
 
+    KERNELSEVTYPE = compressed
+    KERNELSEVNAME = $(call MAKE_KERNEL_SEV_NAME,$(KERNELSEVTYPE))
+    KERNELSEVPATH = $(KERNELDIR)/$(KERNELSEVNAME)
+
     KERNELTDXTYPE = compressed
     KERNELTDXNAME = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE))
     KERNELTDXPATH = $(KERNELDIR)/$(KERNELTDXNAME)
@@ -451,7 +474,9 @@ USER_VARS += SYSCONFIG
 USER_VARS += IMAGENAME
 USER_VARS += IMAGEPATH
 USER_VARS += INITRDNAME
+USER_VARS += INITRDSEVNAME
 USER_VARS += INITRDPATH
+USER_VARS += INITRDSEVPATH
 USER_VARS += DEFROOTFSTYPE
 USER_VARS += MACHINETYPE
 USER_VARS += KERNELDIR
@@ -461,11 +486,13 @@ USER_VARS += KERNELTYPE_ACRN
 USER_VARS += KERNELTYPE_CLH
 USER_VARS += KERNELPATH_ACRN
 USER_VARS += KERNELPATH
+USER_VARS += KERNELSEVPATH
 USER_VARS += KERNELTDXPATH
 USER_VARS += KERNELPATH_CLH
 USER_VARS += KERNELPATH_FC
 USER_VARS += KERNELVIRTIOFSPATH
 USER_VARS += FIRMWAREPATH
+USER_VARS += FIRMWARESEVPATH
 USER_VARS += FIRMWARETDVFPATH
 USER_VARS += FIRMWAREVOLUMEPATH
 USER_VARS += FIRMWARETDVFVOLUMEPATH
@@ -525,6 +552,7 @@ USER_VARS += DEFBLOCKDEVICEAIO_QEMU
 USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_TDX_VIRTIOFS
+USER_VARS += DEFSHAREDFS_QEMU_SEV_VIRTIOFS
 USER_VARS += DEFVIRTIOFSDAEMON
 USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
 USER_VARS += DEFVIRTIOFSCACHESIZE
@@ -546,6 +574,7 @@ USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFSTATICRESOURCEMGMT
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
 USER_VARS += DEFBINDMOUNTS
+USER_VARS += DEFSERVICEOFFLOAD
 USER_VARS += DEFVFIOMODE
 USER_VARS += BUILDFLAGS
 
@@ -630,6 +659,10 @@ define MAKE_KERNEL_VIRTIOFS_NAME
 $(if $(findstring uncompressed,$1),vmlinux-virtiofs.container,vmlinuz-virtiofs.container)
 endef
 
+define MAKE_KERNEL_SEV_NAME
+$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container)
+endef
+
 define MAKE_KERNEL_TDX_NAME
 $(if $(findstring uncompressed,$1),vmlinux-tdx.container,vmlinuz-tdx.container)
 endef
diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in
new file mode 100644
index 000000000..8dba6813e
--- /dev/null
+++ b/src/runtime/config/configuration-qemu-sev.toml.in
@@ -0,0 +1,650 @@
+# Copyright 2022 Advanced Micro Devices, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_SEV_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUPATH@"
+kernel = "@KERNELSEVPATH@"
+initrd = "@INITRDSEVPATH@"
+machine_type = "@MACHINETYPE@"
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+confidential_guest = true
+
+# Enable running QEMU VMM as a non-root user.
+# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+# rootless = true
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWARESEVPATH@"
+
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWAREVOLUMEPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators="@MACHINEACCELERATORS@"
+
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@"
+
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+#enable_virtio_mem = true
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-9p
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_QEMU_SEV_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+#enable_iommu_platform = true
+
+# List of valid annotations values for the vhost user store path
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@
+valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+#file_mem_backend = "@DEFFILEMEMBACKEND@"
+
+# List of valid annotations values for the file_mem_backend annotation
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@
+valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = []
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+#
+# Default is false
+#disable_image_nvdimm = true
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+#pcie_root_port = 2
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance.
+#disable_vhost_net = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
+
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest’s memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+#guest_memory_dump_path="/var/crash/kata"
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+#guest_memory_dump_paging=false
+
+# Enable swap in the guest. Default false.
+# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device
+# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness")
+# is bigger than 0.
+# The size of the swap device should be
+# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes.
+# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.
+# If swap_in_bytes and memory_limit_in_bytes is not set, the size should
+# be default_memory.
+#enable_guest_swap = true
+
+# use legacy serial for guest console if available and implemented for architecture. Default false
+#use_legacy_serial = true
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+#template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 90)
+dial_timeout = 90
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+service_offload = @DEFSERVICEOFFLOAD@
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>
\ No newline at end of file

From 4770d3064a008c4760f21b159f3719f4554c4235 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Thu, 4 May 2023 17:08:21 +0000
Subject: [PATCH 055/150] gha: Build and ship SEV kernel.

SEV requires custom kernel arguments when building.
Thus, adding it to the test and release process.

Fixes: #6572

Signed-off-by: Unmesh Deodhar <udeodhar@amd.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index 5942a5d79..5ea73c246 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -19,6 +19,7 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - kernel-sev
           - kernel-dragonball-experimental
           - kernel-tdx-experimental
           - kernel-gpu

From 45fa36692688d8b419a5136001c8d9284d79b5b4 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Thu, 4 May 2023 17:30:12 +0000
Subject: [PATCH 056/150] gha: Build and ship SEV OVMF

SEV requires special OVMF to work. Thus, building that for test and release.

Fixes: #6572

Signed-Off-By: Unmesh Deodhar <udeodhar@amd.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index 5ea73c246..e800f9260 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -26,6 +26,7 @@ jobs:
           - kernel-gpu-snp
           - kernel-gpu-tdx-experimental
           - nydus
+          - ovmf-sev
           - qemu
           - qemu-tdx-experimental
           - rootfs-image

From 772d4db2629e1e100b040a5c720c5f92b13a28d7 Mon Sep 17 00:00:00 2001
From: Unmesh Deodhar <udeodhar@amd.com>
Date: Thu, 4 May 2023 17:34:13 +0000
Subject: [PATCH 057/150] gha: Build and ship SEV initrd

We have code that builds initrd for SEV.
thus, adding that to the test and release process.

Fixes: #6572

Signed-off-by: Unmesh Deodhar <udeodhar@amd.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index e800f9260..b49f4ca5c 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -31,6 +31,7 @@ jobs:
           - qemu-tdx-experimental
           - rootfs-image
           - rootfs-initrd
+          - rootfs-initrd-sev
           - shim-v2
           - tdvf
           - virtiofsd

From a085a6d7b480ed479c4295b70b85acc185244998 Mon Sep 17 00:00:00 2001
From: Chelsea Mafrica <chelsea.e.mafrica@intel.com>
Date: Tue, 21 Feb 2023 08:35:45 -0800
Subject: [PATCH 058/150] kata-ctl: add generic kvm check

Add kvm check using ioctl macro to create a syscall that checks the kvm
api version and if creation of a vm is successful.

Fixes #5338

Signed-off-by: Chelsea Mafrica <chelsea.e.mafrica@intel.com>
---
 src/tools/kata-ctl/src/arch/x86_64/mod.rs | 17 +++++-
 src/tools/kata-ctl/src/check.rs           | 63 +++++++++++++++++++++++
 src/tools/kata-ctl/src/ops/check_ops.rs   |  3 ++
 src/tools/kata-ctl/src/types.rs           |  1 +
 4 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/src/tools/kata-ctl/src/arch/x86_64/mod.rs b/src/tools/kata-ctl/src/arch/x86_64/mod.rs
index b1ee9f136..206c447a9 100644
--- a/src/tools/kata-ctl/src/arch/x86_64/mod.rs
+++ b/src/tools/kata-ctl/src/arch/x86_64/mod.rs
@@ -13,7 +13,7 @@ mod arch_specific {
     use crate::check::{GuestProtection, ProtectionError};
     use crate::types::*;
     use crate::utils;
-    use anyhow::{anyhow, Result};
+    use anyhow::{anyhow, Context, Result};
     use nix::unistd::Uid;
     use std::fs;
     use std::path::Path;
@@ -41,6 +41,12 @@ mod arch_specific {
             fp: check_kernel_modules,
             perm: PermissionType::NonPrivileged,
         },
+        CheckItem {
+            name: CheckType::KvmIsUsable,
+            descr: "This parameter performs check to see if KVM is usable",
+            fp: check_kvm_is_usable,
+            perm: PermissionType::Privileged,
+        },
     ];
 
     static MODULE_LIST: &[KernelModule] = &[
@@ -114,6 +120,15 @@ mod arch_specific {
         utils::get_generic_cpu_details(check::PROC_CPUINFO)
     }
 
+    // check if kvm is usable
+    fn check_kvm_is_usable(_args: &str) -> Result<()> {
+        println!("INFO: check if kvm is usable: x86_64");
+
+        let result = check::check_kvm_is_usable_generic();
+
+        result.context("KVM check failed")
+    }
+
     pub const TDX_SYS_FIRMWARE_DIR: &str = "/sys/firmware/tdx_seam/";
     pub const TDX_CPU_FLAG: &str = "tdx";
     pub const SEV_KVM_PARAMETER_PATH: &str = "/sys/module/kvm_amd/parameters/sev";
diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs
index 8f21f6fab..7a15aa72a 100644
--- a/src/tools/kata-ctl/src/check.rs
+++ b/src/tools/kata-ctl/src/check.rs
@@ -6,6 +6,10 @@
 // Contains checks that are not architecture-specific
 
 use anyhow::{anyhow, Result};
+use nix::fcntl::{open, OFlag};
+use nix::sys::stat::Mode;
+use nix::unistd::close;
+use nix::{ioctl_write_int_bad, request_code_none};
 use reqwest::header::{CONTENT_TYPE, USER_AGENT};
 use serde::{Deserialize, Serialize};
 use std::fmt;
@@ -176,6 +180,65 @@ pub fn run_network_checks() -> Result<()> {
     Ok(())
 }
 
+// Set of basic checks for kvm. Architectures should implement more specific checks if needed
+#[allow(dead_code)]
+pub fn check_kvm_is_usable_generic() -> Result<()> {
+    // check for root user
+    if !nix::unistd::Uid::effective().is_root() {
+        return Err(anyhow!("Will not perform kvm checks as non root user"));
+    }
+
+    // we do not want to create syscalls to any device besides /dev/kvm
+    const KVM_DEVICE: &str = "/dev/kvm";
+
+    // constants specific to kvm ioctls found in kvm.h
+    const KVM_IOCTL_ID: u8 = 0xAE;
+    const KVM_CREATE_VM: u8 = 0x01;
+    const KVM_GET_API_VERSION: u8 = 0x00;
+    // per kvm api documentation, this number should always be 12
+    // https://www.kernel.org/doc/html/latest/virt/kvm/api.html#kvm-get-api-version
+    const API_VERSION: i32 = 12;
+
+    // open kvm device
+    // since file is not being created, mode argument is not relevant
+    let mode = Mode::empty();
+    let flags = OFlag::O_RDWR | OFlag::O_CLOEXEC;
+    let fd = open(KVM_DEVICE, flags, mode)?;
+
+    // check kvm api version
+    ioctl_write_int_bad!(
+        kvm_api_version,
+        request_code_none!(KVM_IOCTL_ID, KVM_GET_API_VERSION)
+    );
+    // 0 is not used but required to produce output
+    let v = unsafe { kvm_api_version(fd, 0)? };
+    if v != API_VERSION {
+        return Err(anyhow!("KVM API version is not correct"));
+    }
+
+    // check if you can create vm
+    ioctl_write_int_bad!(
+        kvm_create_vm,
+        request_code_none!(KVM_IOCTL_ID, KVM_CREATE_VM)
+    );
+    // 0 is default machine type
+    let vmfd = unsafe { kvm_create_vm(fd, 0) };
+    let _vmfd = match vmfd {
+        Ok(vm) => vm,
+        Err(ref error) if error.to_string() == "EBUSY: Device or resource busy" => {
+            return Err(anyhow!(
+                "Another hypervisor is running. KVM_CREATE_VM error: {:?}",
+                error
+            ))
+        }
+        Err(error) => return Err(anyhow!("Other KVM_CREATE_VM error: {:?}", error)),
+    };
+
+    let _ = close(fd);
+
+    Ok(())
+}
+
 fn get_kata_all_releases_by_url(url: &str) -> std::result::Result<Vec<Release>, reqwest::Error> {
     let releases: Vec<Release> = reqwest::blocking::Client::new()
         .get(url)
diff --git a/src/tools/kata-ctl/src/ops/check_ops.rs b/src/tools/kata-ctl/src/ops/check_ops.rs
index 298e214c5..ed418169d 100644
--- a/src/tools/kata-ctl/src/ops/check_ops.rs
+++ b/src/tools/kata-ctl/src/ops/check_ops.rs
@@ -80,6 +80,9 @@ pub fn handle_check(checkcmd: CheckArgument) -> Result<()> {
 
             // run kernel module checks
             handle_builtin_check(CheckType::KernelModules, "")?;
+
+            // run kvm checks
+            handle_builtin_check(CheckType::KvmIsUsable, "")?;
         }
 
         CheckSubCommand::NoNetworkChecks => {
diff --git a/src/tools/kata-ctl/src/types.rs b/src/tools/kata-ctl/src/types.rs
index 26f5954d8..fcafeb435 100644
--- a/src/tools/kata-ctl/src/types.rs
+++ b/src/tools/kata-ctl/src/types.rs
@@ -15,6 +15,7 @@ pub enum CheckType {
     Cpu,
     Network,
     KernelModules,
+    KvmIsUsable,
 }
 
 // PermissionType is used to show whether a check needs to run with elevated (super-user)

From 5f8008b69c05b0a3aa2885171a50989db34e9535 Mon Sep 17 00:00:00 2001
From: Chelsea Mafrica <chelsea.e.mafrica@intel.com>
Date: Mon, 27 Feb 2023 12:55:17 -0800
Subject: [PATCH 059/150] kata-ctl: add unit test for kvm check

Check that kvm test fails when run as non-root and when device specified
is not /dev/kvm.

Fixes #5338

Signed-off-by: Chelsea Mafrica <chelsea.e.mafrica@intel.com>
---
 src/tools/kata-ctl/src/check.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs
index 7a15aa72a..ef1007f86 100644
--- a/src/tools/kata-ctl/src/check.rs
+++ b/src/tools/kata-ctl/src/check.rs
@@ -395,6 +395,7 @@ mod tests {
     use std::fs;
     use std::io::Write;
     use tempfile::tempdir;
+    use test_utils::skip_if_root;
 
     #[test]
     fn test_get_single_cpu_info() {
@@ -522,6 +523,16 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_check_kvm_is_usable_generic() {
+        skip_if_root!();
+        #[allow(dead_code)]
+        let result = check_kvm_is_usable_generic();
+        assert!(
+            result.err().unwrap().to_string() == "Will not perform kvm checks as non root user"
+        );
+    }
+
     #[test]
     fn test_get_kata_all_releases_by_url() {
         #[derive(Debug)]

From 38ce4a32af80a493b1570cbe7f4341ce80bf0401 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 15:41:39 +0000
Subject: [PATCH 060/150] local-build: add support to build QEMU for SEV-SNP

Add Make targets and helper functions to build the QEMU
needed for SEV-SNP.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
Signed-off-by: Alex Carter <Alex.Carter@ibm.com>
---
 tools/packaging/kata-deploy/local-build/Makefile  |  4 ++++
 .../local-build/kata-deploy-binaries.sh           | 15 +++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index 91d76dab0..a7c323444 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -31,6 +31,7 @@ all: serial-targets \
 	kernel-gpu-tdx-experimental-tarball \
 	nydus-tarball \
 	ovmf-sev-tarball \
+	qemu-snp-experimental-tarball \
 	qemu-tarball \
 	qemu-tdx-experimental-tarball \
 	shim-v2-tarball \
@@ -83,6 +84,9 @@ nydus-tarball:
 ovmf-sev-tarball:
 	${MAKE} $@-build
 
+qemu-snp-experimental-tarball:
+	${MAKE} $@-build
+
 qemu-tarball:
 	${MAKE} $@-build
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 81784379a..cf2953c7f 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -89,6 +89,7 @@ options:
 	nydus
 	ovmf-sev
 	qemu
+	qemu-snp-experimental
 	qemu-tdx-experimental
 	rootfs-image
 	rootfs-initrd
@@ -362,6 +363,17 @@ install_qemu_tdx_experimental() {
 		"${qemu_experimental_builder}"
 }
 
+install_qemu_snp_experimental() {
+	export qemu_suffix="snp-experimental"
+	export qemu_tarball_name="kata-static-qemu-${qemu_suffix}.tar.gz"
+
+	install_qemu_helper \
+		"assets.hypervisor.qemu-${qemu_suffix}.url" \
+		"assets.hypervisor.qemu-${qemu_suffix}.tag" \
+		"qemu-${qemu_suffix}" \
+		"${qemu_experimental_builder}"
+}
+
 # Install static firecracker asset
 install_firecracker() {
 	install_cached_tarball_component \
@@ -522,6 +534,7 @@ handle_build() {
 		install_nydus
 		install_ovmf_sev
 		install_qemu
+		install_qemu_snp_experimental
 		install_qemu_tdx_experimental
 		install_shimv2
 		install_tdvf
@@ -554,6 +567,8 @@ handle_build() {
 
 	qemu) install_qemu ;;
 
+	qemu-snp-experimental) install_qemu_snp_experimental ;;
+
 	qemu-tdx-experimental) install_qemu_tdx_experimental ;;
 
 	rootfs-image) install_image ;;

From 879333bfc78a5231224a17a940accc70f0259c0e Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 15:50:59 +0000
Subject: [PATCH 061/150] versions: update SNP QEMU version

Refactor SNP QEMU entry in versions.yaml to match
qemu-experimental and qemu-tdx-experimental.

Also, update the version of QEMU to what we are using
in CCv0. This is the non-UPM QEMU and it does not
have kernel hashes support.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
Signed-off-by: Alex Carter <Alex.Carter@ibm.com>
---
 .../no_patches.txt                                     |  0
 versions.yaml                                          | 10 +++++-----
 2 files changed, 5 insertions(+), 5 deletions(-)
 create mode 100644 tools/packaging/qemu/patches/tag_patches/3b6a2b6b7466f6dea53243900b7516c3f29027b7/no_patches.txt

diff --git a/tools/packaging/qemu/patches/tag_patches/3b6a2b6b7466f6dea53243900b7516c3f29027b7/no_patches.txt b/tools/packaging/qemu/patches/tag_patches/3b6a2b6b7466f6dea53243900b7516c3f29027b7/no_patches.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/versions.yaml b/versions.yaml
index 4f6e81e4b..a61d89351 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -98,11 +98,6 @@ assets:
       uscan-url: >-
         https://github.com/qemu/qemu/tags
         .*/v?(\d\S+)\.tar\.gz
-      snp:
-        description: "VMM that uses KVM and supports AMD SEV-SNP"
-        url: "https://github.com/AMDESE/qemu"
-        branch: "snp-v3"
-        commit: "ffa95097ee"
 
     qemu-experimental:
       description: "QEMU with virtiofs support"
@@ -115,6 +110,11 @@ assets:
       url: "https://github.com/kata-containers/qemu"
       tag: "ad4c7f529a279685da84297773b4ec8080153c2d-plus-TDX-v1.3"
 
+    qemu-snp-experimental:
+      description: "QEMU with experimental SNP support (no UPM)"
+      url: "https://github.com/AMDESE/qemu"
+      tag: "3b6a2b6b7466f6dea53243900b7516c3f29027b7"
+
   image:
     description: |
       Root filesystem disk image used to boot the guest virtual

From 7a58a91fa6d64b8a4324061f56d7733dd6c7e242 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 15:58:12 +0000
Subject: [PATCH 062/150] docs: update SNP guide

Since we reshuffled versions.yaml, update the guide so that
we can find the SNP QEMU info.

Once runtime support is merged we should overhaul or remove
this guide, but let's keep it for now.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md
index a87c9be62..3a56cbba2 100644
--- a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md
+++ b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md
@@ -44,12 +44,11 @@ $ popd
 - Build a custom QEMU
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu.snp.url")"
-$ qemu_branch="$(get_from_kata_deps "assets.hypervisor.qemu.snp.branch")"
-$ qemu_commit="$(get_from_kata_deps "assets.hypervisor.qemu.snp.commit")"
-$ git clone -b "${qemu_branch}" "${qemu_url}"
+$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ git clone "${qemu_url}"
 $ pushd qemu
-$ git checkout "${qemu_commit}"
+$ git checkout "${qemu_tag}"
 $ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
 $ make -j "$(nproc)"
 $ popd

From f4ee00576a8647dbad18174daad55070338a7afb Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 16:32:21 +0000
Subject: [PATCH 063/150] gha: Build and ship QEMU for SNP

Now that we can build SNP QEMU, let's do that for tests and release.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index b49f4ca5c..c2b961730 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -28,6 +28,7 @@ jobs:
           - nydus
           - ovmf-sev
           - qemu
+          - qemu-snp-experimental
           - qemu-tdx-experimental
           - rootfs-image
           - rootfs-initrd

From db095ddeb4b7a363cc18de9f1faf339d2db75341 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 16:37:05 +0000
Subject: [PATCH 064/150] cache: add SNP flavor to comments

Update comments to include new SNP QEMU option

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tools/packaging/static-build/cache_components_main.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index 1c46c1e01..28e6bfacf 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -14,7 +14,7 @@ source "${script_dir}/../scripts/lib.sh"
 
 KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental
 OVMF_FLAVOUR="${OVMF_FLAVOUR:-x86_64}" # x86_64 | tdx
-QEMU_FLAVOUR="${QEMU_FLAVOUR:-qemu}" # qemu | qemu-tdx-experimental
+QEMU_FLAVOUR="${QEMU_FLAVOUR:-qemu}" # qemu | qemu-tdx-experimental | qemu-snp-experimental
 ROOTFS_IMAGE_TYPE="${ROOTFS_IMAGE_TYPE:-image}" # image | initrd
 
 cache_clh_artifacts() {
@@ -134,7 +134,7 @@ Usage: $0 "[options]"
 			  The default KERNEL_FLAVOUR value is "kernel"
 		-n	Nydus cache
 		-q 	QEMU cache
-			* Export QEMU_FLAVOUR="qemu | qemu-tdx-experimental" for a specific build
+			* Export QEMU_FLAVOUR="qemu | qemu-tdx-experimental | qemu-snp-experimental" for a specific build
 			  The default QEMU_FLAVOUR value is "qemu"
 		-r 	RootFS cache
 			* Export ROOTFS_IMAGE_TYPE="image|initrd" for one of those two types

From cf0ca265f993537d485cb6e381cf5b12b87309dd Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 18:36:09 +0000
Subject: [PATCH 065/150] local-build: Add x86_64 OVMF target

Add targets to build the "plain" x86_64 OVMF.

This will be used by anyone who is using SEV or SNP
without kernel hashes. The SNP QEMU does not yet
support kernel hashes so the OvmfPkg will be used
by default.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
Signed-off-by: Alex Carter <Alex.Carter@ibm.com>
---
 tools/packaging/kata-deploy/local-build/Makefile     |  4 ++++
 .../kata-deploy/local-build/kata-deploy-binaries.sh  | 12 ++++++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index a7c323444..fc3bbcb6e 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -31,6 +31,7 @@ all: serial-targets \
 	kernel-gpu-tdx-experimental-tarball \
 	nydus-tarball \
 	ovmf-sev-tarball \
+	ovmf-tarball \
 	qemu-snp-experimental-tarball \
 	qemu-tarball \
 	qemu-tdx-experimental-tarball \
@@ -84,6 +85,9 @@ nydus-tarball:
 ovmf-sev-tarball:
 	${MAKE} $@-build
 
+ovmf-tarball:
+	${MAKE} $@-build
+
 qemu-snp-experimental-tarball:
 	${MAKE} $@-build
 
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index cf2953c7f..7b80d000e 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -87,6 +87,7 @@ options:
 	kernel-gpu-snp
 	kernel-gpu-tdx-experimental
 	nydus
+	ovmf
 	ovmf-sev
 	qemu
 	qemu-snp-experimental
@@ -532,6 +533,7 @@ handle_build() {
 		install_kernel_dragonball_experimental
 		install_kernel_tdx_experimental
 		install_nydus
+		install_ovmf
 		install_ovmf_sev
 		install_qemu
 		install_qemu_snp_experimental
@@ -547,8 +549,6 @@ handle_build() {
 
 	kernel) install_kernel ;;
 
-	nydus) install_nydus ;;
-
 	kernel-dragonball-experimental) install_kernel_dragonball_experimental ;;
 
 	kernel-experimental) install_kernel_experimental ;;
@@ -557,14 +557,18 @@ handle_build() {
 
 	kernel-sev) install_kernel_sev ;;
 
-	ovmf-sev) install_ovmf_sev ;;
-
 	kernel-gpu) install_kernel_gpu ;;
 
 	kernel-gpu-snp) install_kernel_gpu_snp;;
 
 	kernel-gpu-tdx-experimental) install_kernel_gpu_tdx_experimental;;
 
+	nydus) install_nydus ;;
+
+	ovmf) install_ovmf ;;
+
+	ovmf-sev) install_ovmf_sev ;;
+
 	qemu) install_qemu ;;
 
 	qemu-snp-experimental) install_qemu_snp_experimental ;;

From b8bbe6325fca03101049cb8ccc31a7580c7a13ac Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 19:05:18 +0000
Subject: [PATCH 066/150] gha: build OVMF for tests and release

The x86_64 package of OVMF is required for deployments
that don't use kernel hashes, which includes SEV-SNP
in the short term. We should keep this in the bundle
in the long term in case someone wants to disable
kernel hashes.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index c2b961730..d582f340b 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -26,6 +26,7 @@ jobs:
           - kernel-gpu-snp
           - kernel-gpu-tdx-experimental
           - nydus
+          - ovmf
           - ovmf-sev
           - qemu
           - qemu-snp-experimental

From dbcc3b5cc8770db187a3824dbc0ff576171fc5b9 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 19:38:44 +0000
Subject: [PATCH 067/150] local-build: fix default values for OVMF build

Existing value has wrong name and compression type
leading to installation failure.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 7b80d000e..101f32cd0 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -479,7 +479,7 @@ install_shimv2() {
 
 install_ovmf() {
 	ovmf_type="${1:-x86_64}"
-	tarball_name="${2:-edk2.tar.xz}"
+	tarball_name="${2:-edk2-x86_64.tar.gz}"
 
 	local component_name="ovmf"
 	local component_version="$(get_from_kata_deps "externals.ovmf.${ovmf_type}.version")"

From af7f2519bf5cf9b8dadce8a0e30863220ebac84f Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 21 Apr 2023 20:28:40 +0000
Subject: [PATCH 068/150] versions: update SEV kernel description

SNP and SEV will share a (guest) kernel. Update the description
in versions.yaml to mention this.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 versions.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.yaml b/versions.yaml
index a61d89351..e2a32b7f4 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -165,7 +165,7 @@ assets:
     url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/"
     version: "v5.19.2"
     sev:
-      description: "Linux kernel that supports SEV"
+      description: "Linux kernel that supports SEV and SNP"
       url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/"
       version: "v5.19.2"
     snp:

From 0bb37bff78a4707f3c84815423be315ca3226794 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Mon, 24 Apr 2023 21:38:27 +0000
Subject: [PATCH 069/150] config: Add SNP configuration

SNP requires many specific configurations, so let's make
a new SNP configuration file that we can use with the
kata-qemu-snp runtime class.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
Signed-off-by: Alex Carter <Alex.Carter@ibm.com>
---
 src/runtime/Makefile                          |  34 +
 src/runtime/arch/amd64-options.mk             |   1 +
 .../config/configuration-qemu-snp.toml.in     | 690 ++++++++++++++++++
 3 files changed, 725 insertions(+)
 create mode 100644 src/runtime/config/configuration-qemu-snp.toml.in

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index ee3896c77..776da3ef4 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -98,6 +98,7 @@ GENERATED_VARS = \
 		CONFIG_QEMU_IN \
 		CONFIG_QEMU_SEV_IN \
 		CONFIG_QEMU_TDX_IN \
+		CONFIG_QEMU_SNP_IN \
 		CONFIG_QEMU_GPU_IN \
 		CONFIG_CLH_IN \
 		CONFIG_FC_IN \
@@ -130,6 +131,7 @@ FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd
 FIRMWARETDVFVOLUMEPATH :=
 
 FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
+FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
 
 # Name of default configuration file the runtime will use.
 CONFIG_FILE = configuration.toml
@@ -151,6 +153,9 @@ QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
 QEMUTDXPATH := $(QEMUBINDIR)/$(QEMUTDXCMD)
 QEMUTDXVALIDHYPERVISORPATHS := [\"$(QEMUTDXPATH)\"]
 
+QEMUSNPPATH := $(QEMUBINDIR)/$(QEMUSNPCMD)
+QEMUSNPVALIDHYPERVISORPATHS := [\"$(QEMUSNPPATH)\"]
+
 QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD)
 
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
@@ -210,6 +215,7 @@ DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p
 DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p
+DEFSHAREDFS_QEMU_SNP_VIRTIOFS := virtio-9p
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
 ifeq ($(ARCH),ppc64le)
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/qemu/virtiofsd
@@ -307,6 +313,18 @@ ifneq (,$(QEMUCMD))
 
     CONFIGS += $(CONFIG_QEMU_SEV)
 
+    CONFIG_FILE_QEMU_SNP = configuration-qemu-snp.toml
+    CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP)
+    CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in
+
+    CONFIG_PATH_QEMU_SNP = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SNP))
+    CONFIG_PATHS += $(CONFIG_PATH_QEMU_SNP)
+
+    SYSCONFIG_QEMU_SNP = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SNP))
+    SYSCONFIG_PATHS_SNP += $(SYSCONFIG_QEMU_SNP)
+
+    CONFIGS += $(CONFIG_QEMU_SNP)
+
     CONFIG_FILE_QEMU_GPU = configuration-qemu-gpu.toml
     CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU)
     CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in
@@ -329,6 +347,11 @@ ifneq (,$(QEMUCMD))
     KERNELTDXTYPE = compressed
     KERNELTDXNAME = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE))
     KERNELTDXPATH = $(KERNELDIR)/$(KERNELTDXNAME)
+
+    KERNELSNPTYPE = compressed
+    KERNELSNPNAME = $(call MAKE_KERNEL_SNP_NAME,$(KERNELSNPTYPE))
+    KERNELSNPPATH = $(KERNELDIR)/$(KERNELSNPNAME)
+
 endif
 
 ifneq (,$(CLHCMD))
@@ -488,6 +511,7 @@ USER_VARS += KERNELPATH_ACRN
 USER_VARS += KERNELPATH
 USER_VARS += KERNELSEVPATH
 USER_VARS += KERNELTDXPATH
+USER_VARS += KERNELSNPPATH
 USER_VARS += KERNELPATH_CLH
 USER_VARS += KERNELPATH_FC
 USER_VARS += KERNELVIRTIOFSPATH
@@ -496,6 +520,7 @@ USER_VARS += FIRMWARESEVPATH
 USER_VARS += FIRMWARETDVFPATH
 USER_VARS += FIRMWAREVOLUMEPATH
 USER_VARS += FIRMWARETDVFVOLUMEPATH
+USER_VARS += FIRMWARESNPPATH
 USER_VARS += MACHINEACCELERATORS
 USER_VARS += CPUFEATURES
 USER_VARS += TDXCPUFEATURES
@@ -518,10 +543,13 @@ USER_VARS += PROJECT_URL
 USER_VARS += QEMUBINDIR
 USER_VARS += QEMUCMD
 USER_VARS += QEMUTDXCMD
+USER_VARS += QEMUSNPCMD
 USER_VARS += QEMUPATH
 USER_VARS += QEMUTDXPATH
+USER_VARS += QEMUSNPPATH
 USER_VARS += QEMUVALIDHYPERVISORPATHS
 USER_VARS += QEMUTDXVALIDHYPERVISORPATHS
+USER_VARS += QEMUSNPVALIDHYPERVISORPATHS
 USER_VARS += QEMUVIRTIOFSCMD
 USER_VARS += QEMUVIRTIOFSPATH
 USER_VARS += RUNTIME_NAME
@@ -553,6 +581,7 @@ USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_TDX_VIRTIOFS
 USER_VARS += DEFSHAREDFS_QEMU_SEV_VIRTIOFS
+USER_VARS += DEFSHAREDFS_QEMU_SNP_VIRTIOFS
 USER_VARS += DEFVIRTIOFSDAEMON
 USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
 USER_VARS += DEFVIRTIOFSCACHESIZE
@@ -667,6 +696,11 @@ define MAKE_KERNEL_TDX_NAME
 $(if $(findstring uncompressed,$1),vmlinux-tdx.container,vmlinuz-tdx.container)
 endef
 
+# SNP configuration uses the SEV kernel
+define MAKE_KERNEL_SNP_NAME
+$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container)
+endef
+
 GENERATED_FILES += pkg/katautils/config-settings.go
 
 $(RUNTIME_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) | show-summary
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
index ab82539ba..ab2b1d2d2 100644
--- a/src/runtime/arch/amd64-options.mk
+++ b/src/runtime/arch/amd64-options.mk
@@ -12,6 +12,7 @@ CPUFEATURES := pmu=off
 
 QEMUCMD := qemu-system-x86_64
 QEMUTDXCMD := qemu-system-x86_64-tdx-experimental
+QEMUSNPCMD := qemu-system-x86_64-snp-experimental
 TDXCPUFEATURES := -vmx-rdseed-exit,pmu=off
 
 # Firecracker binary name
diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in
new file mode 100644
index 000000000..e09f14991
--- /dev/null
+++ b/src/runtime/config/configuration-qemu-snp.toml.in
@@ -0,0 +1,690 @@
+# Copyright (c) 2017-2019 Intel Corporation
+# Copyright (c) 2021 Adobe Inc.
+# Copyright (c) 2023 IBM Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_SNP_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUSNPPATH@"
+kernel = "@KERNELSNPPATH@"
+#image = "@IMAGEPATH@"
+initrd = "@INITRDPATH@"
+machine_type = "@MACHINETYPE@"
+
+# rootfs filesystem type:
+#   - ext4 (default)
+#   - xfs
+#   - erofs
+rootfs_type=@DEFROOTFSTYPE@
+
+# Enable confidential guest support.
+# Toggling that setting may trigger different hardware features, ranging
+# from memory encryption to both memory and CPU-state encryption and integrity.
+# The Kata Containers runtime dynamically detects the available feature set and
+# aims at enabling the largest possible one, returning an error if none is
+# available, or none is supported by the hypervisor.
+#
+# Known limitations:
+# * Does not work by design:
+#   - CPU Hotplug 
+#   - Memory Hotplug
+#   - NVDIMM devices
+#
+# Default false
+confidential_guest = true
+
+# enable SEV SNP VMs
+sev_snp_guest = true
+
+# Enable running QEMU VMM as a non-root user.
+# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+# rootless = true
+
+# List of valid annotation names for the hypervisor
+# Each member of the list is a regular expression, which is the base name
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
+enable_annotations = @DEFENABLEANNOTATIONS@
+
+# List of valid annotations values for the hypervisor
+# Each member of the list is a path pattern as described by glob(3).
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @QEMUSNPVALIDHYPERVISORPATHS@
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWARESNPPATH@"
+
+# Path to the firmware volume.
+# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
+# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
+# can be customized per each user while UEFI code is kept same.
+firmware_volume = "@FIRMWARETDVFVOLUMEPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators="@MACHINEACCELERATORS@"
+
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@"
+
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Default maximum memory in MiB per SB / VM
+# unspecified or == 0           --> will be set to the actual amount of physical RAM
+# > 0 <= amount of physical RAM --> will be set to the specified number
+# > amount of physical RAM      --> will be set to the actual amount of physical RAM
+default_maxmemory = @DEFMAXMEMSZ@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+#enable_virtio_mem = true
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# virtio-fs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-9p
+#   - virtio-fs-nydus
+shared_fs = "@DEFSHAREDFS_QEMU_SNP_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# List of valid annotations values for the virtiofs daemon
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@
+valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - never
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# aio is the I/O mechanism used by qemu
+# Options:
+#
+#   - threads
+#     Pthread based disk I/O.
+#
+#   - native
+#     Native Linux I/O.
+#
+#   - io_uring
+#     Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and
+#     qemu >=5.0.
+block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+#enable_iommu_platform = true
+
+# List of valid annotations values for the vhost user store path
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@
+valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@
+
+# The timeout for reconnecting on non-server spdk sockets when the remote end goes away.
+# qemu will delay this many seconds and then attempt to reconnect.
+# Zero disables reconnecting, and the default is zero.
+vhost_user_reconnect_timeout_sec = 0
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+file_mem_backend = ""
+
+# List of valid annotations values for the file_mem_backend annotation
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@
+valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = []
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. And Debug also enable the hmp socket.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+#
+# nvdimm is not supported when `confidential_guest = true`.
+#
+# Default is false
+disable_image_nvdimm = true
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+#pcie_root_port = 2
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance.
+#disable_vhost_net = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# List of valid annotations values for entropy_source
+# The default if not set is empty (all annotations rejected.)
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered while scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
+
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest’s memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+#guest_memory_dump_path="/var/crash/kata"
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+#guest_memory_dump_paging=false
+
+# Enable swap in the guest. Default false.
+# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device
+# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness")
+# is bigger than 0.
+# The size of the swap device should be
+# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes.
+# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.
+# If swap_in_bytes and memory_limit_in_bytes is not set, the size should
+# be default_memory.
+#enable_guest_swap = true
+
+# use legacy serial for guest console if available and implemented for architecture. Default false
+#use_legacy_serial = true
+
+# disable applying SELinux on the VMM process (default false)
+disable_selinux=@DEFDISABLESELINUX@
+
+# disable applying SELinux on the container process
+# If set to false, the type `container_t` is applied to the container process by default.
+# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built
+# with `SELINUX=yes`.
+# (default: true)
+disable_guest_selinux=@DEFDISABLEGUESTSELINUX@
+
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+#template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the agent will generate OpenTelemetry trace spans.
+#
+# Notes:
+#
+# - If the runtime also has tracing enabled, the agent spans will be
+#   associated with the appropriate runtime parent span.
+# - If enabled, the runtime will wait for the container to shutdown,
+#   increasing the container shutdown time slightly.
+#
+# (default: disabled)
+#enable_tracing = true
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+# Enable debug console.
+
+# If enabled, user can connect guest OS running inside hypervisor
+# through "kata-runtime exec <sandbox-id>" command
+
+#debug_console_enabled = true
+
+# Agent connection dialing timeout value in seconds
+# (default: 90)
+dial_timeout = 90
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# vCPUs pinning settings
+# if enabled, each vCPU thread will be scheduled to a fixed CPU
+# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet)
+# enable_vcpus_pinning = false
+
+# Apply a custom SELinux security policy to the container process inside the VM.
+# This is used when you want to apply a type other than the default `container_t`,
+# so general users should not uncomment and apply it.
+# (format: "user:role:type")
+# Note: You cannot specify MCS policy with the label because the sensitivity levels and
+# categories are determined automatically by high-level container runtimes such as containerd.
+#guest_selinux_label="@DEFGUESTSELINUXLABEL@"
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# Set the full url to the Jaeger HTTP Thrift collector.
+# The default if not set will be "http://localhost:14268/api/traces"
+#jaeger_endpoint = ""
+
+# Sets the username to be used if basic auth is required for Jaeger.
+#jaeger_user = ""
+
+# Sets the password to be used if basic auth is required for Jaeger.
+#jaeger_password = ""
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In
+# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful
+# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.
+# Compatibility for determining appropriate sandbox (VM) size:
+# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
+#   does not yet support sandbox sizing annotations.
+# - When running single containers using a tool like ctr, container sizing information will be available.
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
+# VFIO Mode
+# Determines how VFIO devices should be be presented to the container.
+# Options:
+#
+#  - vfio
+#    Matches behaviour of OCI runtimes (e.g. runc) as much as
+#    possible.  VFIO devices will appear in the container as VFIO
+#    character devices under /dev/vfio.  The exact names may differ
+#    from the host (they need to match the VM's IOMMU group numbers
+#    rather than the host's)
+#
+#  - guest-kernel
+#    This is a Kata-specific behaviour that's useful in certain cases.
+#    The VFIO device is managed by whatever driver in the VM kernel
+#    claims it.  This means it will appear as one or more device nodes
+#    or network interfaces depending on the nature of the device.
+#    Using this mode requires specially built workloads that know how
+#    to locate the relevant device interfaces within the VM.
+#
+vfio_mode="@DEFVFIOMODE@"
+
+# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
+# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
+disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@
+
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+# (default: false)
+# enable_pprof = true
+
+# WARNING: All the options in the following section have not been implemented yet.
+# This section was added as a placeholder. DO NOT USE IT!
+[image]
+# Container image service.
+#
+# Offload the CRI image management service to the Kata agent.
+# (default: false)
+#service_offload = true
+
+# Container image decryption keys provisioning.
+# Applies only if service_offload is true.
+# Keys can be provisioned locally (e.g. through a special command or
+# a local file) or remotely (usually after the guest is remotely attested).
+# The provision setting is a complete URL that lets the Kata agent decide
+# which method to use in order to fetch the keys.
+#
+# Keys can be stored in a local file, in a measured and attested initrd:
+#provision=data:///local/key/file
+#
+# Keys could be fetched through a special command or binary from the
+# initrd (guest) image, e.g. a firmware call:
+#provision=file:///path/to/bin/fetcher/in/guest
+#
+# Keys can be remotely provisioned. The Kata agent fetches them from e.g.
+# a HTTPS URL:
+#provision=https://my-key-broker.foo/tenant/<tenant-id>

From 14dd0537582c32f3f3e20850b1b74a13f5cb6ac3 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Fri, 28 Apr 2023 19:29:36 +0000
Subject: [PATCH 070/150] kata-deploy: add kata-qemu-snp runtimeclass

Since SEV-SNP has limited hotplug support, increase
the pod overhead to account for fixed resource usage.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 .../runtimeclasses/kata-runtimeClasses.yaml         | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
index 00197ed3b..4e8da4fea 100644
--- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
@@ -40,6 +40,19 @@ scheduling:
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-snp
+handler: kata-qemu-snp
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
 metadata:
     name: kata-clh
 handler: kata-clh

From 4da6eb588d148b1bf1479e3b86e9f1a08c58afc6 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Wed, 3 May 2023 20:14:50 +0000
Subject: [PATCH 071/150] kata-deploy: Add qemu-snp shim

Now that we have the SNP components in place, make sure that
kata-deploy knows about the qemu-snp shim so that it will be
added to containerd config.

Fixes: #6575

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tools/packaging/kata-deploy/scripts/kata-deploy.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 5ae15802a..d6be5019f 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -18,6 +18,7 @@ shims=(
 	"qemu"
 	"qemu-tdx"
 	"qemu-gpu"
+	"qemu-snp"
 	"clh"
 	"dragonball"
 )

From 2c90cac75169d1835892d77c8c79705d27f6dc7f Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Wed, 10 May 2023 21:23:38 +0000
Subject: [PATCH 072/150] local-build: fixup alphabetization

A few pieces of the local-build tooling are supposed to be
alphabetized. Fixup a couple minor issues that have accumulated.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tools/packaging/kata-deploy/local-build/Makefile     | 10 +++++-----
 .../kata-deploy/local-build/kata-deploy-binaries.sh  | 12 ++++++------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index fc3bbcb6e..26454b107 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -23,12 +23,12 @@ all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh
 
 all: serial-targets \
 	firecracker-tarball \
-	kernel-tarball \
 	kernel-dragonball-experimental-tarball \
-	kernel-tdx-experimental-tarball \
 	kernel-gpu \
 	kernel-gpu-snp-tarball \
 	kernel-gpu-tdx-experimental-tarball \
+	kernel-tarball \
+	kernel-tdx-experimental-tarball \
 	nydus-tarball \
 	ovmf-sev-tarball \
 	ovmf-tarball \
@@ -55,10 +55,10 @@ cloud-hypervisor-tarball:
 firecracker-tarball:
 	${MAKE} $@-build
 
-kernel-tarball:
+kernel-dragonball-experimental-tarball:
 	${MAKE} $@-build
 
-kernel-dragonball-experimental-tarball:
+kernel-experimental-tarball:
 	${MAKE} $@-build
 
 kernel-gpu-tarball:
@@ -70,7 +70,7 @@ kernel-gpu-snp-tarball:
 kernel-gpu-tdx-experimental-tarball:
 	${MAKE} $@-build	
 
-kernel-experimental-tarball:
+kernel-tarball:
 	${MAKE} $@-build
 
 kernel-tdx-experimental-tarball:
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 101f32cd0..51e0e8da9 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -81,11 +81,11 @@ options:
 	kernel
 	kernel-dragonball-experimental
 	kernel-experimental
-	kernel-tdx-experimental
-	kernel-sev-tarball
 	kernel-gpu
 	kernel-gpu-snp
 	kernel-gpu-tdx-experimental
+	kernel-sev-tarball
+	kernel-tdx-experimental
 	nydus
 	ovmf
 	ovmf-sev
@@ -553,16 +553,16 @@ handle_build() {
 
 	kernel-experimental) install_kernel_experimental ;;
 
-	kernel-tdx-experimental) install_kernel_tdx_experimental ;;
-
-	kernel-sev) install_kernel_sev ;;
-
 	kernel-gpu) install_kernel_gpu ;;
 
 	kernel-gpu-snp) install_kernel_gpu_snp;;
 
 	kernel-gpu-tdx-experimental) install_kernel_gpu_tdx_experimental;;
 
+	kernel-tdx-experimental) install_kernel_tdx_experimental ;;
+
+	kernel-sev) install_kernel_sev ;;
+
 	nydus) install_nydus ;;
 
 	ovmf) install_ovmf ;;

From 3665b420455809e2181f1789c7ea6b766a4dcde5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 4 May 2023 10:35:58 +0200
Subject: [PATCH 073/150] gpu: Rename `gpu` targets to `nvidia-gpu`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This will make it easier for other GPU vendors to add the needed bits in
the future.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../build-kata-static-tarball-amd64.yaml      |  6 ++---
 .../kata-deploy/local-build/Makefile          | 12 +++++-----
 .../local-build/kata-deploy-binaries.sh       | 24 +++++++++----------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index d582f340b..e6f82b9de 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -22,9 +22,9 @@ jobs:
           - kernel-sev
           - kernel-dragonball-experimental
           - kernel-tdx-experimental
-          - kernel-gpu
-          - kernel-gpu-snp
-          - kernel-gpu-tdx-experimental
+          - kernel-nvidia-gpu
+          - kernel-nvidia-gpu-snp
+          - kernel-nvidia-gpu-tdx-experimental
           - nydus
           - ovmf
           - ovmf-sev
diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index 26454b107..59ce2d06b 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -24,9 +24,9 @@ all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh
 all: serial-targets \
 	firecracker-tarball \
 	kernel-dragonball-experimental-tarball \
-	kernel-gpu \
-	kernel-gpu-snp-tarball \
-	kernel-gpu-tdx-experimental-tarball \
+	kernel-nvidia-gpu \
+	kernel-nvidia-gpu-snp-tarball \
+	kernel-nvidia-gpu-tdx-experimental-tarball \
 	kernel-tarball \
 	kernel-tdx-experimental-tarball \
 	nydus-tarball \
@@ -61,13 +61,13 @@ kernel-dragonball-experimental-tarball:
 kernel-experimental-tarball:
 	${MAKE} $@-build
 
-kernel-gpu-tarball:
+kernel-nvidia-gpu-tarball:
 	${MAKE} $@-build
 
-kernel-gpu-snp-tarball:
+kernel-nvidia-gpu-snp-tarball:
 	${MAKE} $@-build
 
-kernel-gpu-tdx-experimental-tarball:
+kernel-nvidia-gpu-tdx-experimental-tarball:
 	${MAKE} $@-build	
 
 kernel-tarball:
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 51e0e8da9..673b0c291 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -81,9 +81,9 @@ options:
 	kernel
 	kernel-dragonball-experimental
 	kernel-experimental
-	kernel-gpu
-	kernel-gpu-snp
-	kernel-gpu-tdx-experimental
+	kernel-nvidia-gpu
+	kernel-nvidia-gpu-snp
+	kernel-nvidia-gpu-tdx-experimental
 	kernel-sev-tarball
 	kernel-tdx-experimental
 	nydus
@@ -262,32 +262,32 @@ install_kernel_dragonball_experimental() {
 }
 
 #Install GPU enabled kernel asset
-install_kernel_gpu() {
+install_kernel_nvidia_gpu() {
 	local kernel_url="$(get_from_kata_deps assets.kernel.url)"
 
 	install_kernel_helper \
 		"assets.kernel.version" \
-		"kernel-gpu" \
+		"kernel-nvidia-gpu" \
 		"-g nvidia -u ${kernel_url} -H deb"
 }
 
 #Install GPU and SNP enabled kernel asset
-install_kernel_gpu_snp() {
+install_kernel_nvidia_gpu_snp() {
 	local kernel_url="$(get_from_kata_deps assets.kernel.snp.url)"
 
 	install_kernel_helper \
 		"assets.kernel.snp.version" \
-		"kernel-gpu-snp" \
+		"kernel-nvidia-gpu-snp" \
 		"-x snp -g nvidia -u ${kernel_url} -H deb"
 }
 
 #Install GPU and TDX experimental enabled kernel asset
-install_kernel_gpu_tdx_experimental() {
+install_kernel_nvidia_gpu_tdx_experimental() {
 	local kernel_url="$(get_from_kata_deps assets.kernel-tdx-experimental.url)"
 
 	install_kernel_helper \
 		"assets.kernel-tdx-experimental.version" \
-		"kernel-gpu-tdx" \
+		"kernel-nvidia-gpu-tdx" \
 		"-x tdx -g nvidia -u ${kernel_url} -H deb"
 }
 
@@ -553,11 +553,11 @@ handle_build() {
 
 	kernel-experimental) install_kernel_experimental ;;
 
-	kernel-gpu) install_kernel_gpu ;;
+	kernel-nvidia-gpu) install_kernel_nvidia_gpu ;;
 
-	kernel-gpu-snp) install_kernel_gpu_snp;;
+	kernel-nvidia-gpu-snp) install_kernel_nvidia_gpu_snp;;
 
-	kernel-gpu-tdx-experimental) install_kernel_gpu_tdx_experimental;;
+	kernel-nvidia-gpu-tdx-experimental) install_kernel_nvidia_gpu_tdx_experimental;;
 
 	kernel-tdx-experimental) install_kernel_tdx_experimental ;;
 

From c9bf7808b6345c7902c46ad11204fc076fb7b894 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 4 May 2023 11:01:05 +0200
Subject: [PATCH 074/150] cache: Update the KERNEL_FLAVOUR list to include
 nvidia-gpu
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We need to make sure that, when caching a `-nvidia-gpu` kernel, we still
look at the version of the base kernel used to build the nvidia-gpu
drivers, as the ${vendor}-gpu kernels are based on already existing
entries in the versions.yaml file and do not require a new entry to be
added.

Fixes: #6777

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../static-build/cache_components_main.sh       | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index 28e6bfacf..5c4b56d6e 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -12,7 +12,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 source "${script_dir}/../scripts/lib.sh"
 
-KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental
+KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-nvidia-gpu | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental | kernel-nvidia-gpu-tdx-experimental | kernel-nvidia-gpu-snp
 OVMF_FLAVOUR="${OVMF_FLAVOUR:-x86_64}" # x86_64 | tdx
 QEMU_FLAVOUR="${QEMU_FLAVOUR:-qemu}" # qemu | qemu-tdx-experimental | qemu-snp-experimental
 ROOTFS_IMAGE_TYPE="${ROOTFS_IMAGE_TYPE:-image}" # image | initrd
@@ -36,6 +36,19 @@ cache_kernel_artifacts() {
 	local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")"
 	local kernel_modules_tarball_path="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel-sev-modules.tar.xz"
 
+	# The ${vendor}-gpu kernels are based on an already existing entry, and does not require
+	# adding a new entry to the versions.yaml.
+	#
+	# With this in mind, let's just make sure we get the version from correct entry in the
+	# versions.yaml file.
+	case ${KERNEL_FLAVOUR} in
+		*"nvidia-gpu"*)
+			KERNEL_FLAVOUR=${KERNEL_FLAVOUR//"-nvidia-gpu"/}
+			;;
+		*)
+			;;
+	esac
+
 	if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then
 		current_kernel_version="$(get_from_kata_deps "assets.kernel.sev.version")"
 	fi
@@ -130,7 +143,7 @@ Usage: $0 "[options]"
 		-c	Cloud hypervisor cache
 		-F	Firecracker cache
 		-k	Kernel cache
-			* Export KERNEL_FLAVOUR="kernel | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental" for a specific build
+			* Export KERNEL_FLAVOUR="kernel | kernel-nvidia-gpu | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental | kernel-nvidia-gpu-tdx-experimental | kernel-nvidia-gpu-snp" for a specific build
 			  The default KERNEL_FLAVOUR value is "kernel"
 		-n	Nydus cache
 		-q 	QEMU cache

From b9990c2017ca75398b824bd14072ec7e083aede5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 11 May 2023 13:53:11 +0200
Subject: [PATCH 075/150] cache: Fix nvidia-gpu version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

c9bf7808b6345c7902c46ad11204fc076fb7b894 introduced the logic to
properly get the version of nvidia-gpu kernels, but one important part
was dropped during the rebase into main, which is actually getting the
correct version of the kernel.

Fixing this now, and using the old issue as reference.

Fixes: #6777

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/cache_components_main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index 5c4b56d6e..599a50f2c 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -33,7 +33,6 @@ cache_kernel_artifacts() {
 	local kernel_tarball_name="kata-static-${KERNEL_FLAVOUR}.tar.xz"
 	local current_kernel_image="$(get_kernel_image_name)"
 	local current_kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
-	local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")"
 	local kernel_modules_tarball_path="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel-sev-modules.tar.xz"
 
 	# The ${vendor}-gpu kernels are based on an already existing entry, and does not require
@@ -49,6 +48,7 @@ cache_kernel_artifacts() {
 			;;
 	esac
 
+	local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")"
 	if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then
 		current_kernel_version="$(get_from_kata_deps "assets.kernel.sev.version")"
 	fi

From a133fadbfafc7529b9e2da6aa194b0e406a5adff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 11 May 2023 20:19:49 +0200
Subject: [PATCH 076/150] cache: Fix nvidia-gpu-tdx-experimental cache URL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We were passing "kernel-nvidia-gpu-tdx", missing the "-experimental"
part, leading to a non-valid URL.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 673b0c291..91828c53d 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -287,7 +287,7 @@ install_kernel_nvidia_gpu_tdx_experimental() {
 
 	install_kernel_helper \
 		"assets.kernel-tdx-experimental.version" \
-		"kernel-nvidia-gpu-tdx" \
+		"kernel-nvidia-gpu-tdx-experimental" \
 		"-x tdx -g nvidia -u ${kernel_url} -H deb"
 }
 

From 4d17ea4a018d3aabba3642544564353a4ddd86f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Thu, 11 May 2023 21:26:58 +0200
Subject: [PATCH 077/150] cache: Fix nvidia-snp caching version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All the kernel-foo instances, such as "kernel-sev" or "kernel-snp",
should be transformed into "kernel.foo" when looking at the
versions.yaml file.

This was already done for SEV, but missed on the SNP case.

Fixes: #6777

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../static-build/cache_components_main.sh          | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index 599a50f2c..ac9e3760b 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -48,10 +48,16 @@ cache_kernel_artifacts() {
 			;;
 	esac
 
-	local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")"
-	if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then
-		current_kernel_version="$(get_from_kata_deps "assets.kernel.sev.version")"
-	fi
+	case ${KERNEL_FLAVOUR} in
+		"kernel-sev"|"kernel-snp")
+			# In these cases, like "kernel-foo", it must be set to "kernel.foo" when looking at
+			# the versions.yaml file
+			current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR/-/.}.version")"
+			;;
+		*)
+			current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")"
+			;;
+	esac
 
 	create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}"
 	if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then

From 197c33651669ca88c1b0658cb800c90640b2cdc8 Mon Sep 17 00:00:00 2001
From: Shuaiyi Zhang <zhang_syi@qq.com>
Date: Fri, 12 May 2023 15:14:55 +0800
Subject: [PATCH 078/150] Dragonball: use
 LinuxBootConfigurator::write_bootparams to writes the boot parameters into
 guest memory.

Fixes: #6813

Signed-off-by: Shuaiyi Zhang <zhang_syi@qq.com>
---
 src/dragonball/src/vm/x86_64.rs | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs
index 04cf4605c..9593e8276 100644
--- a/src/dragonball/src/vm/x86_64.rs
+++ b/src/dragonball/src/vm/x86_64.rs
@@ -7,7 +7,6 @@
 // found in the THIRD-PARTY file.
 
 use std::convert::TryInto;
-use std::mem;
 use std::ops::Deref;
 
 use dbs_address_space::AddressSpace;
@@ -16,8 +15,9 @@ use dbs_utils::epoll_manager::EpollManager;
 use dbs_utils::time::TimestampUs;
 use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY};
 use linux_loader::cmdline::Cmdline;
+use linux_loader::configurator::{linux::LinuxBootConfigurator, BootConfigurator, BootParams};
 use slog::info;
-use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory};
+use vm_memory::{Address, GuestAddress, GuestAddressSpace, GuestMemory};
 
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::error::{Error, Result, StartMicroVmError};
@@ -110,15 +110,11 @@ fn configure_system<M: GuestMemory>(
         }
     }
 
-    let zero_page_addr = GuestAddress(layout::ZERO_PAGE_START);
-    guest_mem
-        .checked_offset(zero_page_addr, mem::size_of::<bootparam::boot_params>())
-        .ok_or(Error::ZeroPagePastRamEnd)?;
-    guest_mem
-        .write_obj(params, zero_page_addr)
-        .map_err(|_| Error::ZeroPageSetup)?;
-
-    Ok(())
+    LinuxBootConfigurator::write_bootparams(
+        &BootParams::new(&params, GuestAddress(layout::ZERO_PAGE_START)),
+        guest_mem,
+    )
+    .map_err(|_| Error::ZeroPageSetup)
 }
 
 impl Vm {

From cfe63527c5328611cac647d0c10916bef6cf0408 Mon Sep 17 00:00:00 2001
From: SinghWang <wangxin_0611@126.com>
Date: Sun, 12 Mar 2023 23:55:52 +0800
Subject: [PATCH 079/150] release: Fix multi-arch publishing is not supported
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When release is published, kata-deploy payload and kata-static package
can support multi-arch publishing.

Fixes: #6449

Signed-off-by: SinghWang <wangxin_0611@126.com>
Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/release-amd64.yaml |  63 +++++++++++++
 .github/workflows/release-arm64.yaml |  63 +++++++++++++
 .github/workflows/release-s390x.yaml |  63 +++++++++++++
 .github/workflows/release.yaml       | 134 +++++++++++++++++----------
 4 files changed, 275 insertions(+), 48 deletions(-)
 create mode 100644 .github/workflows/release-amd64.yaml
 create mode 100644 .github/workflows/release-arm64.yaml
 create mode 100644 .github/workflows/release-s390x.yaml

diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml
new file mode 100644
index 000000000..3e897d051
--- /dev/null
+++ b/.github/workflows/release-amd64.yaml
@@ -0,0 +1,63 @@
+name: Publish Kata release artifacts for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+
+  kata-deploy:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+
+      - name: build-and-push-kata-deploy-ci-amd64
+        id: build-and-push-kata-deploy-ci-amd64
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+
+      - name: push-tarball
+        run: |
+          # tag the container image we created and push to DockerHub
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-amd64.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-amd64.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+          done
diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml
new file mode 100644
index 000000000..2d54d0b62
--- /dev/null
+++ b/.github/workflows/release-arm64.yaml
@@ -0,0 +1,63 @@
+name: Publish Kata release artifacts for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+
+  kata-deploy:
+    needs: build-kata-static-tarball-arm64
+    runs-on: arm64
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: build-and-push-kata-deploy-ci-arm64
+        id: build-and-push-kata-deploy-ci-arm64
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+
+      - name: push-tarball
+        run: |
+          # tag the container image we created and push to DockerHub
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-arm64.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-arm64.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+          done
diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml
new file mode 100644
index 000000000..2497be7d2
--- /dev/null
+++ b/.github/workflows/release-s390x.yaml
@@ -0,0 +1,63 @@
+name: Publish Kata release artifacts for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+
+  kata-deploy:
+    needs: create-kata-tarball
+    runs-on: s390x
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v3
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-deploy-ci-s390x
+        id: build-and-push-kata-deploy-ci-s390x
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          pushd $GITHUB_WORKSPACE
+          git checkout $tag
+          pkg_sha=$(git rev-parse HEAD)
+          popd
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
+            "${pkg_sha}-${{ inputs.target-arch }}"
+
+      - name: push-tarball
+        run: |
+          # tag the container image we created and push to DockerHub
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tags=($tag)
+          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
+          for tag in ${tags[@]}; do
+            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-s390x.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-s390x.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
+            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+          done
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index a642fa36f..020804beb 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -5,73 +5,83 @@ on:
       - '[0-9]+.[0-9]+.[0-9]+*'
 
 jobs:
-  build-kata-static-tarball-amd64:
-    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+  build-and-push-assets-amd64:
+    uses: ./.github/workflows/release-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
 
-  kata-deploy:
-    needs: build-kata-static-tarball-amd64
+  build-and-push-assets-arm64:
+    uses: ./.github/workflows/release-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
+
+  build-and-push-assets-s390x:
+    uses: ./.github/workflows/release-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  publish-multi-arch-images:
     runs-on: ubuntu-latest
+    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
         with:
-          name: kata-static-tarball-amd64
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "PKG_SHA=${pkg_sha}" >> $GITHUB_OUTPUT
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
         with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
         run: |
           # tag the container image we created and push to DockerHub
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do \
-            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
-            docker push katadocker/kata-deploy:${tag} && \
-            docker push quay.io/kata-containers/kata-deploy:${tag}; \
+          # push to quay.io and docker.io
+          for tag in ${tags[@]}; do
+            docker manifest create quay.io/kata-containers/kata-deploy:${tag} \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-amd64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-arm64 \
+              --amend quay.io/kata-containers/kata-deploy:${tag}-s390x
+
+            docker manifest create docker.io/katadocker/kata-deploy:${tag} \
+              --amend docker.io/katadocker/kata-deploy:${tag}-amd64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-arm64 \
+              --amend docker.io/katadocker/kata-deploy:${tag}-s390x
+
+            docker manifest push quay.io/kata-containers/kata-deploy:${tag}
+            docker manifest push docker.io/katadocker/kata-deploy:${tag}
           done
 
-  upload-static-tarball:
-    needs: kata-deploy
+  upload-multi-arch-static-tarball:
+    needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: download-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
       - name: install hub
         run: |
           HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
           wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
           tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
+
+      - name: download-artifacts-amd64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64
+      - name: push amd64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-static-$tag-x86_64.tar.xz"
@@ -81,8 +91,36 @@ jobs:
           GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
           popd
 
+      - name: download-artifacts-arm64
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-arm64
+      - name: push arm64 static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-aarch64.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
+      - name: download-artifacts-s390x
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x
+      - name: push s390x static tarball to github
+        run: |
+          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
+          tarball="kata-static-$tag-s390x.tar.xz"
+          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
+          pushd $GITHUB_WORKSPACE
+          echo "uploading asset '${tarball}' for tag: ${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          popd
+
   upload-cargo-vendored-tarball:
-    needs: upload-static-tarball
+    needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

From e8f81ee93dac53eb6b2f4a43a6d9369ea9d259ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 12 May 2023 14:28:23 +0200
Subject: [PATCH 080/150] Revert "kata-deploy: Use readinessProbe to ensure
 everything is ready"

This reverts commit 5ec9ae0f0498b7366fc85ed1448d36e3c9b6ac35, for two
main reasons:
* The readinessProbe was misintepreted by myself when working on the
  original PR
* It's actually causing issues, as the pod ends up marked as not
  healthy.
---
 .../packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml  | 3 ---
 tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml  | 3 ---
 tools/packaging/kata-deploy/scripts/kata-deploy.sh             | 2 --
 3 files changed, 8 deletions(-)

diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
index 23c3efe02..095876b73 100644
--- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
@@ -21,9 +21,6 @@ spec:
         image: quay.io/kata-containers/kata-deploy:latest
         imagePullPolicy: Always
         command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
-        readinessProbe:
-          exec:
-            command: [ "bash", "-c", "[ -f /opt/kata/kata-deployed ]", "&&", "bash", "-c", "[ $? == 1 ]" ]
         env:
         - name: NODE_NAME
           valueFrom:
diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
index 5b5d835b6..97e98ee74 100644
--- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
+++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -18,9 +18,6 @@ spec:
       - name: kube-kata
         image: quay.io/kata-containers/kata-deploy:latest
         imagePullPolicy: Always
-        readinessProbe:
-          exec:
-            command: [ "bash", "-c", "[ -f /opt/kata/kata-deployed ]", "&&", "bash", "-c", "[ $? == 0 ]" ]
         lifecycle:
           preStop:
             exec:
diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index d6be5019f..7e9841ad8 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -325,13 +325,11 @@ function main() {
 			install_artifacts
 			configure_cri_runtime "$runtime"
 			kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true
-			touch /opt/kata/kata-deployed
 			;;
 		cleanup)
 			cleanup_cri_runtime "$runtime"
 			kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup
 			remove_artifacts
-			rm /opt/kata/kata-deployed
 			;;
 		reset)
 			reset_runtime $runtime

From 2856d3f23dfb7b844244e006c270fe244c80e81e Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Fri, 12 May 2023 16:14:19 +0100
Subject: [PATCH 081/150] deploy: Fix arch in image tag

`uname -m` produces `x86_64`, but container image convention
is to use `amd64`, so update this in the tag

Fixes: #6820
Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../local-build/kata-deploy-build-and-upload-payload.sh       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index b0cb5676b..74e0f992b 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -15,7 +15,9 @@ cp ${KATA_DEPLOY_ARTIFACT} ${KATA_DEPLOY_DIR}
 
 pushd ${KATA_DEPLOY_DIR}
 
-IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-$(uname -m)"
+local arch=$(uname -m)
+[ "$arch" = "x86_64" ] && arch="amd64"
+IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-${arch}"
 
 echo "Building the image"
 docker build --tag ${IMAGE_TAG} .

From 73913c8eb79135c7123f70c49d1e4207a8cf024c Mon Sep 17 00:00:00 2001
From: "James O. D. Hunt" <james.o.hunt@intel.com>
Date: Fri, 12 May 2023 16:38:37 +0100
Subject: [PATCH 082/150] kata-manager: Fix '-o' syntax and logic error

Fix the syntax and logic error that is only displayed if the user runs
the script with `-o`. This option requests that "only" Kata Containers
is installed and stops containerd from being installed.

Fixes: #6822.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
---
 utils/kata-manager.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh
index c51fb516c..249a510fa 100755
--- a/utils/kata-manager.sh
+++ b/utils/kata-manager.sh
@@ -262,7 +262,7 @@ pre_checks()
 	command -v "${kata_shim_v2}" &>/dev/null \
 		&& die "Please remove existing $kata_project installation"
 
-	[skip_containerd = "false" ] && return 0
+	[ "$skip_containerd" = 'true' ] && return 0
 
 	local ret
 

From ac88d34e0ca634787c786fc9c4479e0802ca6d52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 12 May 2023 20:50:18 +0200
Subject: [PATCH 083/150] static-build: Use relased binary for CLH (aarch64)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There's no need to build Cloud Hypervisor aarch64 as, for a few releases
already, Cloud Hypervisor provides an official release binary for the
architecture.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../static-build/cloud-hypervisor/build-static-clh.sh     | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
index 0bee1ea04..975a517a1 100755
--- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
+++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
@@ -44,6 +44,9 @@ pull_clh_released_binary() {
 	info "Download cloud-hypervisor version: ${cloud_hypervisor_version}"
 	cloud_hypervisor_binary="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${cloud_hypervisor_version}/cloud-hypervisor-static"
 
+	[ "${ARCH}" == "aarch64" ] && \
+		cloud_hypervisor_binary="${cloud_hypervisor_binary}-aarch64"
+
 	curl --fail -L ${cloud_hypervisor_binary} -o cloud-hypervisor-static || return 1
 	mkdir -p cloud-hypervisor
 	mv -f cloud-hypervisor-static cloud-hypervisor/cloud-hypervisor
@@ -82,11 +85,6 @@ build_clh_from_source() {
 	popd
 }
 
-if [ "${ARCH}" == "aarch64" ]; then
-	info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source"
-	force_build_from_source="true"
-fi
-
 if [ -n "${features}" ]; then
 	info "As an extra build argument has been passed to the script, forcing to build from source"
 	force_build_from_source="true"

From 3bf767cfcd5ca854d3a2e483afee55e592b9b5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 12 May 2023 21:16:39 +0200
Subject: [PATCH 084/150] static-build: Adjust ARCH for nydus
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When building from aarch64, just use "arm64" as that's what's used in
the name of the released nydus tarballs.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 91828c53d..70cfb19ff 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -436,6 +436,8 @@ install_virtiofsd() {
 
 # Install static nydus asset
 install_nydus() {
+	[ "${ARCH}" == "aarch64" ] && ARCH=arm64
+
 	install_cached_tarball_component \
 		"nydus" \
 		"${jenkins_url}/job/kata-containers-main-nydus-$(uname -m)/${cached_artifacts_path}" \

From 136e2415da8475e9ea911765e23bde8e6fbdde9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 12 May 2023 21:37:49 +0200
Subject: [PATCH 085/150] static-build: Download firecracker instead of
 building it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There's no reason for us to build firecracker instead of simply
downloading the official released tarball, as tarballs are provided for
the architectures we want to use them.

Fixes: #6770

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../local-build/kata-deploy-binaries.sh       |  8 +++--
 .../firecracker/build-static-firecracker.sh   | 33 ++++++++++---------
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 70cfb19ff..8dbebee42 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -377,10 +377,12 @@ install_qemu_snp_experimental() {
 
 # Install static firecracker asset
 install_firecracker() {
+	local firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version")
+
 	install_cached_tarball_component \
 		"firecracker" \
 		"${jenkins_url}/job/kata-containers-main-firecracker-$(uname -m)/${cached_artifacts_path}" \
-		"$(get_from_kata_deps "assets.hypervisor.firecracker.version")" \
+		"${firecracker_version}" \
 		"" \
 		"${final_tarball_name}" \
 		"${final_tarball_path}" \
@@ -390,8 +392,8 @@ install_firecracker() {
 	"${firecracker_builder}"
 	info "Install static firecracker"
 	mkdir -p "${destdir}/opt/kata/bin/"
-	sudo install -D --owner root --group root --mode 0744 firecracker/firecracker-static "${destdir}/opt/kata/bin/firecracker"
-	sudo install -D --owner root --group root --mode 0744 firecracker/jailer-static "${destdir}/opt/kata/bin/jailer"
+	sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/firecracker-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/firecracker"
+	sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/jailer-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/jailer"
 }
 
 # Install static cloud-hypervisor asset
diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh
index 159575410..77ffc0229 100755
--- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh
+++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh
@@ -14,30 +14,31 @@ source "${script_dir}/../../scripts/lib.sh"
 
 config_dir="${script_dir}/../../scripts/"
 
-firecracker_repo="${firecracker_repo:-}"
+firecracker_url="${firecracker_url:-}"
 firecracker_dir="firecracker"
 firecracker_version="${firecracker_version:-}"
 
 arch=$(uname -m)
 
-if [ -z "$firecracker_repo" ]; then
-	info "Get firecracker information from runtime versions.yaml"
-	firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url")
-	[ -n "$firecracker_url" ] || die "failed to get firecracker url"
-	firecracker_repo="${firecracker_url}.git"
-fi
-[ -n "$firecracker_repo" ] || die "failed to get firecracker repo"
+[ -n "$firecracker_url" ] ||firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url")
+[ -n "$firecracker_url" ] || die "failed to get firecracker url"
 
 [ -n "$firecracker_version" ] || firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version")
 [ -n "$firecracker_version" ] || die "failed to get firecracker version"
 
-info "Build ${firecracker_repo} version: ${firecracker_version}"
+firecracker_tarball_url="${firecracker_url}/releases/download"
 
-[ -d "${firecracker_dir}" ] || git clone ${firecracker_repo}
-cd "${firecracker_dir}"
-git fetch
-git checkout ${firecracker_version}
-sudo ./tools/devtool --unattended build --release
+file_name="firecracker-${firecracker_version}-${ARCH}.tgz"
+download_url="${firecracker_tarball_url}/${firecracker_version}/${file_name}"
 
-ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/firecracker ./firecracker-static
-ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/jailer ./jailer-static
+info "Download firecracker version: ${firecracker_version} from ${download_url}"
+curl -o ${file_name} -L $download_url
+
+sha256sum="${file_name}.sha256.txt"
+sha256sum_url="${firecracker_tarball_url}/${firecracker_version}/${sha256sum}"
+
+info "Download firecracker ${sha256sum} from ${sha256sum_url}"
+curl -o ${sha256sum} -L $sha256sum_url
+
+sha256sum -c ${sha256sum}
+tar zxvf ${file_name}

From 50cc9c582f154301f6786da725b9cbc84674c61c Mon Sep 17 00:00:00 2001
From: LiuWeijie <weijie.liu@intel.com>
Date: Wed, 26 Apr 2023 21:16:19 +0800
Subject: [PATCH 086/150] tests: Improve coverage for
 virtcontainers/pkg/compatoci/ for Kata 2.0

Add test cases for ParseConfigJson function and GetContainerSpec function

Fixes: #258

Signed-off-by: LiuWeijie <weijie.liu@intel.com>
---
 .../virtcontainers/pkg/compatoci/utils.go     |  6 +-
 .../pkg/compatoci/utils_test.go               | 96 ++++++++++++++++++-
 2 files changed, 94 insertions(+), 8 deletions(-)

diff --git a/src/runtime/virtcontainers/pkg/compatoci/utils.go b/src/runtime/virtcontainers/pkg/compatoci/utils.go
index 01568dd45..e8ea62911 100644
--- a/src/runtime/virtcontainers/pkg/compatoci/utils.go
+++ b/src/runtime/virtcontainers/pkg/compatoci/utils.go
@@ -74,7 +74,7 @@ func containerCapabilities(s compatOCISpec) (specs.LinuxCapabilities, error) {
 				}
 
 			default:
-				return c, fmt.Errorf("Unexpected format for capabilities: %v", caps)
+				return c, fmt.Errorf("unexpected format for capabilities: %v", caps)
 			}
 		}
 	case []interface{}:
@@ -94,7 +94,7 @@ func containerCapabilities(s compatOCISpec) (specs.LinuxCapabilities, error) {
 		ociLog.Debug("Empty capabilities have been passed")
 		return c, nil
 	default:
-		return c, fmt.Errorf("Unexpected format for capabilities: %v", caps)
+		return c, fmt.Errorf("unexpected format for capabilities: %v", caps)
 	}
 
 	return c, nil
@@ -154,5 +154,5 @@ func GetContainerSpec(annotations map[string]string) (specs.Spec, error) {
 
 	ociLog.Errorf("Annotations[%s] not found, cannot find container spec",
 		vcAnnotations.BundlePathKey)
-	return specs.Spec{}, fmt.Errorf("Could not find container spec")
+	return specs.Spec{}, fmt.Errorf("could not find container spec")
 }
diff --git a/src/runtime/virtcontainers/pkg/compatoci/utils_test.go b/src/runtime/virtcontainers/pkg/compatoci/utils_test.go
index ed8bdf864..aa44500bb 100644
--- a/src/runtime/virtcontainers/pkg/compatoci/utils_test.go
+++ b/src/runtime/virtcontainers/pkg/compatoci/utils_test.go
@@ -7,6 +7,8 @@ package compatoci
 
 import (
 	"encoding/json"
+	"os"
+	"path"
 	"path/filepath"
 	"testing"
 
@@ -78,6 +80,18 @@ func TestContainerCapabilities(t *testing.T) {
 	assert.Equal(t, c.Inheritable, []string{"CAP_KILL", "CAP_LEASE", "CAP_SYS_ADMIN"})
 	assert.Equal(t, c.Ambient, []string{""})
 
+	ociSpec.Process.Capabilities = map[string]interface{}{
+		"unexpected": interface{}(""),
+	}
+
+	c, err = ContainerCapabilities(ociSpec)
+	assert.NotNil(t, err)
+	assert.Equal(t, len(c.Bounding), 0)
+	assert.Equal(t, len(c.Effective), 0)
+	assert.Equal(t, len(c.Permitted), 0)
+	assert.Equal(t, len(c.Inheritable), 0)
+	assert.Equal(t, len(c.Ambient), 0)
+
 	ociSpec.Process.Capabilities = []interface{}{"CAP_LEASE", "CAP_SETUID"}
 
 	c, err = ContainerCapabilities(ociSpec)
@@ -92,11 +106,31 @@ func TestContainerCapabilities(t *testing.T) {
 
 	c, err = ContainerCapabilities(ociSpec)
 	assert.Nil(t, err)
-	assert.Equal(t, c.Bounding, []string(nil))
-	assert.Equal(t, c.Effective, []string(nil))
-	assert.Equal(t, c.Permitted, []string(nil))
-	assert.Equal(t, c.Inheritable, []string(nil))
-	assert.Equal(t, c.Ambient, []string(nil))
+	assert.Equal(t, len(c.Bounding), 0)
+	assert.Equal(t, len(c.Effective), 0)
+	assert.Equal(t, len(c.Permitted), 0)
+	assert.Equal(t, len(c.Inheritable), 0)
+	assert.Equal(t, len(c.Ambient), 0)
+
+	ociSpec.Process.Capabilities = interface{}("")
+
+	c, err = ContainerCapabilities(ociSpec)
+	assert.NotNil(t, err)
+	assert.Equal(t, len(c.Bounding), 0)
+	assert.Equal(t, len(c.Effective), 0)
+	assert.Equal(t, len(c.Permitted), 0)
+	assert.Equal(t, len(c.Inheritable), 0)
+	assert.Equal(t, len(c.Ambient), 0)
+
+	ociSpec.Process = nil
+
+	c, err = ContainerCapabilities(ociSpec)
+	assert.NotNil(t, err)
+	assert.Equal(t, len(c.Bounding), 0)
+	assert.Equal(t, len(c.Effective), 0)
+	assert.Equal(t, len(c.Permitted), 0)
+	assert.Equal(t, len(c.Inheritable), 0)
+	assert.Equal(t, len(c.Ambient), 0)
 }
 
 // use specs.Spec to decode the spec, the content of capabilities is [] string
@@ -145,3 +179,55 @@ func TestGetConfigPath(t *testing.T) {
 	configPath := getConfigPath(tempBundlePath)
 	assert.Equal(t, configPath, expected)
 }
+
+func TestParseConfigJSON(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	var ociSpec compatOCISpec
+	var configByte []byte
+
+	ociSpec.Spec.Version = "1.0.0"
+	ociSpec.Process = &compatOCIProcess{}
+	ociSpec.Process.Capabilities = map[string]interface{}{
+		"bounding":    []interface{}{"CAP_KILL"},
+		"effective":   []interface{}{"CAP_KILL", "CAP_LEASE"},
+		"permitted":   []interface{}{"CAP_SETUID"},
+		"inheritable": []interface{}{"CAP_KILL", "CAP_LEASE", "CAP_SYS_ADMIN"},
+		"ambient":     []interface{}{""},
+	}
+
+	file, err := os.Create(path.Join(tmpDir, "config.json"))
+	if err != nil {
+		t.Error("cannot create file")
+	}
+	defer file.Close()
+
+	if configByte, err = json.Marshal(ociSpec); err != nil {
+		t.Error("cannot marshal compat oci spec")
+	}
+
+	_, err = file.Write(configByte)
+	if err != nil {
+		t.Error("cannot write config data into file")
+	}
+
+	spec, err := ParseConfigJSON(tmpDir)
+	assert.Nil(t, err)
+	assert.Equal(t, spec.Version, "1.0.0")
+}
+
+func TestGetContainerSpec(t *testing.T) {
+	annotations := map[string]string{
+		"io.katacontainers.pkg.oci.bundle_path": "",
+	}
+
+	_, err := GetContainerSpec(annotations)
+	assert.NotNil(t, err)
+
+	annotations = map[string]string{
+		"io.katacontainers.pkg.oci.wrong_path": "",
+	}
+
+	_, err = GetContainerSpec(annotations)
+	assert.NotNil(t, err)
+}

From 777c3dc8d24122790d3fa2fbc97a5fdf3522c2e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 15 May 2023 09:17:54 +0200
Subject: [PATCH 087/150] kata-deploy: Do not ship the kata tarball
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There's absolutely no reason to ship the kata-static tarball as part of
the payload image, as:
* The tarball is already part of the release process
* The payload image already has uncompressed content of the tarball
* The tarball itself is not used anywhere by the kata-deploy scripts

Fixes: #6828

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile
index ee5519cd8..6bb470bb5 100644
--- a/tools/packaging/kata-deploy/Dockerfile
+++ b/tools/packaging/kata-deploy/Dockerfile
@@ -24,6 +24,7 @@ apt-get update && \
 apt-get install -y --no-install-recommends kubectl && \
 apt-get clean && rm -rf /var/lib/apt/lists/ && \
 mkdir -p ${DESTINATION} && \
-tar xvf ${KATA_ARTIFACTS} -C ${DESTINATION}
+tar xvf ${WORKDIR}/${KATA_ARTIFACTS} -C ${DESTINATION} && \
+rm -f ${WORKDIR}/${KATA_ARTIFACTS}
 
 COPY scripts ${DESTINATION}/scripts

From 5f3f844a1e1d50b4b192900f53ce5aa4b0bca1f9 Mon Sep 17 00:00:00 2001
From: Pavel Mores <pmores@redhat.com>
Date: Thu, 11 May 2023 11:15:33 +0200
Subject: [PATCH 088/150] runtime-rs: fix building instructions with respect to
 required Rust version

Fixes: #6803

Signed-off-by: Pavel Mores <pmores@redhat.com>
---
 .../kata-containers-3.0-rust-runtime-installation-guide.md  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md
index f83e4ea02..4cfcb392d 100644
--- a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md
+++ b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md
@@ -49,14 +49,14 @@ Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
 
 * Download `Rustup` and install  `Rust`
     > **Notes:**
-    > For Rust version, please see [`versions.yaml`](../../versions.yaml)  file's rust section.
+    > For Rust version, please set `RUST_VERSION` to the value of `languages.rust.meta.newest-version key` in [`versions.yaml`](../../versions.yaml) or, if `yq` is available on your system, run `export RUST_VERSION=$(yq read versions.yaml languages.rust.meta.newest-version)`.
 
     Example for `x86_64`
     ```
     $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
     $ source $HOME/.cargo/env
-    $ rustup install 1.62.0
-    $ rustup default 1.62.0-x86_64-unknown-linux-gnu
+    $ rustup install ${RUST_VERSION}
+    $ rustup default ${RUST_VERSION}-x86_64-unknown-linux-gnu
     ```
 
 * Musl support for fully static binary

From 3e85bf5b1769ac5f7dcdd13791eda07e86462787 Mon Sep 17 00:00:00 2001
From: Peteris Rudzusiks <rye@stripe.com>
Date: Mon, 15 May 2023 16:03:33 +0200
Subject: [PATCH 089/150] resource-control: fix setting CPU affinities on Linux

With this fix the vCPU pinning feature chooses the correct
physical cores to pin the vCPU threads on rather than always using core 0.

Fixes #6831

Signed-off-by: Peteris Rudzusiks <rye@stripe.com>
---
 src/runtime/pkg/resourcecontrol/utils_linux.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime/pkg/resourcecontrol/utils_linux.go b/src/runtime/pkg/resourcecontrol/utils_linux.go
index 59ca788f8..0acbc6c6a 100644
--- a/src/runtime/pkg/resourcecontrol/utils_linux.go
+++ b/src/runtime/pkg/resourcecontrol/utils_linux.go
@@ -156,7 +156,7 @@ func IsCgroupV1() (bool, error) {
 func SetThreadAffinity(threadID int, cpuSetSlice []int) error {
 	unixCPUSet := unix.CPUSet{}
 
-	for cpuId := range cpuSetSlice {
+	for _, cpuId := range cpuSetSlice {
 		unixCPUSet.Set(cpuId)
 	}
 

From b9a1db2601135e34ae9e59e927d3a0e8bc8d0a4e Mon Sep 17 00:00:00 2001
From: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
Date: Mon, 15 May 2023 15:57:29 +0000
Subject: [PATCH 090/150] kata-deploy: Add http_proxy as part of the docker
 build

Add http_proxy and https_proxy as part of the docker build arguments
in order to build properly when we are behind a proxy.

Fixes #6834

Signed-off-by: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
---
 .../kata-deploy/local-build/kata-deploy-binaries-in-docker.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
index defb338d9..70460a33b 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
@@ -15,6 +15,8 @@ kata_dir=$(realpath "${script_dir}/../../../../")
 kata_deploy_create="${script_dir}/kata-deploy-binaries.sh"
 uid=$(id -u ${USER})
 gid=$(id -g ${USER})
+http_proxy="${http_proxy:-}"
+https_proxy="${https_proxy:-}"
 
 if [ "${script_dir}" != "${PWD}" ]; then
 	ln -sf "${script_dir}/build" "${PWD}/build"
@@ -41,6 +43,8 @@ docker build -q -t build-kata-deploy \
 	--build-arg IMG_USER="${USER}" \
 	--build-arg UID=${uid} \
 	--build-arg GID=${gid} \
+	--build-arg http_proxy="${http_proxy}" \
+	--build-arg https_proxy="${https_proxy}" \
 	--build-arg HOST_DOCKER_GID=${docker_gid} \
 	"${script_dir}/dockerbuild/"
 

From 20cb87508722dcc49135c51887888fd3edbf051b Mon Sep 17 00:00:00 2001
From: Tamas K Lengyel <tamas.lengyel@intel.com>
Date: Tue, 25 Apr 2023 13:53:39 +0000
Subject: [PATCH 091/150] virtcontainers/qemu_test.go: Improve test coverage

Rework TestQemuCreateVM routine to be a table driven test with
various config variations passed to it. After CreateVM a handful
of additional functions are exercised to improve code-coverage.
Also add partial coverage for StartVM routine.

Currently improving from 19.7% to 35.7%

Credit PR to Hackathon Team3

Fixes: #267

Signed-off-by: Tamas K Lengyel <tamas.lengyel@intel.com>
---
 src/runtime/virtcontainers/qemu_test.go | 173 +++++++++++++++++++++---
 1 file changed, 155 insertions(+), 18 deletions(-)

diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go
index a8bc6a33d..bfa348145 100644
--- a/src/runtime/virtcontainers/qemu_test.go
+++ b/src/runtime/virtcontainers/qemu_test.go
@@ -76,24 +76,10 @@ func TestQemuKernelParameters(t *testing.T) {
 }
 
 func TestQemuCreateVM(t *testing.T) {
-	qemuConfig := newQemuConfig()
 	assert := assert.New(t)
 
 	store, err := persist.GetDriver()
 	assert.NoError(err)
-	q := &qemu{
-		config: HypervisorConfig{
-			VMStorePath:  store.RunVMStoragePath(),
-			RunStorePath: store.RunStoragePath(),
-		},
-	}
-	sandbox := &Sandbox{
-		ctx: context.Background(),
-		id:  "testSandbox",
-		config: &SandboxConfig{
-			HypervisorConfig: qemuConfig,
-		},
-	}
 
 	// Create the hypervisor fake binary
 	testQemuPath := filepath.Join(testDir, testHypervisor)
@@ -101,15 +87,142 @@ func TestQemuCreateVM(t *testing.T) {
 	assert.NoError(err)
 
 	// Create parent dir path for hypervisor.json
-	parentDir := filepath.Join(store.RunStoragePath(), sandbox.id)
+	parentDir := filepath.Join(store.RunStoragePath(), "testSandbox")
 	assert.NoError(os.MkdirAll(parentDir, DirMode))
 
 	network, err := NewNetwork()
 	assert.NoError(err)
-	err = q.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig)
-	assert.NoError(err)
+
+	config0 := newQemuConfig()
+
+	config1 := newQemuConfig()
+	config1.SeccompSandbox = "enable=1"
+
+	config2 := newQemuConfig()
+	config2.InitrdPath = ""
+	config2.ImagePath = testQemuImagePath
+
+	config3 := newQemuConfig()
+	config3.Debug = true
+
+	config5 := newQemuConfig()
+	config5.GuestMemoryDumpPath = "/tmp"
+
+	config6 := newQemuConfig()
+	config6.DisableGuestSeLinux = false
+
+	config7 := newQemuConfig()
+	config7.PCIeRootPort = 1
+
+	config8 := newQemuConfig()
+	config8.EnableVhostUserStore = true
+	config8.HugePages = true
+
+	config9 := newQemuConfig()
+	config9.EnableVhostUserStore = true
+	config9.HugePages = false
+
+	config10 := newQemuConfig()
+	config10.BootToBeTemplate = true
+
+	config11 := newQemuConfig()
+	config11.BootFromTemplate = true
+
+	config12 := newQemuConfig()
+	config12.BootToBeTemplate = true
+	config12.SharedFS = config.VirtioFS
+
+	config13 := newQemuConfig()
+	config13.FileBackedMemRootDir = "/tmp/xyzabc"
+	config13.HugePages = true
+
+	config14 := newQemuConfig()
+	config14.SharedFS = config.VirtioFS
+
+	config15 := newQemuConfig()
+	config15.BlockDeviceDriver = ""
+
+	config16 := newQemuConfig()
+	config16.SharedFS = config.VirtioFSNydus
+
+	config17 := newQemuConfig()
+	config17.VMid = "testSandbox"
+
+	type testData struct {
+		config      HypervisorConfig
+		expectError bool
+		configMatch bool
+	}
+
+	data := []testData{
+		{config0, false, true},
+		{config1, false, true},
+		{config2, false, true},
+		{config3, false, true},
+		{config5, false, true},
+		{config6, false, false},
+		{config7, false, true},
+		{config8, false, true},
+		{config9, true, false},
+		{config10, false, true},
+		{config11, false, true},
+		{config12, true, false},
+		{config13, false, true},
+		{config14, false, true},
+		{config15, false, true},
+		{config16, false, true},
+		{config17, false, true},
+	}
+
+	for i, d := range data {
+		msg := fmt.Sprintf("test[%d]", i)
+
+		q := &qemu{
+			config: HypervisorConfig{
+				VMStorePath:  store.RunVMStoragePath(),
+				RunStorePath: store.RunStoragePath(),
+			},
+		}
+
+		err = q.CreateVM(context.Background(), "testSandbox", network, &d.config)
+
+		if d.expectError {
+			assert.Error(err, msg)
+			continue
+		}
+
+		assert.NoError(err, msg)
+
+		if d.configMatch {
+			assert.Exactly(d.config, q.config, msg)
+		}
+
+		mem := q.GetTotalMemoryMB(context.Background())
+		assert.True(mem > 0)
+
+		err = q.canDumpGuestMemory("/tmp")
+		assert.NoError(err)
+
+		err = q.dumpGuestMemory("")
+		assert.NoError(err)
+
+		q.dumpSandboxMetaInfo("/tmp/")
+
+		// now we exercise code that should fail since the VM isn't running
+		err = q.dumpGuestMemory("/tmp")
+		assert.Error(err)
+
+		err = q.setupVirtioMem(context.Background())
+		assert.Error(err)
+
+		err = q.SaveVM()
+		assert.Error(err)
+
+		err = q.StopVM(context.Background(), true)
+		assert.Error(err)
+	}
+
 	assert.NoError(os.RemoveAll(parentDir))
-	assert.Exactly(qemuConfig, q.config)
 }
 
 func TestQemuCreateVMMissingParentDirFail(t *testing.T) {
@@ -636,3 +749,27 @@ func TestQemuSetConfig(t *testing.T) {
 
 	assert.Equal(q.config, config)
 }
+
+func TestQemuStartSandbox(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox, err := createQemuSandboxConfig()
+	assert.NoError(err)
+
+	network, err := NewNetwork()
+	assert.NoError(err)
+
+	q := &qemu{
+		config: HypervisorConfig{
+			VMStorePath:  sandbox.store.RunVMStoragePath(),
+			RunStorePath: sandbox.store.RunStoragePath(),
+		},
+		virtiofsDaemon: &virtiofsdMock{},
+	}
+
+	err = q.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig)
+	assert.NoError(err)
+
+	err = q.StartVM(context.Background(), 10)
+	assert.Error(err)
+}

From bdb75fb21ee5488317c4383109bfb836c4faca93 Mon Sep 17 00:00:00 2001
From: Peteris Rudzusiks <rye@stripe.com>
Date: Mon, 15 May 2023 16:59:36 +0200
Subject: [PATCH 092/150] runtime: use enable_vcpus_pinning from toml

Set the default value of runtime's EnableVCPUsPinning to value read from .toml.

Fixes: #6836

Signed-off-by: Peteris Rudzusiks <rye@stripe.com>
---
 src/runtime/pkg/oci/utils.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go
index d2d713ff8..ff5c1912b 100644
--- a/src/runtime/pkg/oci/utils.go
+++ b/src/runtime/pkg/oci/utils.go
@@ -963,6 +963,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 
 		DisableGuestSeccomp: runtime.DisableGuestSeccomp,
 
+		EnableVCPUsPinning: runtime.EnableVCPUsPinning,
+
 		GuestSeLinuxLabel: runtime.GuestSeLinuxLabel,
 
 		Experimental: runtime.Experimental,

From 593840e0758c97c7926c24362546344243590839 Mon Sep 17 00:00:00 2001
From: Narendra Patel <narendra.g.patel@intel.com>
Date: Mon, 15 May 2023 17:21:49 -0400
Subject: [PATCH 093/150] kata-ctl: Allow INSTALL_PATH= to be specified

Update the kata-ctl install rule to allow it to be installed to a given directory

The Makefile was updated to use an INSTALL_PATH variable to track where the
kata-ctl binary should be installed.  If the user doesn't specify anything,
then it uses the default path that cargo uses.  Otherwise, it will install it
in the directory that the user specified.  The README.md file was also updated
to show how to use the new option.

Fixes #5403

Co-authored-by: Cesar Tamayo <cesar.tamayo@intel.com>
Co-authored-by: Kevin Mora Jimenez <kevin.mora.jimenez@intel.com>
Co-authored-by: Narendra Patel <narendra.g.patel@intel.com>
Co-authored-by: Ray Karrenbauer <ray.karrenbauer@intel.com>
Co-authored-by: Srinath Duraisamy <srinath.duraisamy@intel.com>
Signed-off-by: Narendra Patel <narendra.g.patel@intel.com>
---
 src/tools/kata-ctl/Makefile  | 3 ++-
 src/tools/kata-ctl/README.md | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/tools/kata-ctl/Makefile b/src/tools/kata-ctl/Makefile
index fbbd5c112..23ae7ca1e 100644
--- a/src/tools/kata-ctl/Makefile
+++ b/src/tools/kata-ctl/Makefile
@@ -10,6 +10,7 @@ PROJECT_URL = https://github.com/kata-containers
 PROJECT_COMPONENT = kata-ctl
 
 TARGET = $(PROJECT_COMPONENT)
+INSTALL_PATH = $(HOME)/.cargo
 
 VERSION_FILE := ./VERSION
 export VERSION := $(shell grep -v ^\# $(VERSION_FILE))
@@ -55,7 +56,7 @@ test:
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo test --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 install:
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path .
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . --root $(INSTALL_PATH)
 
 check: standard_rust_check
 
diff --git a/src/tools/kata-ctl/README.md b/src/tools/kata-ctl/README.md
index 181e66db7..c37ab9dcd 100644
--- a/src/tools/kata-ctl/README.md
+++ b/src/tools/kata-ctl/README.md
@@ -27,6 +27,11 @@ $ make
 $ make install
 ```
 
+If you would like to install the tool to a specific directory, then you can provide it through the `INSTALL_PATH` variable.
+```bash
+$ make install INSTALL_PATH=/path/to/your/custom/install/directory
+```
+
 ## Run the tool
 
 ```bash

From 4cde844f70027a3742447e4a478bf273f53c4325 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 16 May 2023 13:34:52 +0200
Subject: [PATCH 094/150] local-build: Fix kernel-nvidia-gpu target name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It must have `-tarball` as part of its name.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/local-build/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile
index 59ce2d06b..091e3dc21 100644
--- a/tools/packaging/kata-deploy/local-build/Makefile
+++ b/tools/packaging/kata-deploy/local-build/Makefile
@@ -24,7 +24,7 @@ all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh
 all: serial-targets \
 	firecracker-tarball \
 	kernel-dragonball-experimental-tarball \
-	kernel-nvidia-gpu \
+	kernel-nvidia-gpu-tarball \
 	kernel-nvidia-gpu-snp-tarball \
 	kernel-nvidia-gpu-tdx-experimental-tarball \
 	kernel-tarball \

From 3a9d3c72aaee8a15be2045deb8ac357a222b837f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 16 May 2023 13:44:30 +0200
Subject: [PATCH 095/150] gpu: Rename the last bits from `gpu` to `nvidia-gpu`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's specifically name the `gpu` runtime class as `nvidia-gpu`.  By
doing this we keep the door open and ease the life of the next vendor
adding GPU support for Kata Containers.

Fixes: #6553

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 src/runtime/Makefile                                   | 10 +++++-----
 ...u.toml.in => configuration-qemu-nvidia-gpu.toml.in} |  0
 .../runtimeclasses/kata-runtimeClasses.yaml            |  4 ++--
 tools/packaging/kata-deploy/scripts/kata-deploy.sh     |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)
 rename src/runtime/config/{configuration-qemu-gpu.toml.in => configuration-qemu-nvidia-gpu.toml.in} (100%)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index 776da3ef4..a06aa6fbd 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -96,10 +96,10 @@ GENERATED_VARS = \
 		VERSION \
 		CONFIG_ACRN_IN \
 		CONFIG_QEMU_IN \
+		CONFIG_QEMU_NVIDIA_GPU_IN \
 		CONFIG_QEMU_SEV_IN \
 		CONFIG_QEMU_TDX_IN \
 		CONFIG_QEMU_SNP_IN \
-		CONFIG_QEMU_GPU_IN \
 		CONFIG_CLH_IN \
 		CONFIG_FC_IN \
 		$(USER_VARS)
@@ -325,11 +325,11 @@ ifneq (,$(QEMUCMD))
 
     CONFIGS += $(CONFIG_QEMU_SNP)
 
-    CONFIG_FILE_QEMU_GPU = configuration-qemu-gpu.toml
-    CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU)
-    CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in
+    CONFIG_FILE_QEMU_NVIDIA_GPU = configuration-qemu-nvidia-gpu.toml
+    CONFIG_QEMU_NVIDIA_GPU = config/$(CONFIG_FILE_QEMU_NVIDIA_GPU)
+    CONFIG_QEMU_NVIDIA_GPU_IN = $(CONFIG_QEMU_NVIDIA_GPU).in
 
-    CONFIGS += $(CONFIG_QEMU_GPU)
+    CONFIGS += $(CONFIG_QEMU_NVIDIA_GPU)
 
     # qemu-specific options (all should be suffixed by "_QEMU")
     DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi
diff --git a/src/runtime/config/configuration-qemu-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in
similarity index 100%
rename from src/runtime/config/configuration-qemu-gpu.toml.in
rename to src/runtime/config/configuration-qemu-nvidia-gpu.toml.in
diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
index 4e8da4fea..b55096226 100644
--- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
@@ -93,8 +93,8 @@ scheduling:
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
 metadata:
-    name: kata-qemu-gpu
-handler: kata-qemu-gpu
+    name: kata-qemu-nvidia-gpu
+handler: kata-qemu-nvidia-gpu
 overhead:
     podFixed:
         memory: "160Mi"
diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 7e9841ad8..68fd59b30 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -16,8 +16,8 @@ containerd_conf_file_backup="${containerd_conf_file}.bak"
 shims=(
 	"fc"
 	"qemu"
+	"qemu-nvidia-gpu"
 	"qemu-tdx"
-	"qemu-gpu"
 	"qemu-snp"
 	"clh"
 	"dragonball"

From 05de7b260762af1f1afc4b881b3853540d802bba Mon Sep 17 00:00:00 2001
From: Dov Murik <dovmurik@linux.ibm.com>
Date: Mon, 8 May 2023 10:22:35 +0300
Subject: [PATCH 096/150] runtime: Add sev package

The sev package provides utilities for launching AMD SEV and SEV-ES
confidential guests.

Fixes: #6795

Signed-off-by: Dov Murik <dovmurik@linux.ibm.com>
---
 src/runtime/pkg/README.md                    |   1 +
 src/runtime/pkg/sev/README.md                |  14 ++
 src/runtime/pkg/sev/ovmf.go                  | 101 +++++++++
 src/runtime/pkg/sev/sev.go                   | 203 +++++++++++++++++++
 src/runtime/pkg/sev/sev_test.go              |  54 +++++
 src/runtime/pkg/sev/testdata/README.md       |   9 +
 src/runtime/pkg/sev/testdata/ovmf_suffix.bin | Bin 0 -> 4096 bytes
 src/runtime/pkg/sev/vcpu_sigs.go             |  76 +++++++
 src/runtime/pkg/sev/vcpu_sigs_test.go        |  21 ++
 src/runtime/pkg/sev/vmsa.go                  | 172 ++++++++++++++++
 10 files changed, 651 insertions(+)
 create mode 100644 src/runtime/pkg/sev/README.md
 create mode 100644 src/runtime/pkg/sev/ovmf.go
 create mode 100644 src/runtime/pkg/sev/sev.go
 create mode 100644 src/runtime/pkg/sev/sev_test.go
 create mode 100644 src/runtime/pkg/sev/testdata/README.md
 create mode 100644 src/runtime/pkg/sev/testdata/ovmf_suffix.bin
 create mode 100644 src/runtime/pkg/sev/vcpu_sigs.go
 create mode 100644 src/runtime/pkg/sev/vcpu_sigs_test.go
 create mode 100644 src/runtime/pkg/sev/vmsa.go

diff --git a/src/runtime/pkg/README.md b/src/runtime/pkg/README.md
index b5b0d07d3..72bf3248b 100644
--- a/src/runtime/pkg/README.md
+++ b/src/runtime/pkg/README.md
@@ -7,4 +7,5 @@ This repository contains a number of packages in addition to the
 |-|-|
 | [`katatestutils`](katatestutils) | Unit test utilities. |
 | [`katautils`](katautils) | Utilities. |
+| [`sev`](sev) | AMD SEV confidential guest utilities. |
 | [`signals`](signals) | Signal handling functions. |
diff --git a/src/runtime/pkg/sev/README.md b/src/runtime/pkg/sev/README.md
new file mode 100644
index 000000000..8e864f46f
--- /dev/null
+++ b/src/runtime/pkg/sev/README.md
@@ -0,0 +1,14 @@
+# AMD SEV confidential guest utilities
+
+This package provides utilities for launching AMD SEV confidential guests.
+
+## Calculating expected launch digests
+
+The `CalculateLaunchDigest` function can be used to calculate the expected
+SHA-256 of an SEV confidential guest given its firmware, kernel, initrd, and
+kernel command-line.
+
+### Unit test data
+
+The [`testdata`](testdata) directory contains file used for testing
+`CalculateLaunchDigest`.
diff --git a/src/runtime/pkg/sev/ovmf.go b/src/runtime/pkg/sev/ovmf.go
new file mode 100644
index 000000000..9c6947abe
--- /dev/null
+++ b/src/runtime/pkg/sev/ovmf.go
@@ -0,0 +1,101 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package sev
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"os"
+)
+
+// GUID 96b582de-1fb2-45f7-baea-a366c55a082d
+var ovmfTableFooterGuid = guidLE{0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d}
+
+// GUID 00f771de-1a7e-4fcb-890e-68c77e2fb44e
+var sevEsResetBlockGuid = guidLE{0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e}
+
+type ovmfFooterTableEntry struct {
+	Size uint16
+	Guid guidLE
+}
+
+type ovmf struct {
+	table map[guidLE][]byte
+}
+
+func NewOvmf(filename string) (ovmf, error) {
+	buf, err := os.ReadFile(filename)
+	if err != nil {
+		return ovmf{}, err
+	}
+	table, err := parseFooterTable(buf)
+	if err != nil {
+		return ovmf{}, err
+	}
+	return ovmf{table}, nil
+}
+
+// Parse the OVMF footer table and return a map from GUID to entry value
+func parseFooterTable(data []byte) (map[guidLE][]byte, error) {
+	table := make(map[guidLE][]byte)
+
+	buf := new(bytes.Buffer)
+	err := binary.Write(buf, binary.LittleEndian, ovmfFooterTableEntry{})
+	if err != nil {
+		return table, err
+	}
+	entryHeaderSize := buf.Len()
+
+	// The OVMF table ends 32 bytes before the end of the firmware binary
+	startOfFooterTable := len(data) - 32 - entryHeaderSize
+	footerBytes := bytes.NewReader(data[startOfFooterTable:])
+	var footer ovmfFooterTableEntry
+	err = binary.Read(footerBytes, binary.LittleEndian, &footer)
+	if err != nil {
+		return table, err
+	}
+	if footer.Guid != ovmfTableFooterGuid {
+		// No OVMF footer table
+		return table, nil
+	}
+	tableSize := int(footer.Size) - entryHeaderSize
+	if tableSize < 0 {
+		return table, nil
+	}
+	tableBytes := data[(startOfFooterTable - tableSize):startOfFooterTable]
+	for len(tableBytes) >= entryHeaderSize {
+		tsize := len(tableBytes)
+		entryBytes := bytes.NewReader(tableBytes[tsize-entryHeaderSize:])
+		var entry ovmfFooterTableEntry
+		err := binary.Read(entryBytes, binary.LittleEndian, &entry)
+		if err != nil {
+			return table, err
+		}
+		if int(entry.Size) < entryHeaderSize {
+			return table, errors.New("Invalid entry size")
+		}
+		entryData := tableBytes[tsize-int(entry.Size) : tsize-entryHeaderSize]
+		table[entry.Guid] = entryData
+		tableBytes = tableBytes[:tsize-int(entry.Size)]
+	}
+	return table, nil
+}
+
+func (o *ovmf) tableItem(guid guidLE) ([]byte, error) {
+	value, ok := o.table[guid]
+	if !ok {
+		return []byte{}, errors.New("OVMF footer table entry not found")
+	}
+	return value, nil
+}
+
+func (o *ovmf) sevEsResetEip() (uint32, error) {
+	value, err := o.tableItem(sevEsResetBlockGuid)
+	if err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint32(value), nil
+}
diff --git a/src/runtime/pkg/sev/sev.go b/src/runtime/pkg/sev/sev.go
new file mode 100644
index 000000000..bdf73cf60
--- /dev/null
+++ b/src/runtime/pkg/sev/sev.go
@@ -0,0 +1,203 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Package sev can be used to compute the expected hash values for
+// SEV/-ES pre-launch attestation
+package sev
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"io"
+	"os"
+)
+
+type guidLE [16]byte
+
+// The following definitions must be identical to those in QEMU target/i386/sev.c
+
+// GUID: 9438d606-4f22-4cc9-b479-a793d411fd21
+var sevHashTableHeaderGuid = guidLE{0x06, 0xd6, 0x38, 0x94, 0x22, 0x4f, 0xc9, 0x4c, 0xb4, 0x79, 0xa7, 0x93, 0xd4, 0x11, 0xfd, 0x21}
+
+// GUID: 4de79437-abd2-427f-b835-d5b172d2045b
+var sevKernelEntryGuid = guidLE{0x37, 0x94, 0xe7, 0x4d, 0xd2, 0xab, 0x7f, 0x42, 0xb8, 0x35, 0xd5, 0xb1, 0x72, 0xd2, 0x04, 0x5b}
+
+// GUID: 44baf731-3a2f-4bd7-9af1-41e29169781d
+var sevInitrdEntryGuid = guidLE{0x31, 0xf7, 0xba, 0x44, 0x2f, 0x3a, 0xd7, 0x4b, 0x9a, 0xf1, 0x41, 0xe2, 0x91, 0x69, 0x78, 0x1d}
+
+// GUID: 97d02dd8-bd20-4c94-aa78-e7714d36ab2a
+var sevCmdlineEntryGuid = guidLE{0xd8, 0x2d, 0xd0, 0x97, 0x20, 0xbd, 0x94, 0x4c, 0xaa, 0x78, 0xe7, 0x71, 0x4d, 0x36, 0xab, 0x2a}
+
+type sevHashTableEntry struct {
+	entryGuid guidLE
+	length    uint16
+	hash      [sha256.Size]byte
+}
+
+type sevHashTable struct {
+	tableGuid guidLE
+	length    uint16
+	cmdline   sevHashTableEntry
+	initrd    sevHashTableEntry
+	kernel    sevHashTableEntry
+}
+
+type paddedSevHashTable struct {
+	table   sevHashTable
+	padding [8]byte
+}
+
+func fileSha256(filename string) (res [sha256.Size]byte, err error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return res, err
+	}
+	defer f.Close()
+
+	digest := sha256.New()
+	if _, err := io.Copy(digest, f); err != nil {
+		return res, err
+	}
+
+	copy(res[:], digest.Sum(nil))
+	return res, nil
+}
+
+func constructSevHashesTable(kernelPath, initrdPath, cmdline string) ([]byte, error) {
+	kernelHash, err := fileSha256(kernelPath)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	initrdHash, err := fileSha256(initrdPath)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	cmdlineHash := sha256.Sum256(append([]byte(cmdline), 0))
+
+	buf := new(bytes.Buffer)
+	err = binary.Write(buf, binary.LittleEndian, sevHashTableEntry{})
+	if err != nil {
+		return []byte{}, err
+	}
+	entrySize := uint16(buf.Len())
+
+	buf = new(bytes.Buffer)
+	err = binary.Write(buf, binary.LittleEndian, sevHashTable{})
+	if err != nil {
+		return []byte{}, err
+	}
+	tableSize := uint16(buf.Len())
+
+	ht := paddedSevHashTable{
+		table: sevHashTable{
+			tableGuid: sevHashTableHeaderGuid,
+			length:    tableSize,
+			cmdline: sevHashTableEntry{
+				entryGuid: sevCmdlineEntryGuid,
+				length:    entrySize,
+				hash:      cmdlineHash,
+			},
+			initrd: sevHashTableEntry{
+				entryGuid: sevInitrdEntryGuid,
+				length:    entrySize,
+				hash:      initrdHash,
+			},
+			kernel: sevHashTableEntry{
+				entryGuid: sevKernelEntryGuid,
+				length:    entrySize,
+				hash:      kernelHash,
+			},
+		},
+		padding: [8]byte{0, 0, 0, 0, 0, 0, 0, 0},
+	}
+
+	htBuf := new(bytes.Buffer)
+	err = binary.Write(htBuf, binary.LittleEndian, ht)
+	if err != nil {
+		return []byte{}, err
+	}
+	return htBuf.Bytes(), nil
+}
+
+// CalculateLaunchDigest returns the sha256 encoded SEV launch digest based off
+// the current firmware, kernel, initrd, and the kernel cmdline
+func CalculateLaunchDigest(firmwarePath, kernelPath, initrdPath, cmdline string) (res [sha256.Size]byte, err error) {
+	f, err := os.Open(firmwarePath)
+	if err != nil {
+		return res, err
+	}
+	defer f.Close()
+
+	digest := sha256.New()
+	if _, err := io.Copy(digest, f); err != nil {
+		return res, err
+	}
+
+	// When used for confidential containers in kata-containers, kernelPath
+	// is always set (direct boot).  However, this current package can also
+	// be used by other programs which may calculate launch digests of
+	// arbitrary SEV guests without SEV kernel hashes table.
+	if kernelPath != "" {
+		ht, err := constructSevHashesTable(kernelPath, initrdPath, cmdline)
+		if err != nil {
+			return res, err
+		}
+		digest.Write(ht)
+	}
+
+	copy(res[:], digest.Sum(nil))
+	return res, nil
+}
+
+// CalculateSEVESLaunchDigest returns the sha256 encoded SEV-ES launch digest
+// based off the current firmware, kernel, initrd, and the kernel cmdline, and
+// the number of vcpus and their type
+func CalculateSEVESLaunchDigest(vcpus int, vcpuSig VCPUSig, firmwarePath, kernelPath, initrdPath, cmdline string) (res [sha256.Size]byte, err error) {
+	f, err := os.Open(firmwarePath)
+	if err != nil {
+		return res, err
+	}
+	defer f.Close()
+
+	digest := sha256.New()
+	if _, err := io.Copy(digest, f); err != nil {
+		return res, err
+	}
+
+	// When used for confidential containers in kata-containers, kernelPath
+	// is always set (direct boot).  However, this current package can also
+	// be used by other programs which may calculate launch digests of
+	// arbitrary SEV guests without SEV kernel hashes table.
+	if kernelPath != "" {
+		ht, err := constructSevHashesTable(kernelPath, initrdPath, cmdline)
+		if err != nil {
+			return res, err
+		}
+		digest.Write(ht)
+	}
+
+	o, err := NewOvmf(firmwarePath)
+	if err != nil {
+		return res, err
+	}
+	resetEip, err := o.sevEsResetEip()
+	if err != nil {
+		return res, err
+	}
+	v := vmsaBuilder{uint64(resetEip), vcpuSig}
+	for i := 0; i < vcpus; i++ {
+		vmsaPage, err := v.buildPage(i)
+		if err != nil {
+			return res, err
+		}
+		digest.Write(vmsaPage)
+	}
+
+	copy(res[:], digest.Sum(nil))
+	return res, nil
+}
diff --git a/src/runtime/pkg/sev/sev_test.go b/src/runtime/pkg/sev/sev_test.go
new file mode 100644
index 000000000..68a82ea90
--- /dev/null
+++ b/src/runtime/pkg/sev/sev_test.go
@@ -0,0 +1,54 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package sev
+
+import (
+	"encoding/hex"
+	"testing"
+)
+
+func TestCalculateLaunchDigestWithoutKernelHashes(t *testing.T) {
+	ld, err := CalculateLaunchDigest("testdata/ovmf_suffix.bin", "", "", "")
+	if err != nil {
+		t.Fatalf("unexpected err value: %s", err)
+	}
+	hexld := hex.EncodeToString(ld[:])
+	if hexld != "b184e06e012366fd7b33ebfb361a515d05f00d354dca07b36abbc1e1e177ced5" {
+		t.Fatalf("wrong measurement: %s", hexld)
+	}
+}
+
+func TestCalculateLaunchDigestWithKernelHashes(t *testing.T) {
+	ld, err := CalculateLaunchDigest("testdata/ovmf_suffix.bin", "/dev/null", "/dev/null", "")
+	if err != nil {
+		t.Fatalf("unexpected err value: %s", err)
+	}
+	hexld := hex.EncodeToString(ld[:])
+	if hexld != "d59d7696efd7facfaa653758586e6120c4b6eaec3e327771d278cc6a44786ba5" {
+		t.Fatalf("wrong measurement: %s", hexld)
+	}
+}
+
+func TestCalculateLaunchDigestWithKernelHashesSevEs(t *testing.T) {
+	ld, err := CalculateSEVESLaunchDigest(1, SigEpycV4, "testdata/ovmf_suffix.bin", "/dev/null", "/dev/null", "")
+	if err != nil {
+		t.Fatalf("unexpected err value: %s", err)
+	}
+	hexld := hex.EncodeToString(ld[:])
+	if hexld != "7e5c26fb454621eb466978b4d0242b3c04b44a034de7fc0a2d8dac60ea2b6403" {
+		t.Fatalf("wrong measurement: %s", hexld)
+	}
+}
+
+func TestCalculateLaunchDigestWithKernelHashesSevEsAndSmp(t *testing.T) {
+	ld, err := CalculateSEVESLaunchDigest(4, SigEpycV4, "testdata/ovmf_suffix.bin", "/dev/null", "/dev/null", "")
+	if err != nil {
+		t.Fatalf("unexpected err value: %s", err)
+	}
+	hexld := hex.EncodeToString(ld[:])
+	if hexld != "b2111b0051fc3a06ec216899b2c78da99fb9d56c6ff2e8261dd3fe6cff79ecbc" {
+		t.Fatalf("wrong measurement: %s", hexld)
+	}
+}
diff --git a/src/runtime/pkg/sev/testdata/README.md b/src/runtime/pkg/sev/testdata/README.md
new file mode 100644
index 000000000..34554dc8e
--- /dev/null
+++ b/src/runtime/pkg/sev/testdata/README.md
@@ -0,0 +1,9 @@
+# sev/testdata
+
+The `ovmf_suffix.bin` contains the last 4KB of the `OVMF.fd` binary from edk2's
+`OvmfPkg/AmdSev/AmdSevX64.dsc` build.  To save space, we committed only the
+last 4KB instead of the the full 4MB binary.
+
+The end of the file contains a GUIDed footer table with entries that hold the
+SEV-ES AP reset vector address, which is needed in order to compute VMSAs for
+SEV-ES guests.
diff --git a/src/runtime/pkg/sev/testdata/ovmf_suffix.bin b/src/runtime/pkg/sev/testdata/ovmf_suffix.bin
new file mode 100644
index 0000000000000000000000000000000000000000..cc6d7ca7f0873de1305bfef33a7c1814934d8cf4
GIT binary patch
literal 4096
zcmeGee{2(F^j&)`9Tu+64hk!*gAx%lTp<W!E^g9|A-@KWWg?)&tO^pM-Ca_2K#!x!
zQ7nz3!H}3`L?LR_D25g^yE?W`6d^<7aEN7y?sg`k;C^&t=liZ5KQ#X5UuIv@@BRM1
z_ulv3yQ=EQ8p*&&21YV4l7W#7{C63cX{^uQ65ml8@XX%W)w?%i#XFSga0c|50VY6t
z?aY7)E-4Qz07b}By|a0r#dmqn9!}wX)}r0^<y<H4v%N7#T+f~3efB_8-dXO9;Bx|Z
zQf$&!JGdq#BhrXVY*vQEW_hg%z||pM5W76VwfR)Lw@}d+d+!fO>?}bV<;`>U?XDi*
z)8t^ggXLEKB1Q`tU$8$=F;?gO&hN|Ab4F!+K;n#It8&sMi7kO1PR9<(qw!ok<Z(f=
z6|~k?`+a%~n~i%WJ1c18FOhTbvSLBfdkdX|N{1lP9)8}^)m$EPwE6+6i7%`u1>kb@
z3#KbMC={DKd|}mUZmfQRNsy*9M;|lZ)2<jIBrkFu9v8l`uJ(7!p>f~34=dZ3zBI4r
z{YrOL*%?%$@Mk3Y#ps(f7h$VPNDbGJl#YfH9gtFT4ZlT})&<~iv(n$-r;!i+Da4Ti
z|B#1om+Mk$x9n;86DiiPy_`)0FpdOstjoYmNw_-Xw+tlN+`K3nNX#r0n3#N5N0da&
z92`TVBWyMk|GjoV9Aid;iT{CUmIcw{nvO$4n;d3})P1Q>Rc#RHU^5HM(}*sYRbZV)
zG&wV0)Psp8J4+|8DTDG1OlO3+9ZOioz@1o0ZP9#+l9>Ff40Z0Ba@}79Xi0G^Z*`^C
z9mN13GS%+~kv;_uKSTjf5nIb+2{VS`2SIugC4Jjb!Fp;CL1$+)18w5;DH^TFl8<6O
zTy4WQ7=u|hyh^qsH%i(iLZWWLNA6613a|qry9ZQtCPUds$@rOuq8tQu&qN4d8kl1V
zs&-w}MV!a{C7^aii$U}KAUQ|OGU0wxS6@5krY=<r!lehrc0qA$(emPIW8VEfZm^;W
zS)%5d!uX5qgp<mhOR9*3(;ii$gBZ{n%t3@a$G>IsbkOaz2PZ)S6K26JPCJQV-0h^(
zkQ6PUaby4p1HkN*5GkNIqp*br*0@-R<tx7gOKc+T43^kMv<#Lw5txD{4g_X7jQTAG
z;wbf~(C=je8}Z47Uy0CuGh`jJ<5}cmzq_5b)RWIb)GAgwE$F08=%csJk0mAtE1h<s
zmA<7YhLvMDOBnRA-@pW-*HtyvHCeRo^r&d8e1=C@(bST*3!<}Y(9<R8W~I|CoN#yq
z+NnDoR-qnKh#up*(f$|#PR9`8{lDSGB#Dp2e|wzr1}q|>k7Dl<`gF|BeV7PY9#mEN
zYuut>!?uw``sG1&J9PHOwSJpwAvI0|)|3GdpX$(P#Nq}Mq4NoC)@Tc%iwSMjXd9s`
z2yI6!3Rr_$CubQh;mxFtb5MwFjG*(fEaQvGKcWQt5GT=;cG-m-ZH_Yy{J_GR%^X+7
zCpT=aS;Q^l`!};So47KqEtrPh7~1*SxoK^oFSqXANG|fC_yc0T2=%$~d~#EzyP9M8
zeg`gYlxD8zT#}+wUc;}FLl^?~4u278ZD4y9ORCp(q>U$$PFI}CNc)hgGL)tJPpnQ0
zcp@eUwTGloBGeT+gXdgOj>y@VXqS{4K|0RlYO5zF7R%ewbeKD;#Xy*pY}OiPm~GH}
zlFO2*`Ym~+v}y^)fCmD;2rfUCbc%+0#Piem8N~6q$<QTpfZ(a`0*?$zvris}E1{0#
z?L#v?3^m>pnTkUYN4uxH5#6#tq!_0*emDLWkvn`hBv2uYx2Bfn*{i2Pg9nD;2WfgQ
zG(J?JjBYHcPz)aN1ZJdP38eA}ia|b)N69ODaeI?&<^A}^m;kjrx1~I{xjgqoxtLIY
z)|z(sz6z|IaWJRm`JW!%zklJ2>&gO$8bhDIhbb8co|;|f3HQ*tRmgqnD;V?x&;S!5
zXYML*;e2}WD=$oX`?tg2RKgXM;QX5i>pz~k<iOUZC4N4$E_}9aA9;pxaN)UL*f#OY
z<>J_<<J;~#vTPSz*zw8hALaCx9_v2nX<lv0lV_y<Oc1}SD$DRwD;=g-gFGGGmowDx
EUo&WT4FCWD

literal 0
HcmV?d00001

diff --git a/src/runtime/pkg/sev/vcpu_sigs.go b/src/runtime/pkg/sev/vcpu_sigs.go
new file mode 100644
index 000000000..9cee59e2b
--- /dev/null
+++ b/src/runtime/pkg/sev/vcpu_sigs.go
@@ -0,0 +1,76 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package sev
+
+type VCPUSig uint64
+
+const (
+	// 'EPYC': family=23, model=1, stepping=2
+	SigEpyc VCPUSig = 0x800f12
+
+	// 'EPYC-v1': family=23, model=1, stepping=2
+	SigEpycV1 VCPUSig = 0x800f12
+
+	// 'EPYC-v2': family=23, model=1, stepping=2
+	SigEpycV2 VCPUSig = 0x800f12
+
+	// 'EPYC-IBPB': family=23, model=1, stepping=2
+	SigEpycIBPB VCPUSig = 0x800f12
+
+	// 'EPYC-v3': family=23, model=1, stepping=2
+	SigEpycV3 VCPUSig = 0x800f12
+
+	// 'EPYC-v4': family=23, model=1, stepping=2
+	SigEpycV4 VCPUSig = 0x800f12
+
+	// 'EPYC-Rome': family=23, model=49, stepping=0
+	SigEpycRome VCPUSig = 0x830f10
+
+	// 'EPYC-Rome-v1': family=23, model=49, stepping=0
+	SigEpycRomeV1 VCPUSig = 0x830f10
+
+	// 'EPYC-Rome-v2': family=23, model=49, stepping=0
+	SigEpycRomeV2 VCPUSig = 0x830f10
+
+	// 'EPYC-Rome-v3': family=23, model=49, stepping=0
+	SigEpycRomeV3 VCPUSig = 0x830f10
+
+	// 'EPYC-Milan': family=25, model=1, stepping=1
+	SigEpycMilan VCPUSig = 0xa00f11
+
+	// 'EPYC-Milan-v1': family=25, model=1, stepping=1
+	SigEpycMilanV1 VCPUSig = 0xa00f11
+
+	// 'EPYC-Milan-v2': family=25, model=1, stepping=1
+	SigEpycMilanV2 VCPUSig = 0xa00f11
+)
+
+// NewVCPUSig computes the CPU signature (32-bit value) from the given family,
+// model, and stepping.
+//
+// This computation is described in AMD's CPUID Specification, publication #25481
+// https://www.amd.com/system/files/TechDocs/25481.pdf
+// See section: CPUID Fn0000_0001_EAX Family, Model, Stepping Identifiers
+func NewVCPUSig(family, model, stepping uint32) VCPUSig {
+	var family_low, family_high uint32
+	if family > 0xf {
+		family_low = 0xf
+		family_high = (family - 0x0f) & 0xff
+	} else {
+		family_low = family
+		family_high = 0
+	}
+
+	model_low := model & 0xf
+	model_high := (model >> 4) & 0xf
+
+	stepping_low := stepping & 0xf
+
+	return VCPUSig((family_high << 20) |
+		(model_high << 16) |
+		(family_low << 8) |
+		(model_low << 4) |
+		stepping_low)
+}
diff --git a/src/runtime/pkg/sev/vcpu_sigs_test.go b/src/runtime/pkg/sev/vcpu_sigs_test.go
new file mode 100644
index 000000000..70f848750
--- /dev/null
+++ b/src/runtime/pkg/sev/vcpu_sigs_test.go
@@ -0,0 +1,21 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package sev
+
+import (
+	"testing"
+)
+
+func TestNewVCPUSig(t *testing.T) {
+	if NewVCPUSig(23, 1, 2) != SigEpyc {
+		t.Errorf("wrong EPYC CPU signature")
+	}
+	if NewVCPUSig(23, 49, 0) != SigEpycRome {
+		t.Errorf("wrong EPYC-Rome CPU signature")
+	}
+	if NewVCPUSig(25, 1, 1) != SigEpycMilan {
+		t.Errorf("wrong EPYC-Milan CPU signature")
+	}
+}
diff --git a/src/runtime/pkg/sev/vmsa.go b/src/runtime/pkg/sev/vmsa.go
new file mode 100644
index 000000000..c2bbc4122
--- /dev/null
+++ b/src/runtime/pkg/sev/vmsa.go
@@ -0,0 +1,172 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package sev
+
+import (
+	"bytes"
+	"encoding/binary"
+)
+
+// VMCB Segment (struct vmcb_seg in the linux kernel)
+type vmcbSeg struct {
+	selector uint16
+	attrib   uint16
+	limit    uint32
+	base     uint64
+}
+
+// VMSA page
+//
+// The names of the fields are taken from struct sev_es_work_area in the linux kernel:
+// https://github.com/AMDESE/linux/blob/sev-snp-v12/arch/x86/include/asm/svm.h#L318
+// (following the definitions in AMD APM Vol 2 Table B-4)
+type sevEsSaveArea struct {
+	es                  vmcbSeg
+	cs                  vmcbSeg
+	ss                  vmcbSeg
+	ds                  vmcbSeg
+	fs                  vmcbSeg
+	gs                  vmcbSeg
+	gdtr                vmcbSeg
+	ldtr                vmcbSeg
+	idtr                vmcbSeg
+	tr                  vmcbSeg
+	vmpl0_ssp           uint64   // nolint: unused
+	vmpl1_ssp           uint64   // nolint: unused
+	vmpl2_ssp           uint64   // nolint: unused
+	vmpl3_ssp           uint64   // nolint: unused
+	u_cet               uint64   // nolint: unused
+	reserved_1          [2]uint8 // nolint: unused
+	vmpl                uint8    // nolint: unused
+	cpl                 uint8    // nolint: unused
+	reserved_2          [4]uint8 // nolint: unused
+	efer                uint64
+	reserved_3          [104]uint8 // nolint: unused
+	xss                 uint64     // nolint: unused
+	cr4                 uint64
+	cr3                 uint64 // nolint: unused
+	cr0                 uint64
+	dr7                 uint64
+	dr6                 uint64
+	rflags              uint64
+	rip                 uint64
+	dr0                 uint64    // nolint: unused
+	dr1                 uint64    // nolint: unused
+	dr2                 uint64    // nolint: unused
+	dr3                 uint64    // nolint: unused
+	dr0_addr_mask       uint64    // nolint: unused
+	dr1_addr_mask       uint64    // nolint: unused
+	dr2_addr_mask       uint64    // nolint: unused
+	dr3_addr_mask       uint64    // nolint: unused
+	reserved_4          [24]uint8 // nolint: unused
+	rsp                 uint64    // nolint: unused
+	s_cet               uint64    // nolint: unused
+	ssp                 uint64    // nolint: unused
+	isst_addr           uint64    // nolint: unused
+	rax                 uint64    // nolint: unused
+	star                uint64    // nolint: unused
+	lstar               uint64    // nolint: unused
+	cstar               uint64    // nolint: unused
+	sfmask              uint64    // nolint: unused
+	kernel_gs_base      uint64    // nolint: unused
+	sysenter_cs         uint64    // nolint: unused
+	sysenter_esp        uint64    // nolint: unused
+	sysenter_eip        uint64    // nolint: unused
+	cr2                 uint64    // nolint: unused
+	reserved_5          [32]uint8 // nolint: unused
+	g_pat               uint64
+	dbgctrl             uint64    // nolint: unused
+	br_from             uint64    // nolint: unused
+	br_to               uint64    // nolint: unused
+	last_excp_from      uint64    // nolint: unused
+	last_excp_to        uint64    // nolint: unused
+	reserved_7          [80]uint8 // nolint: unused
+	pkru                uint32    // nolint: unused
+	reserved_8          [20]uint8 // nolint: unused
+	reserved_9          uint64    // nolint: unused
+	rcx                 uint64    // nolint: unused
+	rdx                 uint64
+	rbx                 uint64    // nolint: unused
+	reserved_10         uint64    // nolint: unused
+	rbp                 uint64    // nolint: unused
+	rsi                 uint64    // nolint: unused
+	rdi                 uint64    // nolint: unused
+	r8                  uint64    // nolint: unused
+	r9                  uint64    // nolint: unused
+	r10                 uint64    // nolint: unused
+	r11                 uint64    // nolint: unused
+	r12                 uint64    // nolint: unused
+	r13                 uint64    // nolint: unused
+	r14                 uint64    // nolint: unused
+	r15                 uint64    // nolint: unused
+	reserved_11         [16]uint8 // nolint: unused
+	guest_exit_info_1   uint64    // nolint: unused
+	guest_exit_info_2   uint64    // nolint: unused
+	guest_exit_int_info uint64    // nolint: unused
+	guest_nrip          uint64    // nolint: unused
+	sev_features        uint64
+	vintr_ctrl          uint64 // nolint: unused
+	guest_exit_code     uint64 // nolint: unused
+	virtual_tom         uint64 // nolint: unused
+	tlb_id              uint64 // nolint: unused
+	pcpu_id             uint64 // nolint: unused
+	event_inj           uint64 // nolint: unused
+	xcr0                uint64
+	reserved_12         [16]uint8   // nolint: unused
+	x87_dp              uint64      // nolint: unused
+	mxcsr               uint32      // nolint: unused
+	x87_ftw             uint16      // nolint: unused
+	x87_fsw             uint16      // nolint: unused
+	x87_fcw             uint16      // nolint: unused
+	x87_fop             uint16      // nolint: unused
+	x87_ds              uint16      // nolint: unused
+	x87_cs              uint16      // nolint: unused
+	x87_rip             uint64      // nolint: unused
+	fpreg_x87           [80]uint8   // nolint: unused
+	fpreg_xmm           [256]uint8  // nolint: unused
+	fpreg_ymm           [256]uint8  // nolint: unused
+	unused              [2448]uint8 // nolint: unused
+}
+
+type vmsaBuilder struct {
+	apEIP   uint64
+	vcpuSig VCPUSig
+}
+
+func (v *vmsaBuilder) buildPage(i int) ([]byte, error) {
+	eip := uint64(0xfffffff0) // BSP (first vcpu)
+	if i > 0 {
+		eip = v.apEIP
+	}
+	saveArea := sevEsSaveArea{
+		es:           vmcbSeg{0, 0x93, 0xffff, 0},
+		cs:           vmcbSeg{0xf000, 0x9b, 0xffff, eip & 0xffff0000},
+		ss:           vmcbSeg{0, 0x93, 0xffff, 0},
+		ds:           vmcbSeg{0, 0x93, 0xffff, 0},
+		fs:           vmcbSeg{0, 0x93, 0xffff, 0},
+		gs:           vmcbSeg{0, 0x93, 0xffff, 0},
+		gdtr:         vmcbSeg{0, 0, 0xffff, 0},
+		idtr:         vmcbSeg{0, 0, 0xffff, 0},
+		ldtr:         vmcbSeg{0, 0x82, 0xffff, 0},
+		tr:           vmcbSeg{0, 0x8b, 0xffff, 0},
+		efer:         0x1000, // KVM enables EFER_SVME
+		cr4:          0x40,   // KVM enables X86_CR4_MCE
+		cr0:          0x10,
+		dr7:          0x400,
+		dr6:          0xffff0ff0,
+		rflags:       0x2,
+		rip:          eip & 0xffff,
+		g_pat:        0x7040600070406, // PAT MSR: See AMD APM Vol 2, Section A.3
+		rdx:          uint64(v.vcpuSig),
+		sev_features: 0, // SEV-ES
+		xcr0:         0x1,
+	}
+	page := new(bytes.Buffer)
+	err := binary.Write(page, binary.LittleEndian, saveArea)
+	if err != nil {
+		return []byte{}, err
+	}
+	return page.Bytes(), nil
+}

From dd7562522aa796ad72368b9a12c95a95f9d3a65d Mon Sep 17 00:00:00 2001
From: Dov Murik <dovmurik@linux.ibm.com>
Date: Mon, 8 May 2023 10:23:52 +0300
Subject: [PATCH 097/150] runtime: pkg/sev: Add kbs utility package for SEV
 pre-attestation

Supports both online and offline modes of interaction with simple-kbs
for SEV/SEV-ES confidential guests.

Fixes: #6795

Signed-off-by: Dov Murik <dovmurik@linux.ibm.com>
---
 src/runtime/pkg/sev/kbs/kbs.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 src/runtime/pkg/sev/kbs/kbs.go

diff --git a/src/runtime/pkg/sev/kbs/kbs.go b/src/runtime/pkg/sev/kbs/kbs.go
new file mode 100644
index 000000000..3c7e20134
--- /dev/null
+++ b/src/runtime/pkg/sev/kbs/kbs.go
@@ -0,0 +1,33 @@
+// Copyright contributors to AMD SEV/-ES in Go
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Package kbs can be used interact with simple-kbs, the key broker
+// server for SEV and SEV-ES pre-attestation
+
+package kbs
+
+const (
+	Offline           = "offline"
+	OfflineSecretType = "bundle"
+	OfflineSecretGuid = "e6f5a162-d67f-4750-a67c-5d065f2a9910"
+	Online            = "online"
+	OnlineBootParam   = "online_sev_kbc"
+	OnlineSecretType  = "connection"
+	OnlineSecretGuid  = "1ee27366-0c87-43a6-af48-28543eaf7cb0"
+)
+
+type GuestPreAttestationConfig struct {
+	Proxy            string
+	Keyset           string
+	LaunchId         string
+	KernelPath       string
+	InitrdPath       string
+	FwPath           string
+	KernelParameters string
+	CertChainPath    string
+	SecretType       string
+	SecretGuid       string
+	Policy           uint32
+}

From bfdf0144aa997981514773ae409b77f12712269e Mon Sep 17 00:00:00 2001
From: Amulyam24 <amulmek1@in.ibm.com>
Date: Fri, 12 May 2023 11:57:36 +0530
Subject: [PATCH 098/150] versions: Bump virtiofsd to 1.6.1

virtiofsd v1.6.1  has been released with the fixes required for running
successfully on ppc64le.

Fixes: #4259

Signed-off-by: Amulyam24 <amulmek1@in.ibm.com>
---
 versions.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/versions.yaml b/versions.yaml
index 1d339c9dc..f18f03d27 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -297,14 +297,14 @@ externals:
   virtiofsd:
     description: "vhost-user virtio-fs device backend written in Rust"
     url: "https://gitlab.com/virtio-fs/virtiofsd"
-    version: "v1.3.0"
-    toolchain: "1.62.0"
+    version: "v1.6.1"
+    toolchain: "1.66.0"
     meta:
-      # From https://gitlab.com/virtio-fs/virtiofsd/-/releases/v1.3.0,
-      # this is the link labelled virtiofsd-v1.3.0.zip
+      # From https://gitlab.com/virtio-fs/virtiofsd/-/releases/v1.6.1,
+      # this is the link labelled virtiofsd-v1.6.1.zip
       #
       # yamllint disable-line rule:line-length
-      binary: "https://gitlab.com/virtio-fs/virtiofsd/uploads/9a4f2261fcb1701f1e709694b5c5d980/virtiofsd-v1.3.0.zip"
+      binary: "https://gitlab.com/virtio-fs/virtiofsd/uploads/14c1e8a7acc82d515cec6608727a1e4a/virtiofsd-v1.6.1.zip"
 
 languages:
   description: |

From c5a59caca142c332001aabec87b6c7a4b219b243 Mon Sep 17 00:00:00 2001
From: Amulyam24 <amulmek1@in.ibm.com>
Date: Fri, 12 May 2023 11:59:04 +0530
Subject: [PATCH 099/150] ppc64le: switch virtiofsd from C to rust version

We have been using the C version of virtiofsd on ppc64le. Now that the issue with
rust virtiofsd have been fixed, let's switch to it.

Fixes: #4259

Signed-off-by: Amulyam24 <amulmek1@in.ibm.com>
---
 src/runtime/Makefile                            | 3 ---
 tools/packaging/scripts/configure-hypervisor.sh | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index 776da3ef4..aaaca5aeb 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -217,9 +217,6 @@ DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p
 DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p
 DEFSHAREDFS_QEMU_SNP_VIRTIOFS := virtio-9p
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
-ifeq ($(ARCH),ppc64le)
-DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/qemu/virtiofsd
-endif
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 # Default DAX mapping cache size in MiB
 #if value is 0, DAX is not enabled
diff --git a/tools/packaging/scripts/configure-hypervisor.sh b/tools/packaging/scripts/configure-hypervisor.sh
index 751b2866f..8bdfc94de 100755
--- a/tools/packaging/scripts/configure-hypervisor.sh
+++ b/tools/packaging/scripts/configure-hypervisor.sh
@@ -341,7 +341,7 @@ generate_qemu_options() {
 		qemu_options+=(functionality:--disable-virtiofsd)
 		;;
 	ppc64le)
-		qemu_options+=(functionality:--enable-virtiofsd)
+		qemu_options+=(functionality:--disable-virtiofsd)
 		;;
 	s390x)
 		qemu_options+=(functionality:--disable-virtiofsd)

From 87cb98c01d430c7723797ddd340460e530b94e78 Mon Sep 17 00:00:00 2001
From: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
Date: Tue, 16 May 2023 15:26:21 +0000
Subject: [PATCH 100/150] osbuilder: Fix indentation in rootfs.sh

This PR replaces single spaces to tabs in order to fix the
indentation of the rootfs script.

Fixes #6848

Signed-off-by: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
---
 tools/osbuilder/rootfs-builder/rootfs.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh
index dde4a07d6..a494cb440 100755
--- a/tools/osbuilder/rootfs-builder/rootfs.sh
+++ b/tools/osbuilder/rootfs-builder/rootfs.sh
@@ -207,11 +207,11 @@ docker_extra_args()
 		args+=" -v ${gentoo_local_portage_dir}:/usr/portage/packages"
 		args+=" --volumes-from ${gentoo_portage_container}"
 		;;
-        debian | ubuntu | suse)
+	debian | ubuntu | suse)
 		source /etc/os-release
 
 		case "$ID" in
-                fedora | centos | rhel)
+		fedora | centos | rhel)
 			# Depending on the podman version, we'll face issues when passing
 		        # `--security-opt apparmor=unconfined` on a system where not apparmor is not installed.
 			# Because of this, let's just avoid adding this option when the host OS comes from Red Hat.

From 1a3f8fc1a23297230b4a9fe532232a753ae2701a Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Tue, 16 May 2023 19:30:32 +0100
Subject: [PATCH 101/150] deploy: fix shell script error

- Remove local introduced by bad copy-paste

Fixes: #6814
Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../local-build/kata-deploy-build-and-upload-payload.sh         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index 74e0f992b..193f9511e 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -15,7 +15,7 @@ cp ${KATA_DEPLOY_ARTIFACT} ${KATA_DEPLOY_DIR}
 
 pushd ${KATA_DEPLOY_DIR}
 
-local arch=$(uname -m)
+arch=$(uname -m)
 [ "$arch" = "x86_64" ] && arch="amd64"
 IMAGE_TAG="${REGISTRY}:kata-containers-$(git rev-parse HEAD)-${arch}"
 

From 12f43bea0f1b0fe1cd617967bab4f22858b1b05f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 17 May 2023 09:50:29 +0200
Subject: [PATCH 102/150] gha: tdx: Use the k3s overlay for kata-cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As the TDX CI runs on k3s, we must ensure the cleanup, as already done
for the deploy, used the k3s overlay.

Fixes: #6857

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/run-k8s-tests-on-tdx.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml
index 1777a16c8..311694603 100644
--- a/.github/workflows/run-k8s-tests-on-tdx.yaml
+++ b/.github/workflows/run-k8s-tests-on-tdx.yaml
@@ -57,9 +57,9 @@ jobs:
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
           cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
           cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
           sleep 180s
 
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
+          kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
           kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
           kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

From da0f92cef850afb4cb244394bca5889bd7be5d20 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Thu, 11 May 2023 15:40:37 +0000
Subject: [PATCH 103/150] gha: ci-on-push: Don't skip tests for SEV-SNP

Now that we have SNP artifacts in place and they are built via gha,
remove the condition that skips the tests for SNP.

Fixes: #6809

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tests/integration/kubernetes/run_kubernetes_tests.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
index 281d1d878..daddb756a 100644
--- a/tests/integration/kubernetes/run_kubernetes_tests.sh
+++ b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -58,10 +58,6 @@ if [ ${KATA_HYPERVISOR} == "qemu-sev" ]; then
 	exit 0
 fi
 
-if [ ${KATA_HYPERVISOR} == "qemu-snp" ]; then
-	exit 0
-fi
-
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then

From 72308ddb079d45253a994a3a34c2c515fc65a8f5 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Thu, 11 May 2023 18:37:44 +0000
Subject: [PATCH 104/150] gha: ci-on-push: Don't skip tests for SEV

Now that SEV artifacts are built by GHA, remove
conditional that skips tests when using qemu-sev.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tests/integration/kubernetes/run_kubernetes_tests.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
index daddb756a..db1e16633 100644
--- a/tests/integration/kubernetes/run_kubernetes_tests.sh
+++ b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -54,10 +54,6 @@ else
 	)
 fi
 
-if [ ${KATA_HYPERVISOR} == "qemu-sev" ]; then
-	exit 0
-fi
-
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then

From 521dad2a47e8915d279cde15a3f1d2c9bf05dfb8 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Thu, 11 May 2023 17:07:08 +0000
Subject: [PATCH 105/150] Tests: skip CPU constraints test on SEV and SNP

Currently Kata does not support memory / CPU hotplug for SEV or
SEV-SNP so we need to skip tests that rely on it.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tests/integration/kubernetes/k8s-cpu-ns.bats | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/integration/kubernetes/k8s-cpu-ns.bats b/tests/integration/kubernetes/k8s-cpu-ns.bats
index 4d5f2e883..f3c69a2f6 100644
--- a/tests/integration/kubernetes/k8s-cpu-ns.bats
+++ b/tests/integration/kubernetes/k8s-cpu-ns.bats
@@ -11,7 +11,9 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh"
 setup() {
 	[ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}"
 	[ "${KATA_HYPERVISOR}" == "dragonball" ] && skip "test not working see: ${dragonball_limitations}"
-	[ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "TEEs do not support memory / CPU hotplug"
+	( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || [ "${KATA_HYPERVISOR}" == "qemu-sev" ] ) \
+		&& skip "TEEs do not support memory / CPU hotplug"
+
 
 	pod_name="constraints-cpu-test"
 	container_name="first-cpu-container"
@@ -28,7 +30,9 @@ setup() {
 @test "Check CPU constraints" {
 	[ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}"
 	[ "${KATA_HYPERVISOR}" == "dragonball" ] && skip "test not working see: ${dragonball_limitations}"
-	[ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "TEEs do not support memory / CPU hotplug"
+	( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || [ "${KATA_HYPERVISOR}" == "qemu-sev" ] ) \
+		&& skip "TEEs do not support memory / CPU hotplug"
+
 
 	# Create the pod
 	kubectl create -f "${pod_config_dir}/pod-cpu.yaml"
@@ -73,7 +77,8 @@ setup() {
 teardown() {
 	[ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}"
 	[ "${KATA_HYPERVISOR}" == "dragonball" ] && skip "test not working see: ${dragonball_limitations}"
-	[ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "TEEs do not support memory / CPU hotplug"
+	( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || [ "${KATA_HYPERVISOR}" == "qemu-sev" ] ) \
+		&& skip "TEEs do not support memory / CPU hotplug"
 
 	# Debugging information
 	kubectl describe "pod/$pod_name"

From 724437efb3d9409dc492bd3913bc246417112312 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Thu, 11 May 2023 20:01:48 +0000
Subject: [PATCH 106/150] kata-deploy: add kata-qemu-sev runtimeclass

In order to populate containerd config file with
support for SEV, we need to add the qemu-sev shim
to the kata-deploy script.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 tools/packaging/kata-deploy/scripts/kata-deploy.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 68fd59b30..820ac9d5d 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -18,6 +18,7 @@ shims=(
 	"qemu"
 	"qemu-nvidia-gpu"
 	"qemu-tdx"
+	"qemu-sev"
 	"qemu-snp"
 	"clh"
 	"dragonball"

From cbb9fe8b812429910832b7874624fc483a9f4329 Mon Sep 17 00:00:00 2001
From: Tobin Feldman-Fitzthum <tobin@ibm.com>
Date: Tue, 16 May 2023 02:27:20 +0000
Subject: [PATCH 107/150] config: Use standard OVMF with SEV

The AmdSev firmware package should be used with
measured direct boot. If the expected hashes are not
injected into the firmware binary by the VMM, the
guest will not boot. This is required for security.

Currently the main branch does not have the extended
shim support for SEV, which tells the VMM to inject
the expected hashes.

We ship the standard OVMF package to use with SNP,
so let's switch SEV to that for now. This will need
to be changed back when shim support for SEV(-ES)
is added to main.

Signed-off-by: Tobin Feldman-Fitzthum <tobin@ibm.com>
---
 src/runtime/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index 7628dc43d..7e264dfea 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -130,7 +130,7 @@ FIRMWAREVOLUMEPATH :=
 FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd
 FIRMWARETDVFVOLUMEPATH :=
 
-FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd
+FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
 FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd
 
 # Name of default configuration file the runtime will use.

From fa832f4709f6213946c11601ccd895ab01ee8bab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 17 May 2023 13:38:08 +0200
Subject: [PATCH 108/150] gha: k8s: Make the tests more reliable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We like it or not, every now and then we'll have to deal with flaky
tests, and our tests using GHA are not exempt from that fact.

With this simple commit, we're trying to improve the reliability of the
tests in a few different fronts:

* Giving enough time for the script used by kata-deploy to be executed
  * We've hit issues as the kata-deploy pod is considered "Ready" at the
    moment it starts running, not when it finishes the needed setup. We
    should also be looking on how to solve this on the kata-deploy side
    but, for now, let's ensure our tests do not break with the current
    kata-deploy behavior.

* Merging the "Deploy kata-deploy" and "Run tests" steps
  * We've hit issues re-running tests and seeing even more failures than
    the ones we're trying to debug, as a step will simply be taken as
    succeeded as part of the re-run, in case it was successful executed
    as part of the first run.  This causes issues with the kata-deploy
    deployment, as the tests would start running before even having the
    node set up for running Kata Containers.

Fixes: #6865 #6649

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/run-k8s-tests-on-aks.yaml | 11 +++++++----
 .github/workflows/run-k8s-tests-on-sev.yaml | 11 +++++++----
 .github/workflows/run-k8s-tests-on-snp.yaml | 11 +++++++----
 .github/workflows/run-k8s-tests-on-tdx.yaml | 11 +++++++----
 4 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml
index f9a26debb..b9886e47e 100644
--- a/.github/workflows/run-k8s-tests-on-aks.yaml
+++ b/.github/workflows/run-k8s-tests-on-aks.yaml
@@ -61,7 +61,8 @@ jobs:
         run: |
           az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64
 
-      - name: Deploy kata-deploy
+      - name: Run tests
+        timeout-minutes: 30
         run: |
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -72,9 +73,11 @@ jobs:
           kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
           kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
           pushd tests/integration/kubernetes
           sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
           bash run_kubernetes_tests.sh
diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml
index aeafa242e..98a6db610 100644
--- a/.github/workflows/run-k8s-tests-on-sev.yaml
+++ b/.github/workflows/run-k8s-tests-on-sev.yaml
@@ -27,7 +27,8 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Deploy kata-deploy
+      - name: Run tests
+        timeout-minutes: 30
         run: |
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -38,9 +39,11 @@ jobs:
           kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
           kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
           pushd tests/integration/kubernetes
           sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
           bash run_kubernetes_tests.sh
diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml
index 5bc4aea50..541695e0f 100644
--- a/.github/workflows/run-k8s-tests-on-snp.yaml
+++ b/.github/workflows/run-k8s-tests-on-snp.yaml
@@ -27,7 +27,8 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Deploy kata-deploy
+      - name: Run tests
+        timeout-minutes: 30
         run: |
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -38,9 +39,11 @@ jobs:
           kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
           kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
           pushd tests/integration/kubernetes
           sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
           bash run_kubernetes_tests.sh
diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml
index 1777a16c8..0b74ecc7b 100644
--- a/.github/workflows/run-k8s-tests-on-tdx.yaml
+++ b/.github/workflows/run-k8s-tests-on-tdx.yaml
@@ -27,7 +27,8 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Deploy kata-deploy
+      - name: Run tests
+        timeout-minutes: 30
         run: |
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -38,9 +39,11 @@ jobs:
           kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
           kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 
-      - name: Run tests
-        timeout-minutes: 30
-        run: |
+          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+          # which may cause issues like not having the node properly labeled or the artefacts
+          # properly deployed when the tests actually start running.
+          sleep 60s
+
           pushd tests/integration/kubernetes
           sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
           bash run_kubernetes_tests.sh

From ca1531fe9dd62f889f88890de71cada1059b9280 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 17 May 2023 19:21:52 +0200
Subject: [PATCH 109/150] runtime: Use static_sandbox_resource_mgmt=true for
 TEEs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When this option is enabled the runtime will attempt to determine the
appropriate sandbox size (memory, CPU) before booting the virtual
machine.

As TEEs do not support memory and CPU hotplug, this approach must be
used.

Fixes: #6818

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 src/runtime/Makefile                              | 2 ++
 src/runtime/config/configuration-qemu-sev.toml.in | 2 +-
 src/runtime/config/configuration-qemu-snp.toml.in | 2 +-
 src/runtime/config/configuration-qemu-tdx.toml.in | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/runtime/Makefile b/src/runtime/Makefile
index 7e264dfea..e937b741e 100644
--- a/src/runtime/Makefile
+++ b/src/runtime/Makefile
@@ -242,6 +242,7 @@ DEFVFIOMODE := guest-kernel
 DEFSANDBOXCGROUPONLY ?= false
 
 DEFSTATICRESOURCEMGMT ?= false
+DEFSTATICRESOURCEMGMT_TEE = true
 
 DEFBINDMOUNTS := []
 
@@ -599,6 +600,7 @@ USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
 USER_VARS += DEFSTATICRESOURCEMGMT
 USER_VARS += DEFSTATICRESOURCEMGMT_FC
+USER_VARS += DEFSTATICRESOURCEMGMT_TEE
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFSERVICEOFFLOAD
 USER_VARS += DEFVFIOMODE
diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in
index 8dba6813e..a108b726c 100644
--- a/src/runtime/config/configuration-qemu-sev.toml.in
+++ b/src/runtime/config/configuration-qemu-sev.toml.in
@@ -578,7 +578,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
 #   does not yet support sandbox sizing annotations.
 # - When running single containers using a tool like ctr, container sizing information will be available.
-static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
 
 # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
 # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in
index e09f14991..6a608a133 100644
--- a/src/runtime/config/configuration-qemu-snp.toml.in
+++ b/src/runtime/config/configuration-qemu-snp.toml.in
@@ -618,7 +618,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
 #   does not yet support sandbox sizing annotations.
 # - When running single containers using a tool like ctr, container sizing information will be available.
-static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
 
 # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
 # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in
index b9c130e65..52f415f2f 100644
--- a/src/runtime/config/configuration-qemu-tdx.toml.in
+++ b/src/runtime/config/configuration-qemu-tdx.toml.in
@@ -614,7 +614,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O
 #   does not yet support sandbox sizing annotations.
 # - When running single containers using a tool like ctr, container sizing information will be available.
-static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@
+static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@
 
 # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
 # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.

From ee6deef09dd728a228bd9b8f2f579688d998e255 Mon Sep 17 00:00:00 2001
From: Xuewei Niu <niuxuewei.nxw@antgroup.com>
Date: Thu, 18 May 2023 10:24:47 +0800
Subject: [PATCH 110/150] dragonball: Remove virtio-net and vsock devices
 gracefully

This MR implements removing virtio-net and virtio-vsock devices gracefully when
shutting down VMM.

Fixes: #6684

Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Xuewei Niu <niuxuewei.nxw@antgroup.com>
---
 src/dragonball/src/config_manager.rs            |  5 +++++
 src/dragonball/src/device_manager/mod.rs        |  8 ++++++++
 .../src/device_manager/virtio_net_dev_mgr.rs    | 15 +++++++++++++++
 .../src/device_manager/vsock_dev_mgr.rs         | 17 ++++++++++++++++-
 4 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs
index e1c89d2fa..ff74fb925 100644
--- a/src/dragonball/src/config_manager.rs
+++ b/src/dragonball/src/config_manager.rs
@@ -278,6 +278,11 @@ where
         self.info_list.iter_mut()
     }
 
+    /// Remove the last device config info from the `info_list`.
+    pub fn pop(&mut self) -> Option<DeviceConfigInfo<T>> {
+        self.info_list.pop()
+    }
+
     fn get_index_by_id(&self, config: &T) -> Option<usize> {
         self.info_list
             .iter()
diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs
index ed651f577..766c5eef9 100644
--- a/src/dragonball/src/device_manager/mod.rs
+++ b/src/dragonball/src/device_manager/mod.rs
@@ -714,6 +714,14 @@ impl DeviceManager {
 
         #[cfg(feature = "virtio-blk")]
         self.block_manager.remove_devices(&mut ctx)?;
+        // FIXME: To acquire the full abilities for gracefully removing
+        // virtio-net and virtio-vsock devices, updating dragonball-sandbox
+        // is required.
+        #[cfg(feature = "virtio-net")]
+        self.virtio_net_manager.remove_devices(&mut ctx)?;
+        #[cfg(feature = "virtio-vsock")]
+        self.vsock_manager.remove_devices(&mut ctx)?;
+
         Ok(())
     }
 }
diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs
index c0b0f62da..4cba11a4a 100644
--- a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs
+++ b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs
@@ -374,6 +374,21 @@ impl VirtioNetDeviceMgr {
 
         Ok(Box::new(net_device))
     }
+
+    /// Remove all virtio-net devices.
+    pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
+        while let Some(mut info) = self.info_list.pop() {
+            slog::info!(
+                ctx.logger(),
+                "remove virtio-net device: {}",
+                info.config.iface_id
+            );
+            if let Some(device) = info.device.take() {
+                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+            }
+        }
+        Ok(())
+    }
 }
 
 impl Default for VirtioNetDeviceMgr {
diff --git a/src/dragonball/src/device_manager/vsock_dev_mgr.rs b/src/dragonball/src/device_manager/vsock_dev_mgr.rs
index 8588471b7..791d9ded6 100644
--- a/src/dragonball/src/device_manager/vsock_dev_mgr.rs
+++ b/src/dragonball/src/device_manager/vsock_dev_mgr.rs
@@ -17,7 +17,7 @@ use dbs_virtio_devices::vsock::Vsock;
 use dbs_virtio_devices::Error as VirtioError;
 use serde_derive::{Deserialize, Serialize};
 
-use super::StartMicroVmError;
+use super::{DeviceMgrError, StartMicroVmError};
 use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos};
 use crate::device_manager::{DeviceManager, DeviceOpContext};
 
@@ -284,6 +284,21 @@ impl VsockDeviceMgr {
         // safe to unwrap, because we created the inner connector before
         Ok(self.default_inner_connector.clone().unwrap())
     }
+
+    /// Remove all virtio-vsock devices
+    pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> {
+        while let Some(mut info) = self.info_list.pop() {
+            slog::info!(
+                ctx.logger(),
+                "remove virtio-vsock device: {}",
+                info.config.id
+            );
+            if let Some(device) = info.device.take() {
+                DeviceManager::destroy_mmio_virtio_device(device, ctx)?;
+            }
+        }
+        Ok(())
+    }
 }
 
 impl Default for VsockDeviceMgr {

From ca6892ddb1b1679f094fbb794cd978ddf9720250 Mon Sep 17 00:00:00 2001
From: Peng Tao <bergwolf@hyper.sh>
Date: Thu, 18 May 2023 08:18:22 +0000
Subject: [PATCH 111/150] runtime-rs: update tokio dependency

Unify it to the latest 1.28.1 version.

Signed-off-by: Peng Tao <bergwolf@hyper.sh>
---
 src/runtime-rs/Cargo.lock                     | 166 +++++++++++++-----
 src/runtime-rs/crates/agent/Cargo.toml        |   2 +-
 src/runtime-rs/crates/hypervisor/Cargo.toml   |   2 +-
 .../crates/hypervisor/ch-config/Cargo.toml    |   2 +-
 src/runtime-rs/crates/resource/Cargo.toml     |   2 +-
 src/runtime-rs/crates/runtimes/Cargo.toml     |   2 +-
 .../crates/runtimes/common/Cargo.toml         |   2 +-
 .../runtimes/linux_container/Cargo.toml       |   4 +-
 .../crates/runtimes/virt_container/Cargo.toml |   2 +-
 .../crates/runtimes/wasm_container/Cargo.toml |   4 +-
 src/runtime-rs/crates/service/Cargo.toml      |   2 +-
 src/runtime-rs/crates/shim-ctl/Cargo.toml     |   2 +-
 src/runtime-rs/crates/shim/Cargo.toml         |   2 +-
 13 files changed, 135 insertions(+), 59 deletions(-)

diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock
index a68b485d1..35a80d221 100644
--- a/src/runtime-rs/Cargo.lock
+++ b/src/runtime-rs/Cargo.lock
@@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -221,7 +221,7 @@ checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -276,7 +276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -584,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -611,7 +611,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "scratch",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -628,7 +628,7 @@ checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -793,7 +793,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -1073,7 +1073,7 @@ checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2087,7 +2087,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2130,9 +2130,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8"
 dependencies = [
  "unicode-ident",
 ]
@@ -2175,7 +2175,7 @@ dependencies = [
  "itertools",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2267,9 +2267,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
 dependencies = [
  "proc-macro2",
 ]
@@ -2602,7 +2602,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2635,7 +2635,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2810,9 +2810,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "socket2"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
 dependencies = [
  "libc",
  "winapi",
@@ -2849,7 +2849,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2879,6 +2879,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -2949,7 +2960,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -3035,14 +3046,13 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.26.0"
+version = "1.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64"
+checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
 dependencies = [
  "autocfg",
  "bytes 1.4.0",
  "libc",
- "memchr",
  "mio",
  "num_cpus",
  "parking_lot 0.12.1",
@@ -3050,18 +3060,18 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.45.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.2"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.16",
 ]
 
 [[package]]
@@ -3135,7 +3145,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -3454,7 +3464,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
  "wasm-bindgen-shared",
 ]
 
@@ -3488,7 +3498,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -3577,13 +3587,13 @@ version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.1",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm 0.42.1",
+ "windows_x86_64_msvc 0.42.1",
 ]
 
 [[package]]
@@ -3592,7 +3602,16 @@ version = "0.45.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.42.1",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
 ]
 
 [[package]]
@@ -3601,13 +3620,28 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.1",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm 0.42.1",
+ "windows_x86_64_msvc 0.42.1",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.0",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm 0.48.0",
+ "windows_x86_64_msvc 0.48.0",
 ]
 
 [[package]]
@@ -3616,42 +3650,84 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
 [[package]]
 name = "xattr"
 version = "0.2.3"
diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml
index 4475c6d47..7639cb4f7 100644
--- a/src/runtime-rs/crates/agent/Cargo.toml
+++ b/src/runtime-rs/crates/agent/Cargo.toml
@@ -18,7 +18,7 @@ serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 ttrpc = { version = "0.7.1" }
-tokio = { version = "1.8.0", features = ["fs", "rt"] }
+tokio = { version = "1.28.1", features = ["fs", "rt"] }
 url = "2.2.2"
 nix = "0.24.2"
 
diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml
index c06b6c404..3e3a95d0b 100644
--- a/src/runtime-rs/crates/hypervisor/Cargo.toml
+++ b/src/runtime-rs/crates/hypervisor/Cargo.toml
@@ -21,7 +21,7 @@ serde_json = ">=1.0.9"
 slog = "2.5.2"
 slog-scope = "4.4.0"
 thiserror = "1.0"
-tokio = { version = "1.8.0", features = ["sync", "fs"] }
+tokio = { version = "1.28.1", features = ["sync", "fs"] }
 vmm-sys-util = "0.11.0"
 rand = "0.8.4"
 
diff --git a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml
index 10ed105e3..0e0b45e59 100644
--- a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml
+++ b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml
@@ -13,7 +13,7 @@ edition = "2021"
 anyhow = "1.0.68"
 serde = { version = "1.0.145", features = ["rc", "derive"] }
 serde_json = "1.0.91"
-tokio = { version = "1.25.0", features = ["sync", "rt"] }
+tokio = { version = "1.28.1", features = ["sync", "rt"] }
 
 # Cloud Hypervisor public HTTP API functions
 # Note that the version specified is not necessarily the version of CH
diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml
index baafd28b7..9847ce61b 100644
--- a/src/runtime-rs/crates/resource/Cargo.toml
+++ b/src/runtime-rs/crates/resource/Cargo.toml
@@ -30,7 +30,7 @@ serde = { version = "1.0.138", features = ["derive"] }
 serde_json = "1.0.82"
 slog = "2.5.2"
 slog-scope = "4.4.0"
-tokio = { version = "1.8.0", features = ["process"] }
+tokio = { version = "1.28.1", features = ["process"] }
 uuid = { version = "0.4", features = ["v4"] }
 
 agent = { path = "../agent" }
diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml
index 768122684..f2a4ea522 100644
--- a/src/runtime-rs/crates/runtimes/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/Cargo.toml
@@ -11,7 +11,7 @@ lazy_static = "1.4.0"
 netns-rs = "0.1.0"
 slog = "2.5.2"
 slog-scope = "4.4.0"
-tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
+tokio = { version = "1.28.1", features = ["rt-multi-thread"] }
 hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
 hyperlocal = "0.8"
 serde_json = "1.0.88"
diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml
index 440db1486..a60e1f5f1 100644
--- a/src/runtime-rs/crates/runtimes/common/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml
@@ -19,7 +19,7 @@ slog = "2.5.2"
 slog-scope = "4.4.0"
 strum = { version = "0.24.0", features = ["derive"] }
 thiserror = "^1.0"
-tokio = { version = "1.8.0", features = ["rt-multi-thread", "process", "fs"] }
+tokio = { version = "1.28.1", features = ["rt-multi-thread", "process", "fs"] }
 ttrpc = { version = "0.7.1" }
 persist = {path = "../../persist"}
 agent = { path = "../../agent" }
diff --git a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml
index 58e6f6012..de3c03ffd 100644
--- a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 anyhow = "^1.0"
 async-trait = "0.1.48"
-tokio = { version = "1.8.0" }
+tokio = { version = "1.28.1" }
 
 common = { path = "../common" }
-kata-types = { path = "../../../../libs/kata-types" }
\ No newline at end of file
+kata-types = { path = "../../../../libs/kata-types" }
diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml
index f3d8d9375..d648a2206 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml
@@ -20,7 +20,7 @@ serde_derive = "1.0.27"
 serde_json = "1.0.82"
 slog = "2.5.2"
 slog-scope = "4.4.0"
-tokio = { version = "1.8.0" }
+tokio = { version = "1.28.1" }
 toml = "0.4.2"
 url = "2.1.1"
 async-std = "1.12.0"
diff --git a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml
index b8174ee82..4f098295a 100644
--- a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml
+++ b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 anyhow = "^1.0"
 async-trait = "0.1.48"
-tokio = { version = "1.8.0" }
+tokio = { version = "1.28.1" }
 
 common = { path = "../common" }
-kata-types = { path = "../../../../libs/kata-types" }
\ No newline at end of file
+kata-types = { path = "../../../../libs/kata-types" }
diff --git a/src/runtime-rs/crates/service/Cargo.toml b/src/runtime-rs/crates/service/Cargo.toml
index cb414abe3..8449328c0 100644
--- a/src/runtime-rs/crates/service/Cargo.toml
+++ b/src/runtime-rs/crates/service/Cargo.toml
@@ -10,7 +10,7 @@ anyhow = "^1.0"
 async-trait = "0.1.48"
 slog = "2.5.2"
 slog-scope = "4.4.0"
-tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
+tokio = { version = "1.28.1", features = ["rt-multi-thread"] }
 ttrpc = { version = "0.7.1" }
 
 common = { path = "../runtimes/common" }
diff --git a/src/runtime-rs/crates/shim-ctl/Cargo.toml b/src/runtime-rs/crates/shim-ctl/Cargo.toml
index b08e15daa..b1e844b0c 100644
--- a/src/runtime-rs/crates/shim-ctl/Cargo.toml
+++ b/src/runtime-rs/crates/shim-ctl/Cargo.toml
@@ -10,5 +10,5 @@ anyhow = "^1.0"
 common = { path = "../runtimes/common" }
 logging = { path = "../../../libs/logging"}
 runtimes = { path = "../runtimes" }
-tokio = { version = "1.8.0", features = [ "rt", "rt-multi-thread" ] }
+tokio = { version = "1.28.1", features = [ "rt", "rt-multi-thread" ] }
 
diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml
index 84521eb00..1f5dafb3f 100644
--- a/src/runtime-rs/crates/shim/Cargo.toml
+++ b/src/runtime-rs/crates/shim/Cargo.toml
@@ -27,7 +27,7 @@ slog-async = "2.5.2"
 slog-scope = "4.4.0"
 slog-stdlog = "4.1.0"
 thiserror = "1.0.30"
-tokio = { version = "1.8.0", features = [ "rt", "rt-multi-thread" ] }
+tokio = { version = "1.28.1", features = [ "rt", "rt-multi-thread" ] }
 unix_socket2 = "0.5.4"
 
 kata-types = { path = "../../../libs/kata-types"}

From df615ff25201f9a3c8fb3f1b9d32d90ea0e81907 Mon Sep 17 00:00:00 2001
From: Peng Tao <bergwolf@hyper.sh>
Date: Thu, 18 May 2023 08:24:41 +0000
Subject: [PATCH 112/150] runk: update tokio dependency

Update to 1.28.1 to pick up latest fixes.

Signed-off-by: Peng Tao <bergwolf@hyper.sh>
---
 src/tools/runk/Cargo.lock | 248 +++++++++++++++++++++++++++++---------
 src/tools/runk/Cargo.toml |   2 +-
 2 files changed, 189 insertions(+), 61 deletions(-)

diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock
index 5b66a95fa..b280b2a27 100644
--- a/src/tools/runk/Cargo.lock
+++ b/src/tools/runk/Cargo.lock
@@ -102,7 +102,7 @@ checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -119,7 +119,7 @@ checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -164,7 +164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -285,7 +285,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -362,7 +362,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -373,7 +373,7 @@ checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
 dependencies = [
  "darling_core",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -384,7 +384,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -395,7 +395,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -416,7 +416,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -426,7 +426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73"
 dependencies = [
  "derive_builder_core",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -473,7 +473,7 @@ checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -616,7 +616,7 @@ checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -833,9 +833,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.127"
+version = "0.2.144"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
+checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
 
 [[package]]
 name = "libcontainer"
@@ -949,25 +949,14 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.2"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
+checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
 dependencies = [
  "libc",
  "log",
- "miow",
- "ntapi",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "winapi",
-]
-
-[[package]]
-name = "miow"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
-dependencies = [
- "winapi",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1013,15 +1002,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "ntapi"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -1208,7 +1188,7 @@ dependencies = [
  "proc-macro-error-attr",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
  "version_check",
 ]
 
@@ -1225,11 +1205,11 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.37"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
+checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
@@ -1285,7 +1265,7 @@ dependencies = [
  "itertools",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1376,9 +1356,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.18"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
 dependencies = [
  "proc-macro2",
 ]
@@ -1621,7 +1601,7 @@ checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1643,7 +1623,7 @@ checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1731,9 +1711,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
 
 [[package]]
 name = "socket2"
-version = "0.4.4"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
 dependencies = [
  "libc",
  "winapi",
@@ -1772,6 +1752,17 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "tabwriter"
 version = "1.2.1"
@@ -1840,7 +1831,7 @@ checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1876,33 +1867,32 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.17.0"
+version = "1.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
+checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
 dependencies = [
+ "autocfg",
  "bytes",
  "libc",
- "memchr",
  "mio",
  "num_cpus",
- "once_cell",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.7.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.16",
 ]
 
 [[package]]
@@ -1934,7 +1924,7 @@ checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -1998,6 +1988,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "unicode-ident"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.9.0"
@@ -2133,6 +2129,66 @@ dependencies = [
  "windows_x86_64_msvc 0.36.1",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.0",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm 0.48.0",
+ "windows_x86_64_msvc 0.48.0",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.34.0"
@@ -2145,6 +2201,18 @@ version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.34.0"
@@ -2157,6 +2225,18 @@ version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.34.0"
@@ -2169,6 +2249,18 @@ version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.34.0"
@@ -2181,6 +2273,30 @@ version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.34.0"
@@ -2193,6 +2309,18 @@ version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
 [[package]]
 name = "xattr"
 version = "0.2.3"
@@ -2252,7 +2380,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn",
+ "syn 1.0.91",
 ]
 
 [[package]]
@@ -2289,5 +2417,5 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.91",
 ]
diff --git a/src/tools/runk/Cargo.toml b/src/tools/runk/Cargo.toml
index 19632883d..2379ddb94 100644
--- a/src/tools/runk/Cargo.toml
+++ b/src/tools/runk/Cargo.toml
@@ -19,7 +19,7 @@ anyhow = "1.0.52"
 slog = "2.7.0"
 chrono = { version = "0.4.19", features = ["serde"] }
 slog-async = "2.7.0"
-tokio = { version = "1.15.0", features = ["full"] }
+tokio = { version = "1.28.1", features = ["full"] }
 serde = { version = "1.0.133", features = ["derive"] }
 serde_json = "1.0.74"
 users = "0.11.0"

From 4cb83dc21943a16a119bef7f76c9ec7c4277dfd8 Mon Sep 17 00:00:00 2001
From: Peng Tao <bergwolf@hyper.sh>
Date: Thu, 18 May 2023 08:25:13 +0000
Subject: [PATCH 113/150] kata-ctl: update tokio dependency

Update to 1.28.1 To pick up the latest fixes.

Signed-off-by: Peng Tao <bergwolf@hyper.sh>
---
 src/tools/kata-ctl/Cargo.lock | 17 ++++++++---------
 src/tools/kata-ctl/Cargo.toml |  2 +-
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock
index 333470cc5..ed90e1117 100644
--- a/src/tools/kata-ctl/Cargo.lock
+++ b/src/tools/kata-ctl/Cargo.lock
@@ -1588,9 +1588,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "socket2"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
 dependencies = [
  "libc",
  "winapi",
@@ -1766,31 +1766,30 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.25.0"
+version = "1.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
+checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
 dependencies = [
  "autocfg",
  "bytes 1.4.0",
  "libc",
- "memchr",
  "mio",
  "num_cpus",
  "pin-project-lite",
  "socket2",
  "tokio-macros",
- "windows-sys 0.42.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.2"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.107",
+ "syn 2.0.13",
 ]
 
 [[package]]
diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml
index 642c9a8a8..141a90a0c 100644
--- a/src/tools/kata-ctl/Cargo.toml
+++ b/src/tools/kata-ctl/Cargo.toml
@@ -39,7 +39,7 @@ libc = "0.2.138"
 slog = "2.7.0"
 slog-scope = "4.4.0"
 hyper = "0.14.20"
-tokio = "1.8.0"
+tokio = "1.28.1"
 
 [target.'cfg(target_arch = "s390x")'.dependencies]
 reqwest = { version = "0.11", default-features = false, features = ["json", "blocking", "native-tls"] }

From 4659facb745e29cf3de64f26dd88f5913295d345 Mon Sep 17 00:00:00 2001
From: Shuaiyi Zhang <zhang_syi@qq.com>
Date: Tue, 16 May 2023 20:56:56 +0800
Subject: [PATCH 114/150] dragonball: Convert BlockDeviceMgr function to method

Convert BlockDeviceMgr::insert_device, BlockDeviceMgr::remove_device
and BlockDeviceMgr::update_device_ratelimiters to method.

Fixes: #6880

Signed-off-by: Shuaiyi Zhang <zhang_syi@qq.com>
---
 src/dragonball/src/api/v1/vmm_action.rs       |  12 +-
 .../src/device_manager/blk_dev_mgr.rs         | 155 +++++++++++-------
 2 files changed, 102 insertions(+), 65 deletions(-)

diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs
index 56affcacf..cdaec08a8 100644
--- a/src/dragonball/src/api/v1/vmm_action.rs
+++ b/src/dragonball/src/api/v1/vmm_action.rs
@@ -486,7 +486,9 @@ impl VmmService {
                 VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
             })?;
 
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -500,7 +502,9 @@ impl VmmService {
     ) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
 
-        BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
+        vm.device_manager_mut()
+            .block_manager
+            .update_device_ratelimiters(config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
@@ -518,7 +522,9 @@ impl VmmService {
             .create_device_op_context(Some(event_mgr.epoll_manager()))
             .map_err(|_| VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot))?;
 
-        BlockDeviceMgr::remove_device(vm.device_manager_mut(), ctx, drive_id)
+        vm.device_manager_mut()
+            .block_manager
+            .remove_device(ctx, drive_id)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::Block)
     }
diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs
index 66890d7f9..f7cdfa9b0 100644
--- a/src/dragonball/src/device_manager/blk_dev_mgr.rs
+++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs
@@ -340,7 +340,7 @@ impl BlockDeviceMgr {
     /// the existing entry.
     /// Inserting a secondary root block device will fail.
     pub fn insert_device(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         mut ctx: DeviceOpContext,
         config: BlockDeviceConfigInfo,
     ) -> std::result::Result<(), BlockDeviceError> {
@@ -348,10 +348,8 @@ impl BlockDeviceMgr {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        let mgr = &mut device_mgr.block_manager;
-
         // If the id of the drive already exists in the list, the operation is update.
-        match mgr.get_index_of_drive_id(config.id()) {
+        match self.get_index_of_drive_id(config.id()) {
             Some(index) => {
                 // No support for runtime update yet.
                 if ctx.is_hotplug {
@@ -359,19 +357,19 @@ impl BlockDeviceMgr {
                         config.path_on_host.clone(),
                     ))
                 } else {
-                    for (idx, info) in mgr.info_list.iter().enumerate() {
+                    for (idx, info) in self.info_list.iter().enumerate() {
                         if idx != index {
                             info.config.check_conflicts(&config)?;
                         }
                     }
-                    mgr.update(index, config)
+                    self.update(index, config)
                 }
             }
             None => {
-                for info in mgr.info_list.iter() {
+                for info in self.info_list.iter() {
                     info.config.check_conflicts(&config)?;
                 }
-                let index = mgr.create(config.clone())?;
+                let index = self.create(config.clone())?;
                 if !ctx.is_hotplug {
                     return Ok(());
                 }
@@ -383,17 +381,16 @@ impl BlockDeviceMgr {
                         let dev = DeviceManager::create_mmio_virtio_device(
                             device,
                             &mut ctx,
-                            config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
+                            config.use_shared_irq.unwrap_or(self.use_shared_irq),
                             config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                         )
                         .map_err(BlockDeviceError::DeviceManager)?;
-                        mgr.update_device_by_index(index, Arc::clone(&dev))?;
+                        self.update_device_by_index(index, Arc::clone(&dev))?;
                         // live-upgrade need save/restore device from info.device.
-                        mgr.info_list[index].set_device(dev.clone());
+                        self.info_list[index].set_device(dev.clone());
                         ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| {
                             let logger = ctx.logger().new(slog::o!());
-                            BlockDeviceMgr::remove_device(device_mgr, ctx, &config.drive_id)
-                                .unwrap();
+                            self.remove_device(ctx, &config.drive_id).unwrap();
                             error!(
                                 logger,
                                 "failed to hot-add virtio block device {}, {:?}",
@@ -466,7 +463,7 @@ impl BlockDeviceMgr {
 
     /// remove a block device, it basically is the inverse operation of `insert_device``
     pub fn remove_device(
-        dev_mgr: &mut DeviceManager,
+        &mut self,
         mut ctx: DeviceOpContext,
         drive_id: &str,
     ) -> std::result::Result<(), BlockDeviceError> {
@@ -474,8 +471,7 @@ impl BlockDeviceMgr {
             return Err(BlockDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        let mgr = &mut dev_mgr.block_manager;
-        match mgr.remove(drive_id) {
+        match self.remove(drive_id) {
             Some(mut info) => {
                 info!(ctx.logger(), "remove drive {}", info.config.drive_id);
                 if let Some(device) = info.device.take() {
@@ -731,15 +727,14 @@ impl BlockDeviceMgr {
 
     /// Update the ratelimiter settings of a virtio blk device.
     pub fn update_device_ratelimiters(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         new_cfg: BlockDeviceConfigUpdateInfo,
     ) -> std::result::Result<(), BlockDeviceError> {
-        let mgr = &mut device_mgr.block_manager;
-        match mgr.get_index_of_drive_id(&new_cfg.drive_id) {
+        match self.get_index_of_drive_id(&new_cfg.drive_id) {
             Some(index) => {
-                let config = &mut mgr.info_list[index].config;
+                let config = &mut self.info_list[index].config;
                 config.rate_limiter = new_cfg.rate_limiter.clone();
-                let device = mgr.info_list[index]
+                let device = self.info_list[index]
                     .device
                     .as_mut()
                     .ok_or_else(|| BlockDeviceError::InvalidDeviceId("".to_owned()))?;
@@ -827,12 +822,11 @@ mod tests {
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(BlockDeviceMgr::insert_device(
-            vm.device_manager_mut(),
-            ctx,
-            dummy_block_device.clone(),
-        )
-        .is_ok());
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device.clone(),)
+            .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
         assert!(!vm.device_manager().block_manager.has_root_block_device());
@@ -897,7 +891,9 @@ mod tests {
             use_shared_irq: None,
             use_generic_irq: None,
         };
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device)
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(device_op_ctx, dummy_block_device)
             .unwrap();
 
         let cfg = BlockDeviceConfigUpdateInfo {
@@ -923,7 +919,9 @@ mod tests {
         let expected_error = "could not send patch message to the block epoll handler".to_string();
 
         assert_eq!(
-            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg)
+            vm.device_manager_mut()
+                .block_manager
+                .update_device_ratelimiters(cfg)
                 .unwrap_err()
                 .to_string(),
             expected_error
@@ -938,7 +936,9 @@ mod tests {
         let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id);
 
         assert_eq!(
-            BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2)
+            vm.device_manager_mut()
+                .block_manager
+                .update_device_ratelimiters(cfg2)
                 .unwrap_err()
                 .to_string(),
             expected_error
@@ -968,12 +968,11 @@ mod tests {
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(BlockDeviceMgr::insert_device(
-            vm.device_manager_mut(),
-            ctx,
-            dummy_block_device.clone(),
-        )
-        .is_ok());
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device.clone(),)
+            .is_ok());
 
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 1);
         assert!(vm.device_manager().block_manager.has_root_block);
@@ -1027,12 +1026,16 @@ mod tests {
 
         let mut vm = crate::vm::tests::create_vm_instance();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_1)
+            .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(
-            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2)
-                .is_err()
-        );
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_2)
+            .is_err());
     }
 
     #[test]
@@ -1112,13 +1115,22 @@ mod tests {
         assert_eq!(vm.device_manager().block_manager.info_list.len(), 3);
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device)
+            .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2)
+            .unwrap();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_3)
+            .unwrap();
     }
 
     #[test]
@@ -1182,13 +1194,19 @@ mod tests {
         let mut vm = crate::vm::tests::create_vm_instance();
 
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_3.clone())
             .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device.clone())
             .unwrap();
 
         assert!(vm.device_manager().block_manager.has_root_block_device(),);
@@ -1255,9 +1273,14 @@ mod tests {
 
         // Add 2 block devices.
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device)
+            .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone())
             .unwrap();
 
         // Get index zero.
@@ -1286,7 +1309,9 @@ mod tests {
         // Update OK.
         dummy_block_device_2.is_read_only = true;
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone())
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone())
             .unwrap();
 
         let index = vm
@@ -1306,21 +1331,21 @@ mod tests {
         let dummy_path_3 = PathBuf::from(dummy_filename_3);
         dummy_block_device_2.path_on_host = dummy_path_3;
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(BlockDeviceMgr::insert_device(
-            vm.device_manager_mut(),
-            ctx,
-            dummy_block_device_2.clone(),
-        )
-        .is_err());
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2.clone(),)
+            .is_err());
 
         // Update with 2 root block devices.
         dummy_block_device_2.path_on_host = dummy_path_2.clone();
         dummy_block_device_2.is_root_device = true;
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        assert!(
-            BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,)
-                .is_err(),
-        );
+        assert!(vm
+            .device_manager_mut()
+            .block_manager
+            .insert_device(ctx, dummy_block_device_2,)
+            .is_err(),);
 
         // Switch roots and add a PARTUUID for the new one.
         let root_block_device_old = BlockDeviceConfigInfo {
@@ -1354,9 +1379,15 @@ mod tests {
             use_generic_irq: None,
         };
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_old)
+            .unwrap();
         let ctx = DeviceOpContext::create_boot_ctx(&vm, None);
-        BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap();
+        vm.device_manager_mut()
+            .block_manager
+            .insert_device(ctx, root_block_device_new)
+            .unwrap();
         assert!(vm.device_manager().block_manager.has_part_uuid_root);
     }
 }

From c477ac551fab20377a7be3a7ddc5af248feed3f2 Mon Sep 17 00:00:00 2001
From: Shuaiyi Zhang <zhang_syi@qq.com>
Date: Tue, 16 May 2023 21:13:38 +0800
Subject: [PATCH 115/150] dragonball: Convert VirtioNetDeviceMgr function to
 method

Convert VirtioNetDeviceMgr::insert_device and
VirtioNetDeviceMgr::update_device_ratelimiters to method.

Fixes: #6880

Signed-off-by: Shuaiyi Zhang <zhang_syi@qq.com>
---
 src/dragonball/src/api/v1/vmm_action.rs       |  8 +++++--
 .../src/device_manager/virtio_net_dev_mgr.rs  | 21 ++++++++-----------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs
index cdaec08a8..a271d04cb 100644
--- a/src/dragonball/src/api/v1/vmm_action.rs
+++ b/src/dragonball/src/api/v1/vmm_action.rs
@@ -549,7 +549,9 @@ impl VmmService {
                 }
             })?;
 
-        VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
+        vm.device_manager_mut()
+            .virtio_net_manager
+            .insert_device(ctx, config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::VirtioNet)
     }
@@ -562,7 +564,9 @@ impl VmmService {
     ) -> VmmRequestResult {
         let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
 
-        VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config)
+        vm.device_manager_mut()
+            .virtio_net_manager
+            .update_device_ratelimiters(config)
             .map(|_| VmmData::Empty)
             .map_err(VmmActionError::VirtioNet)
     }
diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs
index c0b0f62da..bfcc600fb 100644
--- a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs
+++ b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs
@@ -223,7 +223,7 @@ impl VirtioNetDeviceMgr {
 
     /// Insert or update a virtio net device into the manager.
     pub fn insert_device(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         mut ctx: DeviceOpContext,
         config: VirtioNetDeviceConfigInfo,
     ) -> std::result::Result<(), VirtioNetDeviceError> {
@@ -234,8 +234,6 @@ impl VirtioNetDeviceMgr {
             return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot);
         }
 
-        let mgr = &mut device_mgr.virtio_net_manager;
-
         slog::info!(
             ctx.logger(),
             "add virtio-net device configuration";
@@ -244,7 +242,7 @@ impl VirtioNetDeviceMgr {
             "host_dev_name" => &config.host_dev_name,
         );
 
-        let device_index = mgr.info_list.insert_or_update(&config)?;
+        let device_index = self.info_list.insert_or_update(&config)?;
 
         if ctx.is_hotplug {
             slog::info!(
@@ -260,17 +258,17 @@ impl VirtioNetDeviceMgr {
                     let dev = DeviceManager::create_mmio_virtio_device(
                         device,
                         &mut ctx,
-                        config.use_shared_irq.unwrap_or(mgr.use_shared_irq),
+                        config.use_shared_irq.unwrap_or(self.use_shared_irq),
                         config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ),
                     )
                     .map_err(VirtioNetDeviceError::DeviceManager)?;
                     ctx.insert_hotplug_mmio_device(&dev, None)
                         .map_err(VirtioNetDeviceError::DeviceManager)?;
                     // live-upgrade need save/restore device from info.device.
-                    mgr.info_list[device_index].set_device(dev);
+                    self.info_list[device_index].set_device(dev);
                 }
                 Err(e) => {
-                    mgr.info_list.remove(device_index);
+                    self.info_list.remove(device_index);
                     return Err(VirtioNetDeviceError::Virtio(e));
                 }
             }
@@ -281,16 +279,15 @@ impl VirtioNetDeviceMgr {
 
     /// Update the ratelimiter settings of a virtio net device.
     pub fn update_device_ratelimiters(
-        device_mgr: &mut DeviceManager,
+        &mut self,
         new_cfg: VirtioNetDeviceConfigUpdateInfo,
     ) -> std::result::Result<(), VirtioNetDeviceError> {
-        let mgr = &mut device_mgr.virtio_net_manager;
-        match mgr.get_index_of_iface_id(&new_cfg.iface_id) {
+        match self.get_index_of_iface_id(&new_cfg.iface_id) {
             Some(index) => {
-                let config = &mut mgr.info_list[index].config;
+                let config = &mut self.info_list[index].config;
                 config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone();
                 config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone();
-                let device = mgr.info_list[index].device.as_mut().ok_or_else(|| {
+                let device = self.info_list[index].device.as_mut().ok_or_else(|| {
                     VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone())
                 })?;
 

From f6e1b1152cd0413dfe2cd0821b63520df776d9d1 Mon Sep 17 00:00:00 2001
From: Peng Tao <bergwolf@hyper.sh>
Date: Thu, 18 May 2023 09:36:06 +0000
Subject: [PATCH 116/150] agent: update tokio dependency

To 1.28.1 to bring in the latest fixes.

Fixes: #6881
Signed-off-by: Peng Tao <bergwolf@hyper.sh>
---
 src/agent/Cargo.lock                | 158 ++++++++++++++++++++--------
 src/agent/Cargo.toml                |   2 +-
 src/agent/rustjail/Cargo.toml       |   2 +-
 src/agent/src/signal.rs             |   2 +-
 src/agent/vsock-exporter/Cargo.toml |   2 +-
 5 files changed, 121 insertions(+), 45 deletions(-)

diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock
index d5c557a3d..3b0c3efc8 100644
--- a/src/agent/Cargo.lock
+++ b/src/agent/Cargo.lock
@@ -120,7 +120,7 @@ checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -137,7 +137,7 @@ checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -191,7 +191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2"
 dependencies = [
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -329,7 +329,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -399,7 +399,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -410,7 +410,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -457,7 +457,7 @@ checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -600,7 +600,7 @@ checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -992,7 +992,7 @@ dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys",
+ "windows-sys 0.36.1",
 ]
 
 [[package]]
@@ -1257,7 +1257,7 @@ dependencies = [
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys",
+ "windows-sys 0.36.1",
 ]
 
 [[package]]
@@ -1318,7 +1318,7 @@ checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -1379,7 +1379,7 @@ dependencies = [
  "proc-macro-error-attr",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
  "version_check",
 ]
 
@@ -1396,9 +1396,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.40"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7"
+checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8"
 dependencies = [
  "unicode-ident",
 ]
@@ -1473,7 +1473,7 @@ dependencies = [
  "itertools",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -1565,9 +1565,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.20"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804"
+checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
 dependencies = [
  "proc-macro2",
 ]
@@ -1799,7 +1799,7 @@ checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -1821,7 +1821,7 @@ checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -1843,7 +1843,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -1951,9 +1951,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
 
 [[package]]
 name = "socket2"
-version = "0.4.4"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
 dependencies = [
  "libc",
  "winapi",
@@ -1992,6 +1992,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -2051,7 +2062,7 @@ checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -2087,33 +2098,32 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.19.2"
+version = "1.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439"
+checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
 dependencies = [
+ "autocfg",
  "bytes 1.1.0",
  "libc",
- "memchr",
  "mio",
  "num_cpus",
- "once_cell",
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.16",
 ]
 
 [[package]]
@@ -2183,7 +2193,7 @@ checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -2404,7 +2414,7 @@ dependencies = [
  "log",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
  "wasm-bindgen-shared",
 ]
 
@@ -2426,7 +2436,7 @@ checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2494,43 +2504,109 @@ version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
 dependencies = [
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_msvc",
+ "windows_aarch64_msvc 0.36.1",
+ "windows_i686_gnu 0.36.1",
+ "windows_i686_msvc 0.36.1",
+ "windows_x86_64_gnu 0.36.1",
+ "windows_x86_64_msvc 0.36.1",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc 0.48.0",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
 [[package]]
 name = "xattr"
 version = "0.2.3"
@@ -2590,7 +2666,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -2627,5 +2703,5 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.98",
 ]
diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml
index d8ba20db3..bb14dfd36 100644
--- a/src/agent/Cargo.toml
+++ b/src/agent/Cargo.toml
@@ -30,7 +30,7 @@ async-recursion = "0.3.2"
 futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.14.0", features = ["full"] }
+tokio = { version = "1.28.1", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml
index 19602bee2..3113ac643 100644
--- a/src/agent/rustjail/Cargo.toml
+++ b/src/agent/rustjail/Cargo.toml
@@ -29,7 +29,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
-tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
+tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
diff --git a/src/agent/src/signal.rs b/src/agent/src/signal.rs
index 79dea3b08..d67000b80 100644
--- a/src/agent/src/signal.rs
+++ b/src/agent/src/signal.rs
@@ -24,7 +24,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
     loop {
         // Avoid reaping the undesirable child's signal, e.g., execute_hook's
         // The lock should be released immediately.
-        rustjail::container::WAIT_PID_LOCKER.lock().await;
+        let _locker = rustjail::container::WAIT_PID_LOCKER.lock().await;
         let result = wait::waitpid(
             Some(Pid::from_raw(-1)),
             Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL),
diff --git a/src/agent/vsock-exporter/Cargo.toml b/src/agent/vsock-exporter/Cargo.toml
index 0cdf0b91d..7bec1d87a 100644
--- a/src/agent/vsock-exporter/Cargo.toml
+++ b/src/agent/vsock-exporter/Cargo.toml
@@ -18,4 +18,4 @@ bincode = "1.3.3"
 byteorder = "1.4.3"
 slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
 async-trait = "0.1.50"
-tokio = "1.2.0"
+tokio = "1.28.1"

From 11a34a72e2b16d39d888bbd42221e5170b2c8d35 Mon Sep 17 00:00:00 2001
From: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
Date: Thu, 18 May 2023 15:08:08 +0000
Subject: [PATCH 117/150] docs: Update container network model url

This PR updates the container network model url that is part of the
virtcontainers documentation.

Fixes #6889

Signed-off-by: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
---
 src/runtime/virtcontainers/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/runtime/virtcontainers/README.md b/src/runtime/virtcontainers/README.md
index c9556829d..cbf8da778 100644
--- a/src/runtime/virtcontainers/README.md
+++ b/src/runtime/virtcontainers/README.md
@@ -77,7 +77,7 @@ For further details, see the [API documentation](documentation/api/1.0/api.md).
 
 Typically the former is the Docker default networking model while the later is used on Kubernetes deployments.
 
-[cnm]: https://github.com/docker/libnetwork/blob/master/docs/design.md
+[cnm]: https://github.com/moby/libnetwork/blob/master/docs/design.md
 [cni]: https://github.com/containernetworking/cni/
 
 ## CNM

From eff6ed2d5ffa5a1adeedc4fa0500b4b764dd2b80 Mon Sep 17 00:00:00 2001
From: Krister Johansen <kjlx@templeofstupid.com>
Date: Wed, 17 May 2023 16:21:32 -0700
Subject: [PATCH 118/150] runtime: make debug console work with
 sandbox_cgroup_only

If a hypervisor debug console is enabled and sandbox_cgroup_only is set,
the hypervisor can fail to open /dev/ptmx, which prevents the sandbox
from launching.

This is caused by the absence of a device cgroup entry to allow access
to /dev/ptmx.  When sandbox_cgroup_only is not set, the hypervisor
inherits the default unrestrcited device cgroup, but with it enabled it
runs into allow / deny list restrictions.

Fix by adding an allowlist entry for /dev/ptmx when debug is enabled,
sandbox_cgroup_only is true, and no /dev/ptmx is already in the list of
devices.

Fixes: #6870

Signed-off-by: Krister Johansen <kjlx@templeofstupid.com>
---
 src/runtime/virtcontainers/sandbox.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index 0eb866bb0..b0697fd84 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -697,6 +697,7 @@ func (s *Sandbox) createResourceController() error {
 			// Determine if device /dev/null and /dev/urandom exist, and add if they don't
 			nullDeviceExist := false
 			urandomDeviceExist := false
+			ptmxDeviceExist := false
 			for _, device := range resources.Devices {
 				if device.Type == "c" && device.Major == intptr(1) && device.Minor == intptr(3) {
 					nullDeviceExist = true
@@ -705,6 +706,10 @@ func (s *Sandbox) createResourceController() error {
 				if device.Type == "c" && device.Major == intptr(1) && device.Minor == intptr(9) {
 					urandomDeviceExist = true
 				}
+
+				if device.Type == "c" && device.Major == intptr(5) && device.Minor == intptr(2) {
+					ptmxDeviceExist = true
+				}
 			}
 
 			if !nullDeviceExist {
@@ -720,6 +725,18 @@ func (s *Sandbox) createResourceController() error {
 				}...)
 			}
 
+			// If the hypervisor debug console is enabled and
+			// sandbox_cgroup_only are configured, then the vmm needs access to
+			// /dev/ptmx.  Add this to the device allowlist if it is not
+			// already present in the config.
+			if s.config.HypervisorConfig.Debug && s.config.SandboxCgroupOnly && !ptmxDeviceExist {
+				// "/dev/ptmx"
+				resources.Devices = append(resources.Devices, []specs.LinuxDeviceCgroup{
+					{Type: "c", Major: intptr(5), Minor: intptr(2), Access: rwm, Allow: true},
+				}...)
+
+			}
+
 			if spec.Linux.Resources.CPU != nil {
 				resources.CPU = &specs.LinuxCPU{
 					Cpus: spec.Linux.Resources.CPU.Cpus,

From f527f614c1be4b22cd2683983c32d2ad655ecddc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 19 May 2023 09:26:36 +0200
Subject: [PATCH 119/150] release: Kata Containers 3.2.0-alpha1
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- runtime: Use static_sandbox_resource_mgmt=true for TEEs
- update tokio dependency
- resource-control: fix setting CPU affinities on Linux
- runtime: use enable_vcpus_pinning from toml
- gha: k8s: Make the tests more reliable
- gha: Enable SEV-SNP tests on main
- gha: tdx: Use the k3s overlay for kata-cleanup
- runtime: Port sev package to main
- gpu: Rename the last bits from `gpu` to `nvidia-gpu`
- deploy: fix shell script error
- ppc64le: switch virtiofsd from C to rust version
- osbuilder: Fix indentation in rootfs.sh
- virtcontainers/qemu_test.go: Improve coverage
- agent: Add context to errors that may occur when AgentConfig file is …
- virtcontainers/pkg/compatoci/: Improved coverage for  for Kata 2.0
- kata-manager: Fix '-o' syntax and logic error
- kata-ctl:  Add the option to install kata-ctl to a user specified directory
- runtime-rs: fix building instructions to use correct required Rust ve…
- Dragonball: use LinuxBootConfigurator::write_bootparams
- kata-deploy: Add http_proxy as part of the docker build
- kata-deploy: Do not ship the kata tarball
- kata-deploy: Build improvements
- deploy: Fix arch in image tag
- Revert "kata-deploy: Use readinessProbe to ensure everything is ready"
- virtcontainers: Improved test coverage for fc.go from 4.6% to 18.5%
- main | release: Fix multi-arch publishing is not supported
- cache: More fixes to nvidia-gpu kernels caching
- runtime: remove overriding ARCH value by default for ppc64le
- gha: Fix Body Line Length action flagging empty body commit messages
- gha: Fix snap creation workflow
- cache: Fix nvidia-gpu version
- cache: Update the KERNEL_FLAVOUR list to include nvidia-gpu
- packaging: Add SEV-SNP artifacts to main
- docs: Mark snap installation method as unmaintained
- packaging: Add sev artifacts to main
- kata-ctl: add generic kvm check & unit test
- Log-parser-rs
- warning_fix: fix warnings when build with cargo-1.68.0
- cross-compile: Include documentation and configuration for cross-compile
- runtime: Fix virtiofs fd leak
- gpu: cold plug VFIO devices
- pkg/signals: Improved test coverage 60% to 100%
- virtcontainers/persist: Improved test coverage 65% to 87.5%
- virtcontainers/clh_test.go: improve unit test coverage
- virtcontainers/factory: Improved test coverage
- gha: Also run k8s tests on qemu-snp
- gha: sev: fix for kata-deploy error
- gha: Also run k8s tests on qemu-sev
- Implement the "kata-ctl env" command
- runtime-rs: support keep_abnormal in toml config
- gpu: Build and Ship an GPU enabled Kernel
- kata-ctl: checks for kvm, kvm_intel modules loaded
- osbuilder: Fix D-Bus enabling in the dracut case
- snap: fix docker start fail issue
- kata-manager: Fix containerd download
- agent: Fix ut issue caused by fd double closed
- Bump ttrpc to 0.7.2 and protobuf to 3.2.0
- gpu: Add GPU enabled confguration and runtime
- gpu: Do not pass-through PCI (Host) Bridges
- cache-components: Fix caching of TDVF and QEMU for TDX
- gha: tdx: Ensure kata-deploy is removed after the tests run
- versions: Upgrade to Cloud Hypervisor v31.0
- osbuilder: Enable dbus in the dracut case
- runtime: Don't create socket file in /run/kata
- nydus_rootfs/prefetch_files: add prefetch_files for RAFS
- runtime-rs/virtio-fs: add support extra handler for cache mode.
- runtime-rs: enable nerdctl to setup cni plugin
- tdx: Add artefacts from the latest TDX tools release into main
- runtime: support non-root for clh
- gha: ci-on-push: Run k8s tests with dragonball
- rustjail: Use CPUWeight with systemd and CgroupsV2
- gha: k8s-on-aks: {create,delete} AKS must be a coded-in step
- docs: update the rust version from version.yaml
- gha: k8s-on-aks: Set {create,delete}_aks as steps
- gha: k8s-on-aks: Fix cluster name
- gha: Also run k8s tests on AKS with dragonball
- gha: Only push images to registry after merging a PR
- gha: aks: Use D4s_v5 instance
- tools: Avoid building the kernel twice
- rustjail: Fix panic when cgroup manager fails
- runtime: add filter metrics with specific names
- gha: Use ghcr.io for the k8s CI
- GHA |Switch "kubernetes tests" from jenkins to GitHub actions
- docs: Update CNM url in networking document
- kata-ctl: add function to get platform protection.

f6e1b1152 agent: update tokio dependency
4cb83dc21 kata-ctl: update tokio dependency
df615ff25 runk: update tokio dependency
ca6892ddb runtime-rs: update tokio dependency
ca1531fe9 runtime: Use static_sandbox_resource_mgmt=true for TEEs
fa832f470 gha: k8s: Make the tests more reliable
cbb9fe8b8 config: Use standard OVMF with SEV
724437efb kata-deploy: add kata-qemu-sev runtimeclass
521dad2a4 Tests: skip CPU constraints test on SEV and SNP
72308ddb0 gha: ci-on-push: Don't skip tests for SEV
da0f92cef gha: ci-on-push: Don't skip tests for SEV-SNP
12f43bea0 gha: tdx: Use the k3s overlay for kata-cleanup
1a3f8fc1a deploy: fix shell script error
87cb98c01 osbuilder: Fix indentation in rootfs.sh
c5a59caca ppc64le: switch virtiofsd from C to rust version
bfdf0144a versions: Bump virtiofsd to 1.6.1
dd7562522 runtime: pkg/sev: Add kbs utility package for SEV pre-attestation
05de7b260 runtime: Add sev package
3a9d3c72a gpu: Rename the last bits from `gpu` to `nvidia-gpu`
4cde844f7 local-build: Fix kernel-nvidia-gpu target name
593840e07 kata-ctl: Allow INSTALL_PATH= to be specified
bdb75fb21 runtime: use enable_vcpus_pinning from toml
20cb87508 virtcontainers/qemu_test.go: Improve test coverage
b9a1db260 kata-deploy: Add http_proxy as part of the docker build
3e85bf5b1 resource-control: fix setting CPU affinities on Linux
5f3f844a1 runtime-rs: fix building instructions with respect to required Rust version
777c3dc8d kata-deploy: Do not ship the kata tarball
50cc9c582 tests: Improve coverage for virtcontainers/pkg/compatoci/ for Kata 2.0
136e2415d static-build: Download firecracker instead of building it
3bf767cfc static-build: Adjust ARCH for nydus
ac88d34e0 static-build: Use relased binary for CLH (aarch64)
73913c8eb kata-manager: Fix '-o' syntax and logic error
2856d3f23 deploy: Fix arch in image tag
e8f81ee93 Revert "kata-deploy: Use readinessProbe to ensure everything is ready"
cfe63527c release: Fix multi-arch publishing is not supported
197c33651 Dragonball: use LinuxBootConfigurator::write_bootparams to writes the boot parameters into guest memory.
4d17ea4a0 cache: Fix nvidia-snp caching version
a133fadbf cache: Fix nvidia-gpu-tdx-experimental cache URL
b9990c201 cache: Fix nvidia-gpu version
c9bf7808b cache: Update the KERNEL_FLAVOUR list to include nvidia-gpu
3665b4204 gpu: Rename `gpu` targets to `nvidia-gpu`
2c90cac75 local-build: fixup alphabetization
4da6eb588 kata-deploy: Add qemu-snp shim
14dd05375 kata-deploy: add kata-qemu-snp runtimeclass
0bb37bff7 config: Add SNP configuration
af7f2519b versions: update SEV kernel description
dbcc3b5cc local-build: fix default values for OVMF build
b8bbe6325 gha: build OVMF for tests and release
cf0ca265f local-build: Add x86_64 OVMF target
db095ddeb cache: add SNP flavor to comments
f4ee00576 gha: Build and ship QEMU for SNP
7a58a91fa docs: update SNP guide
879333bfc versions: update SNP QEMU version
38ce4a32a local-build: add support to build QEMU for SEV-SNP
5f8008b69 kata-ctl: add unit test for kvm check
a085a6d7b kata-ctl: add generic kvm check
772d4db26 gha: Build and ship SEV initrd
45fa36692 gha: Build and ship SEV OVMF
4770d3064 gha: Build and ship SEV kernel.
fb9c1fc36 runtime: Add qemu-sev config
813e4c576 runtimeClasses: add sev runtime class
af18806a8 static-build: Add caching support to sev ovmf
76ae7a3ab packaging: adding caching capability for kernel
12c5ef902 packaging: add support to build OVMF for SEV
b87820ee8 packaging: add support to build initrd for sev
e1f3b871c docs: Mark snap installation method as unmaintained
022a33de9 agent: Add context to errors when AgentConfig file is missing
b0e6a094b packaging: Add sev kernel build capability
a4c0303d8 virtcontainers: Fixed static checks for improved test coverage for fc.go
8495f830b cross-compile: Include documentation and configuration for cross-compile
13d7f39c7 gpu: Check for VFIO port assignments
6594a9329 tools: made log-parser-rs
03a8cd69c virtcontainers: Improved test coverage for fc.go from 4.6% to 18.5%
9e2b7ff17 gha: sev: fix for kata-deploy error
5c9246db1 gha: Also run k8s tests on qemu-snp
c57a44436 gha: Add the ability to test qemu-snp
406419289 env: Utilize arch specific functionality to get cpu details
fb40c71a2 env: Check for root privileges
1016bc17b config: Add api to fetch config from default config path
b908a780a kata-env: Pass cmd option for file path
b1920198b config: Workaround the way agent and hypervisor configs are fetched
f2b2621de kata-env: Implement the kata-env command.
c849bdb0a gha: Also run k8s tests on qemu-sev
6bf1fc605 virtcontainers/factory: Improved test coverage
0d49ceee0 gha: Fix snap creation workflow warnings
138ada049 gpu: Cold Plug VFIO toml setting
defb64334 runtime: remove overriding ARCH value by default for ppc64le
f7ad75cb1 gpu: Cold-plug extend the api.md
0fec2e698 gpu: Add cold-plug test
f2ebdd81c utils: Get rid of spurious print statement left behind.
9a94f1f14 make: Export VERSION and COMMIT
2f81f48da config: Add file under /opt as another location to look for the config
07f7d17db config: Make the pipe_size field optional
68f635773 config: Make function to get the default conf file public
7565b3356 kata-ctl: Implement Display trait for GuestProtection enum
94a00f934 utils: Make certain constants in utils.rs public
572b338b3 gitignore: Ignore .swp and .swo editor backup files
376884b8a cargo: Update version of clap to 4.1.13
17daeb9dd warning_fix: fix warnings when build with cargo-1.68.0
521519d74 gha: Add the ability to test qemu-sev
205909fbe runtime: Fix virtiofs fd leak
5226f15c8 gha: Fix Body Line Length action flagging empty body commit messages
0f45b0faa virtcontainers/clh_test.go: improve unit test coverage
dded731db gpu: Add OVMF setting for MMIO aperture
2a830177c gpu: Add fwcfg helper function
131f056a1 gpu: Extract VFIO Functions to drivers
c8cf7ed3b gpu: Add ColdPlug of VFIO devices with devManager
e2b5e7f73 gpu: Add Rawdevices to hypervisor
6107c32d7 gpu: Assign default value to cold-plug
377ebc2ad gpu: Add configuration option for cold-plug VFIO
c18ceae10 gpu: Add new struct PCIePort
9c38204f1 virtcontainers/persist: Improved test coverage 65% to 87.5%
1c1ee8057 pkg/signals: Improved test coverage 60% to 100%
cc8ea3232 runtime-rs: support keep_abnormal in toml config
96e8470db kata-manager: Fix containerd download
432d40744 kata-ctl: checks for kvm, kvm_intel modules loaded
b1730e4a6 gpu: Add new kernel build option to usage()
3e7b90226 osbuilder: Fix D-Bus enabling in the dracut case
53c749a9d agent: Fix ut issue caused by fd double closed
2e3f19af9 agent: fix clippy warnings caused by protobuf3
4849c56fa agent: Fix unit test issue cuased by protobuf upgrade
0a582f781 trace-forwarder: remove unused crate protobuf
73253850e kata-ctl: remove unused crate ttrpc
76d2e3054 agent-ctl: Bump ttrpc from 0.6.0 to 0.7.1
eb3d20dcc protocols: Add ut for Serde
59568c79d protocols: add support for Serde
a6b4d92c8 runtime-rs: Bump ttrpc from 0.6.0 to 0.7.1
ac7c63bc6 gpu: Add containerd shim for qemu-gpu
a0cc8a75f gpu: Add a kube runtime class
a81fff706 gpu: Adding a GPU enabled configuration
8af6fc77c agent: Bump ttrpc from 0.6.0 to 0.7.1
009b42dbf protocols: Fix unit test
392732e21 protocols: Bump ttrpc from 0.6.0  to 0.7.1
f4f958d53 gpu: Do not pass-through PCI (Host) Bridges
825e76948 gpu: Add GPU support to default kernel without any TEE
e4ee07f7d gpu: Add GPU TDX experimental kernel
a1272bcf1 gha: tdx: Fix typo overlay -> overlays
3fa0890e5 cache-components: Fix TDVF caching
80e3a2d40 cache-components: Fix TDX QEMU caching
87ea43cd4 gpu: Add configuration fragment
aca6ff728 gpu: Build and Ship an GPU enabled Kernel
dc662333d runtime: Increase the dial_timeout
eb1762e81 osbuilder: Enable dbus in the dracut case
f478b9115 clh: tdx: Update timeouts for confidential guest
3b76abb36 kata-deploy: Ensure node is ready after CRI Engine restart
5ec9ae0f0 kata-deploy: Use readinessProbe to ensure everything is ready
ea386700f kata-deploy: Update podOverhead for TDX
e31efc861 gha: tdx: Use the k3s overlay
542bb0f3f gha: tdx: Set KUBECONFIG env at the job level
d7fdf19e9 gha: tdx: Delete kata-deploy after the tests finish
da35241a9 tests: k8s: Skip k8s-cpu-ns when testing TDX
db2cac34d runtime: Don't create socket file in /run/kata
6d315719f snap: fix docker start fail issue
e4b3b0887 gpu: Add proper CONFIG_LOCALVERSION depending on TEE
69ba2098f runtime-rs: remove network entities and netns
b31f103d1 runtime-rs: enable nerdctl cni plugin
69d7a959c gha: ci-on-push: Run tests on TDX
5a0727ecb kata-deploy: Ship kata-qemu-tdx runtimeClass
98682805b config: Add configuration for QEMU TDX
3e1580019 govmm: Directly pass the firmware using -bios with TDX
3c5ffb0c8 govmm: Set "sept-ve-disable=on"
ed145365e runtime/qemu: Drop "kvm-type=tdx"
25b3cdd38 virtcontainers: Drop check for the `tdx` CPU flag
01bdacb4e virtcontainers: Also check /sys/firmwares/tdx for TDX
9feec533c cache: Add ability to cache OVMF
ce8d98251 gha: Build and ship the OVMF for TDX
39c3fab7b local-build: Add support to build OVMF for TDX
054174d3e versions: Bump OVMF for TDX
800fb49da packaging: Add get_ovmf_image_name() helper
fbf03d7ac cache: Document kernel-tdx-experimental
5d79e9696 cache: Add a space to ease the reading of the kernel flavours
6e4726e45 cache: Fix typos
fc22ed0a8 gha: Build and ship the Kernel for TDX
502844ced local-build: Add support to build Kernel for TDX
b2585eecf local-build: Avoid code duplication building the kernel
f33345c31 versions: Update Kernel TDX version
20ab2c242 versions: Move Kernel TDX to its own experimental entry
3d9ce3982 cache: Allow specifying the QEMU_FLAVOUR
33dc6c65a gha: Build and ship QEMU for TDX
eceaae30a local-build: Add support to build QEMU for TDX
f7b7c187e static-build: Improve qemu-experimental build script
3018c9ad5 versions: Update QEMU TDX version
800ee5cd8 versions: Move QEMU TDX to its own experimental entry
1315bb45f local-build: Add dragonball kernel to the `all` target
73e108136 local-build: Rename non vanilla kernel build functions
1d851b4be local-build: Cosmetic changes in build targets
49ce685eb gha: k8s-on-aks: Always delete the AKS cluster
e2a770df5 gha: ci-on-push: Run k8s tests with dragonball
d1f550bd1 docs: update the rust version from versions.yaml
f3595e48b nydus_rootfs/prefetch_files: add prefetch_files for RAFS
3bfaafbf4 fix: oci hook
c1fbaae8d rustjail: Use CPUWeight with systemd and CgroupsV2
375187e04 versions: Upgrade to Cloud Hypervisor v31.0
79f3047f0 gha: k8s-on-aks: {create,delete} AKS must be a coded-in step
2f35b4d4e gha: ci-on-push: Only run on `main` branch
e7bd2545e Revert "gha: ci-on-push: Depend on Commit Message Check"
0d96d4963 Revert "gha: ci-on-push: Adjust to using workflow_run"
c7ee45f7e Revert "gha: ci-on-push: Adapt chained jobs to workflow_run"
5d4d72064 Revert "gha: k8s-on-aks: Fix cluster name"
13d857a56 gha: k8s-on-aks: Set {create,delete}_aks as steps
dc6569dbb runtime-rs/virtio-fs: add support extra handler for cache mode.
85cc5bb53 gha: k8s-on-aks: Fix cluster name
1688e4f3f gha: aks: Use D4s_v5 instance
108d80a86 gha: Add the ability to also test Dragonball
2550d4462 gha: build-kata-static-tarball: Only push to registry after merge
e81b8b8ee local-build: build-and-upload-payload is not quay.io specific
13929fc61 gha: publish-kata-deploy-payload: Improve registry login
41026f003 gha: payload-after-push: Pass registry / repo as inputs
7855b4306 gha: ci-on-push: Adapt chained jobs to workflow_run
3a760a157 gha: ci-on-push: Adjust to using workflow_run
a159ffdba gha: ci-on-push: Depend on Commit Message Check
8086c75f6 gha: Also run k8s tests on AKS with dragonball
fe86c08a6 tools: Avoid building the kernel twice
3215860a4 gha: Set ci-on-push to run on `pull_request_target`
d17dfe4cd gha: Use ghcr.io for the k8s CI
b661e0cf3 rustjail: Add anyhow context for D-Bus connections
60c62c3b6 gha: Remove kata-deploy-test.yaml
43894e945 gha: Remove kata-deploy-push.yaml
cab9ca043 gha: Add a CI pipeline for Kata Containers
53b526b6b gha: k8s: Add snippet to run k8s tests on aks clusters
c444c24bc gha: aks: Add snippets to create / delete aks clusters
11e0099fb tests: Move k8s tests to this repo
73be4bd3f gha: Update actions for release.yaml
d38d7fbf1 gha: Remove code duplication from release.yaml
56331bd7b gha: Split payload-after-push-*.yaml
a552a1953 docs: Update CNM url in networking document
7796e6ccc rustjail: Fix minor grammatical error in function name
41fdda1d8 rustjail: Do  not unwrap potential error with cgroup manager
a914283ce kata-ctl: add function to get platform protection.
0f7351556 runtime: add filter metrics with specific names
cbe6ad903 runtime: support non-root for clh
d3bb25418 utils: Add function to check vhost-vsock

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 2f81ab203..dcd25b873 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.2.0-alpha0
+3.2.0-alpha1

From a89b44aabfd7ed7e5cbe4c1c94aa2642845709e1 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Fri, 19 May 2023 09:28:22 +0100
Subject: [PATCH 120/150] tools: Fix arch bug

Fix mismatched case of `arch`

Fixes: #6895
Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../static-build/firecracker/build-static-firecracker.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh
index 77ffc0229..95cba1389 100755
--- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh
+++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh
@@ -28,7 +28,7 @@ arch=$(uname -m)
 
 firecracker_tarball_url="${firecracker_url}/releases/download"
 
-file_name="firecracker-${firecracker_version}-${ARCH}.tgz"
+file_name="firecracker-${firecracker_version}-${arch}.tgz"
 download_url="${firecracker_tarball_url}/${firecracker_version}/${file_name}"
 
 info "Download firecracker version: ${firecracker_version} from ${download_url}"

From 75330ab3f96f644fff48312e2954ecb1c2fa4081 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 19 May 2023 14:00:39 +0200
Subject: [PATCH 121/150] cache: Fix OVMF caching
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

OVMF has been cached, but it's not been used from cache as the `version`
set in the cached builds has always been empty.

The reason for that is because we've been trying to look for
`externals.ovmf.ovmf.version`, while we should be actually looking for
`externals.ovmf.x86_64.version`.

Setting `x86_64` as the OVMF_FLAVOUR would cause another bug, as the
expected tarball name would then be `kata-static-x86_64.tar.xz`, instead
of `kata-static-ovmf-x86_64.tar.xz`.

With everything said, let's simplify the OVMF_FLAVOUR usage, by using it
as it's passed, and only adapting the tarball name for the TDVF case,
which is the abnormal one.

Fixes: #6897

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/static-build/cache_components_main.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index ac9e3760b..3391cd15e 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -77,9 +77,8 @@ cache_nydus_artifacts() {
 
 cache_ovmf_artifacts() {
 	local current_ovmf_version="$(get_from_kata_deps "externals.ovmf.${OVMF_FLAVOUR}.version")"
-	[ "${OVMF_FLAVOUR}" == "tdx" ] && OVMF_FLAVOUR="tdvf"
-	[ "${OVMF_FLAVOUR}" == "sev" ] && OVMF_FLAVOUR="ovmf-sev"
-	local ovmf_tarball_name="kata-static-${OVMF_FLAVOUR}.tar.xz"
+	local ovmf_tarball_name="kata-static-ovmf-${OVMF_FLAVOUR}.tar.xz"
+	[ "${OVMF_FLAVOUR}" == "tdx" ] && ovmf_tarball_name="kata-static-tdvf.tar.xz"
 	local current_ovmf_image="$(get_ovmf_image_name)"
 	create_cache_asset "${ovmf_tarball_name}" "${current_ovmf_version}" "${current_ovmf_image}"
 }

From cfd8f4ff76b8493841aa8776118868a675ea6e9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 19 May 2023 14:49:18 +0200
Subject: [PATCH 122/150] gha: payload-after-push: Pass secrets down
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The "build-assets-${arch}" jobs need to have access to the secrets in
order to log into the container registry in the cases where
"push-to-registry", which is used to push the builder containers to
quay.io, is set to "yes".

Now that "build-assets-${arch}" pass the secrets down, we need to log
into the container registry in the "build-kata-static-tarball-${arch}"
files, in case "push-to-registry" is set to "yes".

Fixes: #6899

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 9 +++++++++
 .github/workflows/build-kata-static-tarball-arm64.yaml | 8 ++++++++
 .github/workflows/build-kata-static-tarball-s390x.yaml | 8 ++++++++
 .github/workflows/payload-after-push.yaml              | 3 +++
 4 files changed, 28 insertions(+)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index e6f82b9de..ade4eb9e7 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -38,10 +38,19 @@ jobs:
           - tdvf
           - virtiofsd
     steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
diff --git a/.github/workflows/build-kata-static-tarball-arm64.yaml b/.github/workflows/build-kata-static-tarball-arm64.yaml
index 753bcf13a..1fc981733 100644
--- a/.github/workflows/build-kata-static-tarball-arm64.yaml
+++ b/.github/workflows/build-kata-static-tarball-arm64.yaml
@@ -31,6 +31,14 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
diff --git a/.github/workflows/build-kata-static-tarball-s390x.yaml b/.github/workflows/build-kata-static-tarball-s390x.yaml
index 95e4a5ff5..58186ab8c 100644
--- a/.github/workflows/build-kata-static-tarball-s390x.yaml
+++ b/.github/workflows/build-kata-static-tarball-s390x.yaml
@@ -27,6 +27,14 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
diff --git a/.github/workflows/payload-after-push.yaml b/.github/workflows/payload-after-push.yaml
index 25a7a18c2..97bb309b1 100644
--- a/.github/workflows/payload-after-push.yaml
+++ b/.github/workflows/payload-after-push.yaml
@@ -10,16 +10,19 @@ jobs:
     uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
     with:
       push-to-registry: yes
+    secrets: inherit
 
   build-assets-arm64:
     uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
     with:
       push-to-registry: yes
+    secrets: inherit
 
   build-assets-s390x:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
     with:
       push-to-registry: yes
+    secrets: inherit
 
   publish-kata-deploy-payload-amd64:
     needs: build-assets-amd64

From 191b6dd9ddd8ee74a70997c40784f4d95e39797a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 19 May 2023 16:14:59 +0200
Subject: [PATCH 123/150] gha: release: Fix s390x worklow
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

GitHub is warning us that:
"""
The workflow is not valid. In .github/workflows/release.yaml (Line: 21,
Col: 11): Error from called workflow
kata-containers/kata-containers/.github/workflows/release-s390x.yaml@d2e92c9ec993f56537044950a4673e50707369b5
(Line: 14, Col: 12): Job 'kata-deploy' depends on unknown job
'create-kata-tarball'.
"""

This is happening as we need to reference
"build-kata-static-tarball-s390x" instead of "create-kata-tarball".

Fixes: #6903

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/release-s390x.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml
index 2497be7d2..e9fc5fa7e 100644
--- a/.github/workflows/release-s390x.yaml
+++ b/.github/workflows/release-s390x.yaml
@@ -11,7 +11,7 @@ jobs:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
 
   kata-deploy:
-    needs: create-kata-tarball
+    needs: build-kata-static-tarball-s390x
     runs-on: s390x
     steps:
       - name: Login to Kata Containers docker.io

From ad324adf1d59c8fe6657e71c203afdc43a9591be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Fri, 19 May 2023 16:37:41 +0200
Subject: [PATCH 124/150] gha: aks: Wait a little bit more before run the tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fa832f4709f6213946c11601ccd895ab01ee8bab increased the timeout, which
helped a lot, mainly in the TEE machines.  However, we're still seeing
some failures here and there with the AKS tests.

Let's bump it yet again and, hopefully, those errors to start the tests
will go away.

Fixes: #6905

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/run-k8s-tests-on-aks.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml
index b9886e47e..4ed5a2d30 100644
--- a/.github/workflows/run-k8s-tests-on-aks.yaml
+++ b/.github/workflows/run-k8s-tests-on-aks.yaml
@@ -62,7 +62,7 @@ jobs:
           az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64
 
       - name: Run tests
-        timeout-minutes: 30
+        timeout-minutes: 35
         run: |
           sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
           cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
@@ -76,7 +76,7 @@ jobs:
           # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
           # which may cause issues like not having the node properly labeled or the artefacts
           # properly deployed when the tests actually start running.
-          sleep 60s
+          sleep 150s
 
           pushd tests/integration/kubernetes
           sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml

From cfee99c5777c3aa9c3d1e99cc84e39dd15a77c8c Mon Sep 17 00:00:00 2001
From: Bo Chen <chen.bo@intel.com>
Date: Fri, 19 May 2023 12:07:34 -0700
Subject: [PATCH 125/150] versions: Upgrade to Cloud Hypervisor v32.0

Details of this release can be found in ourroadmap project as iteration
v32.0: https://github.com/orgs/cloud-hypervisor/projects/6.

Fixes: #6682

Signed-off-by: Bo Chen <chen.bo@intel.com>
---
 versions.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.yaml b/versions.yaml
index f18f03d27..9642541dd 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -75,7 +75,7 @@ assets:
       url: "https://github.com/cloud-hypervisor/cloud-hypervisor"
       uscan-url: >-
         https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz
-      version: "v31.0"
+      version: "v32.0"
 
     firecracker:
       description: "Firecracker micro-VMM"

From 35c3d7b4bc1593329a9c2918aab16afcd03dd23e Mon Sep 17 00:00:00 2001
From: Bo Chen <chen.bo@intel.com>
Date: Fri, 19 May 2023 12:49:45 -0700
Subject: [PATCH 126/150] runtime: clh: Re-generate the client code

This patch re-generates the client code for Cloud Hypervisor v32.0.
Note: The client code of cloud-hypervisor's OpenAPI is automatically
generated by openapi-generator.

Fixes: #6632

Signed-off-by: Bo Chen <chen.bo@intel.com>
---
 .../cloud-hypervisor/client/api/openapi.yaml  |  7 ++
 .../client/docs/VmmPingResponse.md            | 52 +++++++++++++
 .../client/model_vmm_ping_response.go         | 74 ++++++++++++++++++-
 .../cloud-hypervisor/cloud-hypervisor.yaml    |  5 ++
 4 files changed, 137 insertions(+), 1 deletion(-)

diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml
index d514ae7b8..3af93f9c2 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml
@@ -415,10 +415,17 @@ components:
     VmmPingResponse:
       description: Virtual Machine Monitor information
       example:
+        build_version: build_version
+        pid: 0
         version: version
       properties:
+        build_version:
+          type: string
         version:
           type: string
+        pid:
+          format: int64
+          type: integer
       required:
       - version
       type: object
diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md
index f48f42989..29abc3c88 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md
@@ -4,7 +4,9 @@
 
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
+**BuildVersion** | Pointer to **string** |  | [optional] 
 **Version** | **string** |  | 
+**Pid** | Pointer to **int64** |  | [optional] 
 
 ## Methods
 
@@ -25,6 +27,31 @@ NewVmmPingResponseWithDefaults instantiates a new VmmPingResponse object
 This constructor will only assign default values to properties that have it defined,
 but it doesn't guarantee that properties required by API are set
 
+### GetBuildVersion
+
+`func (o *VmmPingResponse) GetBuildVersion() string`
+
+GetBuildVersion returns the BuildVersion field if non-nil, zero value otherwise.
+
+### GetBuildVersionOk
+
+`func (o *VmmPingResponse) GetBuildVersionOk() (*string, bool)`
+
+GetBuildVersionOk returns a tuple with the BuildVersion field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetBuildVersion
+
+`func (o *VmmPingResponse) SetBuildVersion(v string)`
+
+SetBuildVersion sets BuildVersion field to given value.
+
+### HasBuildVersion
+
+`func (o *VmmPingResponse) HasBuildVersion() bool`
+
+HasBuildVersion returns a boolean if a field has been set.
+
 ### GetVersion
 
 `func (o *VmmPingResponse) GetVersion() string`
@@ -45,6 +72,31 @@ and a boolean to check if the value has been set.
 SetVersion sets Version field to given value.
 
 
+### GetPid
+
+`func (o *VmmPingResponse) GetPid() int64`
+
+GetPid returns the Pid field if non-nil, zero value otherwise.
+
+### GetPidOk
+
+`func (o *VmmPingResponse) GetPidOk() (*int64, bool)`
+
+GetPidOk returns a tuple with the Pid field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetPid
+
+`func (o *VmmPingResponse) SetPid(v int64)`
+
+SetPid sets Pid field to given value.
+
+### HasPid
+
+`func (o *VmmPingResponse) HasPid() bool`
+
+HasPid returns a boolean if a field has been set.
+
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 
diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go
index 63c7b0f50..314d7da33 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go
@@ -16,7 +16,9 @@ import (
 
 // VmmPingResponse Virtual Machine Monitor information
 type VmmPingResponse struct {
-	Version string `json:"version"`
+	BuildVersion *string `json:"build_version,omitempty"`
+	Version      string  `json:"version"`
+	Pid          *int64  `json:"pid,omitempty"`
 }
 
 // NewVmmPingResponse instantiates a new VmmPingResponse object
@@ -37,6 +39,38 @@ func NewVmmPingResponseWithDefaults() *VmmPingResponse {
 	return &this
 }
 
+// GetBuildVersion returns the BuildVersion field value if set, zero value otherwise.
+func (o *VmmPingResponse) GetBuildVersion() string {
+	if o == nil || o.BuildVersion == nil {
+		var ret string
+		return ret
+	}
+	return *o.BuildVersion
+}
+
+// GetBuildVersionOk returns a tuple with the BuildVersion field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *VmmPingResponse) GetBuildVersionOk() (*string, bool) {
+	if o == nil || o.BuildVersion == nil {
+		return nil, false
+	}
+	return o.BuildVersion, true
+}
+
+// HasBuildVersion returns a boolean if a field has been set.
+func (o *VmmPingResponse) HasBuildVersion() bool {
+	if o != nil && o.BuildVersion != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetBuildVersion gets a reference to the given string and assigns it to the BuildVersion field.
+func (o *VmmPingResponse) SetBuildVersion(v string) {
+	o.BuildVersion = &v
+}
+
 // GetVersion returns the Version field value
 func (o *VmmPingResponse) GetVersion() string {
 	if o == nil {
@@ -61,11 +95,49 @@ func (o *VmmPingResponse) SetVersion(v string) {
 	o.Version = v
 }
 
+// GetPid returns the Pid field value if set, zero value otherwise.
+func (o *VmmPingResponse) GetPid() int64 {
+	if o == nil || o.Pid == nil {
+		var ret int64
+		return ret
+	}
+	return *o.Pid
+}
+
+// GetPidOk returns a tuple with the Pid field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *VmmPingResponse) GetPidOk() (*int64, bool) {
+	if o == nil || o.Pid == nil {
+		return nil, false
+	}
+	return o.Pid, true
+}
+
+// HasPid returns a boolean if a field has been set.
+func (o *VmmPingResponse) HasPid() bool {
+	if o != nil && o.Pid != nil {
+		return true
+	}
+
+	return false
+}
+
+// SetPid gets a reference to the given int64 and assigns it to the Pid field.
+func (o *VmmPingResponse) SetPid(v int64) {
+	o.Pid = &v
+}
+
 func (o VmmPingResponse) MarshalJSON() ([]byte, error) {
 	toSerialize := map[string]interface{}{}
+	if o.BuildVersion != nil {
+		toSerialize["build_version"] = o.BuildVersion
+	}
 	if true {
 		toSerialize["version"] = o.Version
 	}
+	if o.Pid != nil {
+		toSerialize["pid"] = o.Pid
+	}
 	return json.Marshal(toSerialize)
 }
 
diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml
index a2392f988..0300d4022 100644
--- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml
+++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml
@@ -438,8 +438,13 @@ components:
         - version
       type: object
       properties:
+        build_version:
+          type: string
         version:
           type: string
+        pid:
+          type: integer
+          format: int64
       description: Virtual Machine Monitor information
 
     VmInfo:

From ae24dc73c13124835bc9805b097a7c747d8d13e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 22 May 2023 08:36:01 +0200
Subject: [PATCH 127/150] local-build: Standardise what's set for the local
 build scripts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We've a discrepancy on what's set along the scripts used to build the
Kata Cotainers artefacts locally.

Some of those were missing a way to easily debug them in case of a
failure happens, but one specific one (build-and-upload-payload.sh)
could actually silently fail.

All of those have been changed as part of this commut.

Fixes: #6908

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../local-build/kata-deploy-binaries-in-docker.sh           | 1 +
 .../local-build/kata-deploy-build-and-upload-payload.sh     | 6 ++++++
 .../local-build/kata-deploy-copy-yq-installer.sh            | 1 +
 .../kata-deploy/local-build/kata-deploy-merge-builds.sh     | 1 +
 4 files changed, 9 insertions(+)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
index 70460a33b..9f4f700e9 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -z "${DEBUG}" ] || set -x
 set -o errexit
 set -o nounset
 set -o pipefail
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
index 193f9511e..be460ffa7 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh
@@ -5,6 +5,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -z "${DEBUG}" ] || set -x
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
 KATA_DEPLOY_DIR="`dirname ${0}`/../../kata-deploy"
 KATA_DEPLOY_ARTIFACT="${1:-"kata-static.tar.xz"}"
 REGISTRY="${2:-"quay.io/kata-containers/kata-deploy"}"
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
index 1271fd882..73323dd28 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -z "${DEBUG}" ] || set -x
 set -o errexit
 set -o nounset
 set -o pipefail
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh
index d99c49b4e..dea0f3b89 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -z "${DEBUG}" ] || set -x
 set -o errexit
 set -o nounset
 set -o pipefail

From 636539bf0cc6eafa8db40af3fb5f1608db219f8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 22 May 2023 09:46:46 +0200
Subject: [PATCH 128/150] kata-deploy: Use apt-key.gpg from k8s.io
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We're facing some issues to download / use the public key provided by
google for installing kubernetes as part of the kata-deploy image.
```
The following signatures couldn't be verified because the public key is
not available: NO_PUBKEY B53DC80D13EDEF05
Reading package lists... Done
W: GPG error: https://packages.cloud.google.com/apt kubernetes-xenial
   InRelease: The following signatures couldn't be verified because the
   public key is not available: NO_PUBKEY B53DC80D13EDEF05 E: The
   repository 'https://apt.kubernetes.io kubernetes-xenial InRelease' is
   not signed.
N: Updating from such a repository can't be done securely, and is
   therefore disabled by default.
N: See apt-secure(8) manpage for repository creation and user
   configuration details.
```

Let's work this around following the suggestion made by @dims, at:
https://github.com/kubernetes/k8s.io/pull/4837#issuecomment-1446426585

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile
index 6bb470bb5..bb579631b 100644
--- a/tools/packaging/kata-deploy/Dockerfile
+++ b/tools/packaging/kata-deploy/Dockerfile
@@ -18,7 +18,7 @@ RUN \
 apt-get update && \
 apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \
 mkdir -p /etc/apt/keyrings/ && \
-curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \
+curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && \
 echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \
 apt-get update && \
 apt-get install -y --no-install-recommends kubectl && \

From b8ffcd1b9b69bdcb64289820ca865c751cfe56f9 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 22 May 2023 13:47:49 +0100
Subject: [PATCH 129/150] osbuilder: Bump fedora image version

- Swap out an EoL fedora image for the latest

Fixes: #6923
Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 tools/osbuilder/dockerfiles/QAT/Dockerfile | 2 +-
 tools/osbuilder/image-builder/Dockerfile   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/osbuilder/dockerfiles/QAT/Dockerfile b/tools/osbuilder/dockerfiles/QAT/Dockerfile
index 3455e4fa1..d13d2c656 100644
--- a/tools/osbuilder/dockerfiles/QAT/Dockerfile
+++ b/tools/osbuilder/dockerfiles/QAT/Dockerfile
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Kata osbuilder 'works best' on Fedora
-FROM fedora:34
+FROM fedora:38
 
 # Version of the Dockerfile - update if you change this file to avoid 'stale'
 # images being pulled from the registry.
diff --git a/tools/osbuilder/image-builder/Dockerfile b/tools/osbuilder/image-builder/Dockerfile
index 02f93475f..87f1e570f 100644
--- a/tools/osbuilder/image-builder/Dockerfile
+++ b/tools/osbuilder/image-builder/Dockerfile
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 ARG IMAGE_REGISTRY=registry.fedoraproject.org
-FROM ${IMAGE_REGISTRY}/fedora:34
+FROM ${IMAGE_REGISTRY}/fedora:38
 
 RUN ([ -n "$http_proxy" ] && \
     sed -i '$ a proxy='$http_proxy /etc/dnf/dnf.conf ; true) && \

From b7341cd9689f167b5c04e504cf8b32f6829bf4b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 22 May 2023 10:05:28 +0200
Subject: [PATCH 130/150] cache: Use "initrd" as `initrd_type` to build
 rootfs-initrd
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We've been defaulting to "", which would lead to a mismatch with the
latest version from the cache, causing a miss, and finally having to
build the rootfs-initrd as part of the tests, every single time.

Fixes: #6917

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 8dbebee42..877576ca6 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -159,7 +159,7 @@ install_image() {
 
 #Install guest initrd
 install_initrd() {
-	local initrd_type="${1:-""}"
+	local initrd_type="${1:-"initrd"}"
 	local initrd_suffix="${2:-""}"
 	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}"
 	local component="rootfs-${initrd_type}"

From 22154e0a3b354640a6a1ef976eb5d8379a066955 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Mon, 22 May 2023 10:09:34 +0200
Subject: [PATCH 131/150] cache: Fix OVMF tarball name for different flavours
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

75330ab3f96f644fff48312e2954ecb1c2fa4081 tried to fix OVMF caching, but
didn't consider that the "vanilla" OVMF tarball name is not
"kata-static-ovmf-x86_64.tar.xz", but rather "kata-static-ovmf.tar.xz".

The fact we missed that, led to the cache builds of OVMF failing, and
the need to build the component on every single PR.

Fixes: #6917 (hopefully for good this time).

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../static-build/cache_components_main.sh          | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh
index 3391cd15e..50b2ba83e 100755
--- a/tools/packaging/static-build/cache_components_main.sh
+++ b/tools/packaging/static-build/cache_components_main.sh
@@ -77,8 +77,18 @@ cache_nydus_artifacts() {
 
 cache_ovmf_artifacts() {
 	local current_ovmf_version="$(get_from_kata_deps "externals.ovmf.${OVMF_FLAVOUR}.version")"
-	local ovmf_tarball_name="kata-static-ovmf-${OVMF_FLAVOUR}.tar.xz"
-	[ "${OVMF_FLAVOUR}" == "tdx" ] && ovmf_tarball_name="kata-static-tdvf.tar.xz"
+	case ${OVMF_FLAVOUR} in
+		"tdx")
+			ovmf_tarball_name="kata-static-tdvf.tar.xz"
+			;;
+		"x86_64")
+			ovmf_tarball_name="kata-static-ovmf.tar.xz"
+			;;
+		*)
+			ovmf_tarball_name="kata-static-ovmf-${OVMF_FLAVOUR}.tar.xz"
+			;;
+	esac
+			
 	local current_ovmf_image="$(get_ovmf_image_name)"
 	create_cache_asset "${ovmf_tarball_name}" "${current_ovmf_version}" "${current_ovmf_image}"
 }

From e4c5c74a75724f40717c2276509da9e15cf6cb3c Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Wed, 10 May 2023 14:40:05 +0800
Subject: [PATCH 132/150] runtime-rs: device manager

Support device manager for runtime-rs, add block device handler for
device manager

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 src/runtime-rs/Cargo.lock                     |  42 ++++
 src/runtime-rs/crates/hypervisor/Cargo.toml   |   2 +
 .../crates/hypervisor/src/ch/inner.rs         |   4 +-
 .../crates/hypervisor/src/ch/inner_device.rs  |  17 +-
 .../hypervisor/src/ch/inner_hypervisor.rs     |   4 +-
 .../crates/hypervisor/src/ch/mod.rs           |   6 +-
 .../crates/hypervisor/src/device/block.rs     |  24 ---
 .../hypervisor/src/device/device_manager.rs   | 199 ++++++++++++++++++
 .../hypervisor/src/device/driver/mod.rs       |  36 ++++
 .../src/device/{ => driver}/vfio.rs           |   4 +-
 .../src/device/driver/virtio_blk.rs           | 124 +++++++++++
 .../virtio_fs.rs}                             |  26 ++-
 .../{network.rs => driver/virtio_net.rs}      |   3 +-
 .../{vsock.rs => driver/virtio_vsock.rs}      |   0
 .../crates/hypervisor/src/device/mod.rs       |  56 +++--
 .../hypervisor/src/device/share_fs_device.rs  |  27 ---
 .../crates/hypervisor/src/device/util.rs      |  88 ++++++++
 .../crates/hypervisor/src/dragonball/inner.rs |   6 +-
 .../hypervisor/src/dragonball/inner_device.rs |  26 +--
 .../src/dragonball/inner_hypervisor.rs        |   2 +-
 .../crates/hypervisor/src/dragonball/mod.rs   |   6 +-
 src/runtime-rs/crates/hypervisor/src/lib.rs   |   7 +-
 .../crates/hypervisor/src/qemu/inner.rs       |   6 +-
 .../crates/hypervisor/src/qemu/mod.rs         |   6 +-
 .../src/network/endpoint/ipvlan_endpoint.rs   |   6 +-
 .../src/network/endpoint/macvlan_endpoint.rs  |   6 +-
 .../src/network/endpoint/physical_endpoint.rs |  10 +-
 .../src/network/endpoint/veth_endpoint.rs     |   6 +-
 .../src/network/endpoint/vlan_endpoint.rs     |   6 +-
 .../resource/src/share_fs/share_virtio_fs.rs  |   4 +-
 30 files changed, 611 insertions(+), 148 deletions(-)
 delete mode 100644 src/runtime-rs/crates/hypervisor/src/device/block.rs
 create mode 100644 src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
 create mode 100644 src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
 rename src/runtime-rs/crates/hypervisor/src/device/{ => driver}/vfio.rs (98%)
 create mode 100644 src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
 rename src/runtime-rs/crates/hypervisor/src/device/{share_fs_mount.rs => driver/virtio_fs.rs} (64%)
 rename src/runtime-rs/crates/hypervisor/src/device/{network.rs => driver/virtio_net.rs} (94%)
 rename src/runtime-rs/crates/hypervisor/src/device/{vsock.rs => driver/virtio_vsock.rs} (100%)
 delete mode 100644 src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs
 create mode 100644 src/runtime-rs/crates/hypervisor/src/device/util.rs

diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock
index 35a80d221..276facdbb 100644
--- a/src/runtime-rs/Cargo.lock
+++ b/src/runtime-rs/Cargo.lock
@@ -61,6 +61,17 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.8",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.20"
@@ -816,6 +827,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dlv-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
+
 [[package]]
 name = "dragonball"
 version = "0.1.0"
@@ -1176,6 +1193,9 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
 
 [[package]]
 name = "heck"
@@ -1281,6 +1301,7 @@ dependencies = [
 name = "hypervisor"
 version = "0.1.0"
 dependencies = [
+ "actix-rt",
  "anyhow",
  "async-trait",
  "ch-config",
@@ -1296,6 +1317,7 @@ dependencies = [
  "nix 0.24.3",
  "persist",
  "rand 0.8.5",
+ "rust-ini",
  "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "seccompiler",
  "serde",
@@ -1979,6 +2001,16 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "ordered-multimap"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
+dependencies = [
+ "dlv-list",
+ "hashbrown",
+]
+
 [[package]]
 name = "parking"
 version = "2.0.0"
@@ -2510,6 +2542,16 @@ dependencies = [
  "wasm_container",
 ]
 
+[[package]]
+name = "rust-ini"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
+dependencies = [
+ "cfg-if 1.0.0",
+ "ordered-multimap",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.21"
diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml
index 3e3a95d0b..eb613aad3 100644
--- a/src/runtime-rs/crates/hypervisor/Cargo.toml
+++ b/src/runtime-rs/crates/hypervisor/Cargo.toml
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+actix-rt = "2.7.0"
 anyhow = "^1.0"
 async-trait = "0.1.48"
 dbs-utils = "0.2.0"
@@ -15,6 +16,7 @@ go-flag = "0.1.0"
 libc = ">=0.2.39"
 nix = "0.24.2"
 persist = { path = "../persist" }
+rust-ini = "0.18.0"
 seccompiler = "0.2.0"
 serde = { version = "1.0.138", features = ["derive"] }
 serde_json = ">=1.0.9"
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
index b2bd32176..33a3fbc56 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::HypervisorState;
-use crate::device::Device;
+use crate::driver::DeviceConfig;
 use crate::VmmState;
 use anyhow::Result;
 use async_trait::async_trait;
@@ -44,7 +44,7 @@ pub struct CloudHypervisorInner {
     pub(crate) jailer_root: String,
 
     /// List of devices that will be added to the VM once it boots
-    pub(crate) pending_devices: Option<Vec<Device>>,
+    pub(crate) pending_devices: Option<Vec<DeviceConfig>>,
 
     pub(crate) _capabilities: Capabilities,
 
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
index c04531de4..ff0f874ec 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
@@ -5,7 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::inner::CloudHypervisorInner;
-use crate::device::{Device, ShareFsDeviceConfig};
+use crate::driver::{DeviceConfig, ShareFsDeviceConfig};
 use crate::HybridVsockConfig;
 use crate::VmmState;
 use anyhow::{anyhow, Context, Result};
@@ -18,9 +18,10 @@ use std::path::PathBuf;
 const VIRTIO_FS: &str = "virtio-fs";
 
 impl CloudHypervisorInner {
-    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
         if self.state != VmmState::VmRunning {
-            let mut devices: Vec<Device> = if let Some(devices) = self.pending_devices.take() {
+            let mut devices: Vec<DeviceConfig> = if let Some(devices) = self.pending_devices.take()
+            {
                 devices
             } else {
                 vec![]
@@ -38,10 +39,10 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    async fn handle_add_device(&mut self, device: Device) -> Result<()> {
+    async fn handle_add_device(&mut self, device: DeviceConfig) -> Result<()> {
         match device {
-            Device::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
-            Device::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
+            DeviceConfig::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
+            DeviceConfig::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
             _ => Err(anyhow!("unhandled device: {:?}", device)),
         }
     }
@@ -66,7 +67,7 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    pub(crate) async fn remove_device(&mut self, _device: Device) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, _device: DeviceConfig) -> Result<()> {
         Ok(())
     }
 
@@ -132,7 +133,7 @@ impl CloudHypervisorInner {
         if let Some(devices) = pending_root_devices {
             for dev in devices {
                 match dev {
-                    Device::ShareFsDevice(dev) => {
+                    DeviceConfig::ShareFsDevice(dev) => {
                         let settings = ShareFsSettings::new(dev, self.vm_path.clone());
 
                         let fs_cfg = FsConfig::try_from(settings)?;
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
index fa1979e83..48eeccc83 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
@@ -7,7 +7,7 @@ use super::inner::CloudHypervisorInner;
 use crate::ch::utils::get_api_socket_path;
 use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path};
 use crate::kernel_param::KernelParams;
-use crate::Device;
+use crate::DeviceConfig;
 use crate::VsockConfig;
 use crate::VM_ROOTFS_DRIVER_PMEM;
 use crate::{VcpuThreadIds, VmmState};
@@ -419,7 +419,7 @@ impl CloudHypervisorInner {
 
         let vsock_cfg = VsockConfig::new(self.id.clone()).await?;
 
-        let dev = Device::Vsock(vsock_cfg);
+        let dev = DeviceConfig::Vsock(vsock_cfg);
         self.add_device(dev).await.context("add vsock device")?;
 
         self.start_hypervisor(self.timeout_secs).await?;
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
index f8f44710e..baeadd2d2 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::HypervisorState;
-use crate::{device::Device, Hypervisor, VcpuThreadIds};
+use crate::{driver::DeviceConfig, Hypervisor, VcpuThreadIds};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_types::capabilities::Capabilities;
@@ -78,12 +78,12 @@ impl Hypervisor for CloudHypervisor {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: Device) -> Result<()> {
+    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: Device) -> Result<()> {
+    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/device/block.rs b/src/runtime-rs/crates/hypervisor/src/device/block.rs
deleted file mode 100644
index 4f59cc0ea..000000000
--- a/src/runtime-rs/crates/hypervisor/src/device/block.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#[derive(Debug)]
-pub struct BlockConfig {
-    /// Unique identifier of the drive.
-    pub id: String,
-
-    /// Path of the drive.
-    pub path_on_host: String,
-
-    /// If set to true, the drive is opened in read-only mode. Otherwise, the
-    /// drive is opened as read-write.
-    pub is_readonly: bool,
-
-    /// Don't close `path_on_host` file when dropping the device.
-    pub no_drop: bool,
-
-    /// device index
-    pub index: u64,
-}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
new file mode 100644
index 000000000..8d6d06652
--- /dev/null
+++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
@@ -0,0 +1,199 @@
+// Copyright (c) 2019-2023 Alibaba Cloud
+// Copyright (c) 2019-2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::{collections::HashMap, sync::Arc};
+
+use anyhow::{anyhow, Context, Ok, Result};
+use kata_sys_util::rand::RandomBytes;
+use tokio::sync::Mutex;
+
+use crate::{
+    BlockConfig, DeviceConfig, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE,
+    VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
+};
+
+use super::{
+    util::{get_host_path, get_virt_drive_name},
+    Device,
+};
+pub type ArcMutexBoxDevice = Arc<Mutex<dyn Device>>;
+
+/// block_index and released_block_index are used to search an available block index
+/// in Sandbox.
+///
+/// @block_index generally default is 1 for <vdb>;
+/// @released_block_index for blk devices removed and indexes will released at the same time.
+#[derive(Clone, Debug, Default)]
+struct SharedInfo {
+    block_index: u64,
+    released_block_index: Vec<u64>,
+}
+
+impl SharedInfo {
+    fn new() -> Self {
+        SharedInfo {
+            block_index: 1,
+            released_block_index: vec![],
+        }
+    }
+
+    // declare the available block index
+    fn declare_device_index(&mut self) -> Result<u64> {
+        let current_index = if let Some(index) = self.released_block_index.pop() {
+            index
+        } else {
+            self.block_index
+        };
+        self.block_index += 1;
+
+        Ok(current_index)
+    }
+
+    fn release_device_index(&mut self, index: u64) {
+        self.released_block_index.push(index);
+        self.released_block_index.sort_by(|a, b| b.cmp(a));
+    }
+}
+
+// Device manager will manage the lifecycle of sandbox device
+pub struct DeviceManager {
+    devices: HashMap<String, ArcMutexBoxDevice>,
+    hypervisor: Arc<dyn Hypervisor>,
+    shared_info: SharedInfo,
+}
+
+impl DeviceManager {
+    pub async fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
+        let devices = HashMap::<String, ArcMutexBoxDevice>::new();
+        Ok(DeviceManager {
+            devices,
+            hypervisor,
+            shared_info: SharedInfo::new(),
+        })
+    }
+
+    pub async fn new_device(&mut self, device_config: DeviceConfig) -> Result<String> {
+        let device_id = if let Some(dev) = self.find_device(&device_config).await {
+            dev
+        } else {
+            self.create_device(&device_config)
+                .await
+                .context("failed to create device")?
+        };
+        Ok(device_id)
+    }
+
+    pub async fn try_add_device(&mut self, device_id: String) -> Result<()> {
+        // get device
+        let device = self
+            .devices
+            .get(&device_id)
+            .context("failed to find device")?;
+        // attach device
+        let result = device.lock().await.attach(self.hypervisor.as_ref()).await;
+        // handle attach error
+        if let Err(e) = result {
+            if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await {
+                self.shared_info.release_device_index(config.index);
+            };
+            self.devices.remove(&device_id);
+            return Err(e);
+        }
+        Ok(())
+    }
+
+    async fn find_device(&self, device_config: &DeviceConfig) -> Option<String> {
+        for (device_id, dev) in &self.devices {
+            match dev.lock().await.get_device_info().await {
+                DeviceConfig::Block(config) => match device_config {
+                    DeviceConfig::Block(ref config_new) => {
+                        if config_new.path_on_host == config.path_on_host {
+                            return Some(device_id.to_string());
+                        }
+                    }
+                    _ => {
+                        continue;
+                    }
+                },
+                _ => {
+                    // TODO: support find other device type
+                    continue;
+                }
+            }
+        }
+        None
+    }
+
+    async fn create_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
+        // device ID must be generated by manager instead of device itself
+        // in case of ID collision
+        let device_id = self.new_device_id()?;
+        let dev: ArcMutexBoxDevice = match device_config {
+            DeviceConfig::Block(config) => self
+                .create_block_device(config, device_id.clone())
+                .await
+                .context("failed to create device")?,
+            _ => {
+                return Err(anyhow!("invliad device type"));
+            }
+        };
+        // register device to devices
+        self.devices.insert(device_id.clone(), dev.clone());
+        Ok(device_id)
+    }
+
+    async fn create_block_device(
+        &mut self,
+        config: &BlockConfig,
+        device_id: String,
+    ) -> Result<ArcMutexBoxDevice> {
+        let mut block_config = config.clone();
+        block_config.id = device_id.clone();
+        // get hypervisor block driver
+        let block_driver = match self
+            .hypervisor
+            .hypervisor_config()
+            .await
+            .blockdev_info
+            .block_device_driver
+            .as_str()
+        {
+            // convert the block driver to kata type
+            VIRTIO_BLOCK_MMIO => KATA_MMIO_BLK_DEV_TYPE.to_string(),
+            VIRTIO_BLOCK_PCI => KATA_BLK_DEV_TYPE.to_string(),
+            _ => "".to_string(),
+        };
+        block_config.driver_option = block_driver;
+        // generate virt path
+        let current_index = self.shared_info.declare_device_index()?;
+        block_config.index = current_index;
+        let drive_name = get_virt_drive_name(current_index as i32)?;
+        block_config.virt_path = format!("/dev/{}", drive_name);
+        // if the path on host is empty, we need to get device host path from major and minor
+        // Otherwise, it might be rawfile based block device
+        if block_config.path_on_host.is_empty() {
+            block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor)
+                .context("failed to get host path")?;
+        }
+        Ok(Arc::new(Mutex::new(BlockConfig::new(block_config))))
+    }
+
+    // device ID must be generated by device manager instead of device itself
+    // in case of ID collision
+    fn new_device_id(&self) -> Result<String> {
+        for _ in 0..5 {
+            let rand_bytes = RandomBytes::new(8);
+            let id = format!("{:x}", rand_bytes);
+
+            // check collision in devices
+            if self.devices.get(&id).is_none() {
+                return Ok(id);
+            }
+        }
+
+        Err(anyhow!("ID are exhausted"))
+    }
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
new file mode 100644
index 000000000..984422b7b
--- /dev/null
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
@@ -0,0 +1,36 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+mod virtio_blk;
+pub use virtio_blk::{
+    BlockConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
+};
+mod virtio_net;
+pub use virtio_net::{Address, NetworkConfig};
+mod vfio;
+pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig};
+mod virtio_fs;
+pub use virtio_fs::{ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountType, ShareFsOperation};
+mod virtio_vsock;
+use std::fmt;
+pub use virtio_vsock::{HybridVsockConfig, VsockConfig};
+
+#[derive(Debug)]
+pub enum DeviceConfig {
+    Block(BlockConfig),
+    Network(NetworkConfig),
+    ShareFsDevice(ShareFsDeviceConfig),
+    Vfio(VfioConfig),
+    ShareFsMount(ShareFsMountConfig),
+    Vsock(VsockConfig),
+    HybridVsock(HybridVsockConfig),
+}
+
+impl fmt::Display for DeviceConfig {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
similarity index 98%
rename from src/runtime-rs/crates/hypervisor/src/device/vfio.rs
rename to src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
index 608091379..fcee1bb8c 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
@@ -25,7 +25,7 @@ const VFIO_UNBIND_PATH: &str = "/sys/bus/pci/drivers/vfio-pci/unbind";
 
 pub const VFIO_PCI: &str = "vfio-pci";
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum VfioBusMode {
     PCI,
     MMIO,
@@ -40,7 +40,7 @@ impl VfioBusMode {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct VfioConfig {
     /// Unique identifier of the device
     pub id: String,
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
new file mode 100644
index 000000000..c80be671f
--- /dev/null
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
@@ -0,0 +1,124 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio";
+use crate::Hypervisor as hypervisor;
+use crate::{device::Device, DeviceConfig};
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+/// VIRTIO_BLOCK_PCI indicates block driver is virtio-pci based
+pub const VIRTIO_BLOCK_PCI: &str = "virtio-blk-pci";
+pub const KATA_MMIO_BLK_DEV_TYPE: &str = "mmioblk";
+pub const KATA_BLK_DEV_TYPE: &str = "blk";
+
+#[derive(Debug, Clone, Default)]
+pub struct BlockConfig {
+    /// Unique identifier of the drive.
+    pub id: String,
+
+    /// Path of the drive.
+    pub path_on_host: String,
+
+    /// If set to true, the drive is opened in read-only mode. Otherwise, the
+    /// drive is opened as read-write.
+    pub is_readonly: bool,
+
+    /// Don't close `path_on_host` file when dropping the device.
+    pub no_drop: bool,
+
+    /// device index
+    pub index: u64,
+
+    /// driver type for block device
+    pub driver_option: String,
+
+    /// device path in guest
+    pub virt_path: String,
+
+    /// device attach count
+    pub attach_count: u64,
+
+    /// device major number
+    pub major: i64,
+
+    /// device minor number
+    pub minor: i64,
+}
+
+impl BlockConfig {
+    // new creates a new VirtioBlkDevice
+    pub fn new(dev_info: BlockConfig) -> Self {
+        dev_info
+    }
+}
+
+#[async_trait]
+impl Device for BlockConfig {
+    async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
+        // increase attach count, skip attach the device if the device is already attached
+        if self
+            .increase_attach_count()
+            .await
+            .context("failed to increase attach count")?
+        {
+            return Ok(());
+        }
+        if let Err(e) = h.add_device(DeviceConfig::Block(self.clone())).await {
+            self.decrease_attach_count().await?;
+            return Err(e);
+        }
+        return Ok(());
+    }
+
+    async fn detach(&mut self, h: &dyn hypervisor) -> Result<Option<u64>> {
+        if self
+            .decrease_attach_count()
+            .await
+            .context("failed to decrease attach count")?
+        {
+            return Ok(None);
+        }
+        if let Err(e) = h.remove_device(DeviceConfig::Block(self.clone())).await {
+            self.increase_attach_count().await?;
+            return Err(e);
+        }
+        Ok(Some(self.index))
+    }
+
+    async fn get_device_info(&self) -> DeviceConfig {
+        DeviceConfig::Block(self.clone())
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        match self.attach_count {
+            0 => {
+                // do real attach
+                self.attach_count += 1;
+                Ok(false)
+            }
+            std::u64::MAX => Err(anyhow!("device was attached too many times")),
+            _ => {
+                self.attach_count += 1;
+                Ok(true)
+            }
+        }
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        match self.attach_count {
+            0 => Err(anyhow!("detaching a device that wasn't attached")),
+            1 => {
+                // do real wrok
+                self.attach_count -= 1;
+                Ok(false)
+            }
+            _ => {
+                self.attach_count -= 1;
+                Ok(true)
+            }
+        }
+    }
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs
similarity index 64%
rename from src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs
rename to src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs
index 85f516456..b40133c65 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs
@@ -11,14 +11,14 @@ pub enum ShareFsOperation {
     Update,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum ShareFsMountType {
     PASSTHROUGH,
     RAFS,
 }
 
 /// ShareFsMountConfig: share fs mount config
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ShareFsMountConfig {
     /// source: the passthrough fs exported dir or rafs meta file of rafs
     pub source: String,
@@ -41,3 +41,25 @@ pub struct ShareFsMountConfig {
     /// prefetch_list_path: path to file that contains file lists that should be prefetched by rafs
     pub prefetch_list_path: Option<String>,
 }
+
+/// ShareFsDeviceConfig: share fs device config
+#[derive(Debug, Clone)]
+pub struct ShareFsDeviceConfig {
+    /// fs_type: virtiofs or inline-virtiofs
+    pub fs_type: String,
+
+    /// socket_path: socket path for virtiofs
+    pub sock_path: String,
+
+    /// mount_tag: a label used as a hint to the guest.
+    pub mount_tag: String,
+
+    /// host_path: the host filesystem path for this volume.
+    pub host_path: String,
+
+    /// queue_size: queue size
+    pub queue_size: u64,
+
+    /// queue_num: queue number
+    pub queue_num: u64,
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/network.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs
similarity index 94%
rename from src/runtime-rs/crates/hypervisor/src/device/network.rs
rename to src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs
index 6c13a9ca1..18b983039 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/network.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs
@@ -6,6 +6,7 @@
 
 use std::fmt;
 
+#[derive(Clone)]
 pub struct Address(pub [u8; 6]);
 
 impl fmt::Debug for Address {
@@ -19,7 +20,7 @@ impl fmt::Debug for Address {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct NetworkConfig {
     /// Unique identifier of the device
     pub id: String,
diff --git a/src/runtime-rs/crates/hypervisor/src/device/vsock.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs
similarity index 100%
rename from src/runtime-rs/crates/hypervisor/src/device/vsock.rs
rename to src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs
diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs
index bbd14fb1d..131c7482c 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs
@@ -1,37 +1,33 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
+// Copyright (c) 2019-2023 Alibaba Cloud
+// Copyright (c) 2019-2023 Ant Group
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 
-mod block;
-pub use block::BlockConfig;
-mod network;
-pub use network::{Address, NetworkConfig};
-mod share_fs_device;
-pub use share_fs_device::ShareFsDeviceConfig;
-mod vfio;
-pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig};
-mod share_fs_mount;
-pub use share_fs_mount::{ShareFsMountConfig, ShareFsMountType, ShareFsOperation};
-mod vsock;
-pub use vsock::{HybridVsockConfig, VsockConfig};
+use crate::{DeviceConfig, Hypervisor as hypervisor};
+use anyhow::Result;
+use async_trait::async_trait;
 
-use std::fmt;
+pub mod device_manager;
+pub mod driver;
+pub mod util;
 
-#[derive(Debug)]
-pub enum Device {
-    Block(BlockConfig),
-    Network(NetworkConfig),
-    ShareFsDevice(ShareFsDeviceConfig),
-    Vfio(VfioConfig),
-    ShareFsMount(ShareFsMountConfig),
-    Vsock(VsockConfig),
-    HybridVsock(HybridVsockConfig),
-}
-
-impl fmt::Display for Device {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
+#[async_trait]
+pub trait Device: Send + Sync {
+    // attach is to plug device into VM
+    async fn attach(&mut self, h: &dyn hypervisor) -> Result<()>;
+    // detach is to unplug device from VM
+    async fn detach(&mut self, h: &dyn hypervisor) -> Result<Option<u64>>;
+    // get_device_info returns device config
+    async fn get_device_info(&self) -> DeviceConfig;
+    // increase_attach_count is used to increase the attach count for a device
+    // return values:
+    // * true: no need to do real attach when current attach count is zero, skip following actions.
+    // * err error: error while do increase attach count
+    async fn increase_attach_count(&mut self) -> Result<bool>;
+    // decrease_attach_count is used to decrease the attach count for a device
+    // return values:
+    // * false: no need to do real dettach when current attach count is not zero, skip following actions.
+    // * err error: error while do decrease attach count
+    async fn decrease_attach_count(&mut self) -> Result<bool>;
 }
diff --git a/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs b/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs
deleted file mode 100644
index 4bf73eab7..000000000
--- a/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) 2019-2022 Alibaba Cloud
-// Copyright (c) 2019-2022 Ant Group
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-/// ShareFsDeviceConfig: share fs device config
-#[derive(Debug)]
-pub struct ShareFsDeviceConfig {
-    /// fs_type: virtiofs or inline-virtiofs
-    pub fs_type: String,
-
-    /// socket_path: socket path for virtiofs
-    pub sock_path: String,
-
-    /// mount_tag: a label used as a hint to the guest.
-    pub mount_tag: String,
-
-    /// host_path: the host filesystem path for this volume.
-    pub host_path: String,
-
-    /// queue_size: queue size
-    pub queue_size: u64,
-
-    /// queue_num: queue number
-    pub queue_num: u64,
-}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/util.rs b/src/runtime-rs/crates/hypervisor/src/device/util.rs
new file mode 100644
index 000000000..3aa5f7b0a
--- /dev/null
+++ b/src/runtime-rs/crates/hypervisor/src/device/util.rs
@@ -0,0 +1,88 @@
+// Copyright (c) 2019-2023 Alibaba Cloud
+// Copyright (c) 2019-2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use ini::Ini;
+
+const SYS_DEV_PREFIX: &str = "/sys/dev";
+
+// get_host_path is used to fetch the host path for the device.
+// The path passed in the spec refers to the path that should appear inside the container.
+// We need to find the actual device path on the host based on the major-minor numbers of the device.
+pub fn get_host_path(dev_type: String, major: i64, minor: i64) -> Result<String> {
+    let path_comp = match dev_type.as_str() {
+        "c" | "u" => "char",
+        "b" => "block",
+        // for device type p will return an empty string
+        _ => return Ok(String::new()),
+    };
+    let format = format!("{}:{}", major, minor);
+    let sys_dev_path = std::path::Path::new(SYS_DEV_PREFIX)
+        .join(path_comp)
+        .join(format)
+        .join("uevent");
+    std::fs::metadata(&sys_dev_path)?;
+    let conf = Ini::load_from_file(&sys_dev_path)?;
+    let dev_name = conf
+        .section::<String>(None)
+        .ok_or_else(|| anyhow!("has no section"))?
+        .get("DEVNAME")
+        .ok_or_else(|| anyhow!("has no DEVNAME"))?;
+    Ok(format!("/dev/{}", dev_name))
+}
+
+// get_virt_drive_name returns the disk name format for virtio-blk
+// Reference: https://github.com/torvalds/linux/blob/master/drivers/block/virtio_blk.c @c0aa3e0916d7e531e69b02e426f7162dfb1c6c0
+pub(crate) fn get_virt_drive_name(mut index: i32) -> Result<String> {
+    if index < 0 {
+        return Err(anyhow!("Index cannot be negative"));
+    }
+
+    // Prefix used for virtio-block devices
+    const PREFIX: &str = "vd";
+
+    // Refer to DISK_NAME_LEN: https://github.com/torvalds/linux/blob/08c521a2011ff492490aa9ed6cc574be4235ce2b/include/linux/genhd.h#L61
+    let disk_name_len = 32usize;
+    let base = 26i32;
+
+    let suff_len = disk_name_len - PREFIX.len();
+    let mut disk_letters = vec![0u8; suff_len];
+
+    let mut i = 0usize;
+    while i < suff_len && index >= 0 {
+        let letter: u8 = b'a' + (index % base) as u8;
+        disk_letters[i] = letter;
+        index = (index / base) - 1;
+        i += 1;
+    }
+    if index >= 0 {
+        return Err(anyhow!("Index not supported"));
+    }
+    disk_letters.truncate(i);
+    disk_letters.reverse();
+    Ok(String::from(PREFIX) + std::str::from_utf8(&disk_letters)?)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::device::util::get_virt_drive_name;
+
+    #[actix_rt::test]
+    async fn test_get_virt_drive_name() {
+        for &(input, output) in [
+            (0i32, "vda"),
+            (25, "vdz"),
+            (27, "vdab"),
+            (704, "vdaac"),
+            (18277, "vdzzz"),
+        ]
+        .iter()
+        {
+            let out = get_virt_drive_name(input).unwrap();
+            assert_eq!(&out, output);
+        }
+    }
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
index d2d2cd86d..710a0e50d 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
@@ -6,8 +6,8 @@
 
 use super::vmm_instance::VmmInstance;
 use crate::{
-    device::Device, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
-    DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
+    driver::DeviceConfig, hypervisor_persist::HypervisorState, kernel_param::KernelParams,
+    VmmState, DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
 };
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
@@ -56,7 +56,7 @@ pub struct DragonballInner {
     pub(crate) run_dir: String,
 
     /// pending device
-    pub(crate) pending_devices: Vec<Device>,
+    pub(crate) pending_devices: Vec<DeviceConfig>,
 
     /// cached block device
     pub(crate) cached_block_devices: HashSet<String>,
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
index 48d9a3508..966202d79 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
@@ -15,8 +15,8 @@ use dragonball::api::v1::{
 
 use super::DragonballInner;
 use crate::{
-    device::Device, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig,
-    ShareFsMountType, ShareFsOperation, VmmState,
+    driver::DeviceConfig, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig,
+    ShareFsMountConfig, ShareFsMountType, ShareFsOperation, VmmState,
 };
 
 const MB_TO_B: u32 = 1024 * 1024;
@@ -31,7 +31,7 @@ pub(crate) fn drive_index_to_id(index: u64) -> String {
 }
 
 impl DragonballInner {
-    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
         if self.state == VmmState::NotReady {
             info!(sl!(), "VMM not ready, queueing device {}", device);
 
@@ -44,11 +44,11 @@ impl DragonballInner {
 
         info!(sl!(), "dragonball add device {:?}", &device);
         match device {
-            Device::Network(config) => self.add_net_device(&config).context("add net device"),
-            Device::Vfio(_config) => {
+            DeviceConfig::Network(config) => self.add_net_device(&config).context("add net device"),
+            DeviceConfig::Vfio(_config) => {
                 todo!()
             }
-            Device::Block(config) => self
+            DeviceConfig::Block(config) => self
                 .add_block_device(
                     config.path_on_host.as_str(),
                     config.id.as_str(),
@@ -56,29 +56,29 @@ impl DragonballInner {
                     config.no_drop,
                 )
                 .context("add block device"),
-            Device::HybridVsock(config) => self.add_hvsock(&config).context("add vsock"),
-            Device::ShareFsDevice(config) => self
+            DeviceConfig::HybridVsock(config) => self.add_hvsock(&config).context("add vsock"),
+            DeviceConfig::ShareFsDevice(config) => self
                 .add_share_fs_device(&config)
                 .context("add share fs device"),
-            Device::ShareFsMount(config) => self
+            DeviceConfig::ShareFsMount(config) => self
                 .add_share_fs_mount(&config)
                 .context("add share fs mount"),
-            Device::Vsock(_) => {
+            DeviceConfig::Vsock(_) => {
                 todo!()
             }
         }
     }
 
-    pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, device: DeviceConfig) -> Result<()> {
         info!(sl!(), "remove device {} ", device);
 
         match device {
-            Device::Block(config) => {
+            DeviceConfig::Block(config) => {
                 let drive_id = drive_index_to_id(config.index);
                 self.remove_block_drive(drive_id.as_str())
                     .context("remove block drive")
             }
-            Device::Vfio(_config) => {
+            DeviceConfig::Vfio(_config) => {
                 todo!()
             }
             _ => Err(anyhow!("unsupported device {:?}", device)),
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
index adc016807..10640216f 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
@@ -32,7 +32,7 @@ impl DragonballInner {
 
         // prepare vsock
         let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/");
-        let d = crate::device::Device::HybridVsock(crate::device::HybridVsockConfig {
+        let d = crate::driver::DeviceConfig::HybridVsock(crate::driver::HybridVsockConfig {
             id: format!("vsock-{}", &self.id),
             guest_cid: 3,
             uds_path,
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
index 951af2bc7..c187911b4 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
@@ -20,7 +20,7 @@ use kata_types::capabilities::Capabilities;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use tokio::sync::RwLock;
 
-use crate::{device::Device, Hypervisor, VcpuThreadIds};
+use crate::{driver::DeviceConfig, Hypervisor, VcpuThreadIds};
 
 pub struct Dragonball {
     inner: Arc<RwLock<DragonballInner>>,
@@ -77,12 +77,12 @@ impl Hypervisor for Dragonball {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: Device) -> Result<()> {
+    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: Device) -> Result<()> {
+    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs
index 364cce0f2..a6e5566bb 100644
--- a/src/runtime-rs/crates/hypervisor/src/lib.rs
+++ b/src/runtime-rs/crates/hypervisor/src/lib.rs
@@ -11,7 +11,8 @@ logging::logger_with_subsystem!(sl, "hypervisor");
 
 pub mod device;
 pub mod hypervisor_persist;
-pub use device::*;
+use device::driver;
+pub use device::driver::*;
 pub mod dragonball;
 mod kernel_param;
 pub mod qemu;
@@ -78,8 +79,8 @@ pub trait Hypervisor: Send + Sync {
     async fn resume_vm(&self) -> Result<()>;
 
     // device manager
-    async fn add_device(&self, device: device::Device) -> Result<()>;
-    async fn remove_device(&self, device: device::Device) -> Result<()>;
+    async fn add_device(&self, device: driver::DeviceConfig) -> Result<()>;
+    async fn remove_device(&self, device: driver::DeviceConfig) -> Result<()>;
 
     // utils
     async fn get_agent_socket(&self) -> Result<String>;
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
index 0c0efbde6..8043fd3ba 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
@@ -133,16 +133,16 @@ impl QemuInner {
     }
 }
 
-use crate::device::Device;
+use crate::driver::DeviceConfig;
 
 // device manager part of Hypervisor
 impl QemuInner {
-    pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
         info!(sl!(), "QemuInner::add_device() {}", device);
         todo!()
     }
 
-    pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, device: DeviceConfig) -> Result<()> {
         info!(sl!(), "QemuInner::remove_device() {} ", device);
         todo!()
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
index c73fb23d4..4f638cfae 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
@@ -5,7 +5,7 @@
 
 mod inner;
 
-use crate::device::Device;
+use crate::driver::DeviceConfig;
 use crate::hypervisor_persist::HypervisorState;
 use crate::Hypervisor;
 use crate::{HypervisorConfig, VcpuThreadIds};
@@ -73,12 +73,12 @@ impl Hypervisor for Qemu {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: Device) -> Result<()> {
+    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: Device) -> Result<()> {
+    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
index eebabe59c..cea445e2f 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
@@ -13,7 +13,7 @@ use async_trait::async_trait;
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 
 // IPVlanEndpoint is the endpoint bridged to VM
 #[derive(Debug)]
@@ -67,7 +67,7 @@ impl Endpoint for IPVlanEndpoint {
             .await
             .context("error adding network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("error adding device by hypervisor")?;
 
@@ -82,7 +82,7 @@ impl Endpoint for IPVlanEndpoint {
         let config = self
             .get_network_config()
             .context("error getting network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("error removing device by hypervisor")?;
 
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
index 364a5792b..7f31c028b 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
@@ -11,7 +11,7 @@ use super::Endpoint;
 use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 
 #[derive(Debug)]
 pub struct MacVlanEndpoint {
@@ -64,7 +64,7 @@ impl Endpoint for MacVlanEndpoint {
             .await
             .context("add network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("Error add device")?;
         Ok(())
@@ -76,7 +76,7 @@ impl Endpoint for MacVlanEndpoint {
             .await
             .context("del network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("remove device")?;
         Ok(())
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
index 1be181c0d..9f3ccaf90 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
@@ -8,7 +8,7 @@ use std::path::Path;
 
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device, Hypervisor};
+use hypervisor::{device::driver, Hypervisor};
 
 use super::endpoint_persist::{EndpointState, PhysicalEndpointState};
 use super::Endpoint;
@@ -94,7 +94,7 @@ impl Endpoint for PhysicalEndpoint {
 
     async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()> {
         // bind physical interface from host driver and bind to vfio
-        device::bind_device_to_vfio(
+        driver::bind_device_to_vfio(
             &self.bdf,
             &self.driver,
             &self.vendor_device_id.vendor_device_id(),
@@ -108,11 +108,11 @@ impl Endpoint for PhysicalEndpoint {
         };
 
         // add vfio device
-        let d = device::Device::Vfio(device::VfioConfig {
+        let d = driver::DeviceConfig::Vfio(driver::VfioConfig {
             id: format!("physical_nic_{}", self.name().await),
             sysfs_path: "".to_string(),
             bus_slot_func: self.bdf.clone(),
-            mode: device::VfioBusMode::new(mode)
+            mode: driver::VfioBusMode::new(mode)
                 .with_context(|| format!("new vfio bus mode {:?}", mode))?,
         });
         hypervisor.add_device(d).await.context("add device")?;
@@ -128,7 +128,7 @@ impl Endpoint for PhysicalEndpoint {
 
         // we do not need to enter the network namespace to bind back the
         // physical interface to host driver.
-        device::bind_device_to_host(
+        driver::bind_device_to_host(
             &self.bdf,
             &self.driver,
             &self.vendor_device_id.vendor_device_id(),
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
index 38ce2e335..7e822f764 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
@@ -11,7 +11,7 @@ use super::Endpoint;
 use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 
 #[derive(Debug)]
 pub struct VethEndpoint {
@@ -64,7 +64,7 @@ impl Endpoint for VethEndpoint {
             .await
             .context("add network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("Error add device")?;
         Ok(())
@@ -76,7 +76,7 @@ impl Endpoint for VethEndpoint {
             .await
             .context("del network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("remove device")?;
         Ok(())
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
index 8a90f4a75..7ece7d168 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
@@ -13,7 +13,7 @@ use super::endpoint_persist::{EndpointState, VlanEndpointState};
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
-use hypervisor::{device::NetworkConfig, Device, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
 #[derive(Debug)]
 pub struct VlanEndpoint {
     pub(crate) net_pair: NetworkPair,
@@ -64,7 +64,7 @@ impl Endpoint for VlanEndpoint {
             .await
             .context("error adding network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(Device::Network(config))
+        h.add_device(DeviceConfig::Network(config))
             .await
             .context("error adding device by hypervisor")?;
 
@@ -79,7 +79,7 @@ impl Endpoint for VlanEndpoint {
         let config = self
             .get_network_config()
             .context("error getting network config")?;
-        h.remove_device(Device::Network(config))
+        h.remove_device(DeviceConfig::Network(config))
             .await
             .context("error removing device by hypervisor")?;
 
diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs
index 4aed2cc19..cebce769a 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs
@@ -8,7 +8,9 @@ use std::path::Path;
 
 use anyhow::{Context, Result};
 use hypervisor::{
-    device::{Device as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation},
+    device::driver::{
+        DeviceConfig as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation,
+    },
     Hypervisor, ShareFsDeviceConfig,
 };
 use kata_sys_util::mount;

From cc9c915384900916a388e5a80e13ca96ed480b4e Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Wed, 10 May 2023 15:16:05 +0800
Subject: [PATCH 133/150] runtime-rs: implement trait for vfio device

add the trait implementation for vfio device,

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 .../hypervisor/src/device/driver/vfio.rs      | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
index fcee1bb8c..4c43be010 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
@@ -6,9 +6,13 @@
 
 use std::{fs, path::Path, process::Command};
 
+use crate::{driver::hypervisor, DeviceConfig};
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use anyhow::anyhow;
 use anyhow::{Context, Result};
+use async_trait::async_trait;
+
+use crate::Device;
 
 fn override_driver(bdf: &str, driver: &str) -> Result<()> {
     let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf);
@@ -145,3 +149,26 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str
 
     Ok(())
 }
+
+#[async_trait]
+impl Device for VfioConfig {
+    async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
+        todo!()
+    }
+
+    async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
+        todo!()
+    }
+
+    async fn get_device_info(&self) -> DeviceConfig {
+        todo!()
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        todo!()
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        todo!()
+    }
+}

From 6e273d6ccc5120258a8f65229b86063ad83f9929 Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Wed, 10 May 2023 15:41:32 +0800
Subject: [PATCH 134/150] runtime-rs: implement trait for vhost-user device

add the trait implementation for vhost-user device

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
---
 .../hypervisor/src/device/driver/mod.rs       |  1 +
 .../src/device/driver/vhost_user.rs           | 53 +++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs

diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
index 984422b7b..71db1e4fd 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
@@ -4,6 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+mod vhost_user;
 mod virtio_blk;
 pub use virtio_blk::{
     BlockConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
new file mode 100644
index 000000000..53d4a0e01
--- /dev/null
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
@@ -0,0 +1,53 @@
+// Copyright (c) 2019-2023 Alibaba Cloud
+// Copyright (c) 2019-2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::Device;
+use crate::{driver::hypervisor, DeviceConfig};
+use anyhow::Result;
+use async_trait::async_trait;
+
+/// VhostUserConfig represents data shared by most vhost-user devices
+pub struct VhostUserConfig {
+    /// Device id
+    pub dev_id: String,
+    /// Socket path
+    pub socket_path: String,
+    /// Mac_address is only meaningful for vhost user net device
+    pub mac_address: String,
+    /// These are only meaningful for vhost user fs devices
+    pub tag: String,
+    pub cache: String,
+    pub device_type: String,
+    /// Pci_addr is the PCI address used to identify the slot at which the drive is attached.
+    pub pci_addr: Option<String>,
+    /// Block index of the device if assigned
+    pub index: u8,
+    pub cache_size: u32,
+    pub queue_siez: u32,
+}
+
+#[async_trait]
+impl Device for VhostUserConfig {
+    async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
+        todo!()
+    }
+
+    async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
+        todo!()
+    }
+
+    async fn get_device_info(&self) -> DeviceConfig {
+        todo!()
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        todo!()
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        todo!()
+    }
+}

From b076d46db37bd76fa9393b1fda3c349f376592ef Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Sat, 15 Oct 2022 14:08:30 +0800
Subject: [PATCH 135/150] agent: handle hotplug virtio-mmio device

As dragonball support hotplug virtio-mmio device, we should handle it in agent

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 src/agent/src/device.rs | 91 +++++++++++++++++++++++++++++++++++++----
 src/agent/src/mount.rs  | 17 +++++---
 2 files changed, 95 insertions(+), 13 deletions(-)

diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs
index 67d35651b..53fe77b1b 100644
--- a/src/agent/src/device.rs
+++ b/src/agent/src/device.rs
@@ -34,7 +34,7 @@ macro_rules! sl {
 }
 
 const VM_ROOTFS: &str = "/";
-
+const BLOCK: &str = "block";
 pub const DRIVER_9P_TYPE: &str = "9p";
 pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs";
 pub const DRIVER_BLK_TYPE: &str = "blk";
@@ -204,7 +204,7 @@ impl ScsiBlockMatcher {
 
 impl UeventMatcher for ScsiBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+        uev.subsystem == BLOCK && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
     }
 }
 
@@ -238,7 +238,7 @@ impl VirtioBlkPciMatcher {
 
 impl UeventMatcher for VirtioBlkPciMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+        uev.subsystem == BLOCK && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
     }
 }
 
@@ -311,7 +311,7 @@ impl PmemBlockMatcher {
 
 impl UeventMatcher for PmemBlockMatcher {
     fn is_match(&self, uev: &Uevent) -> bool {
-        uev.subsystem == "block"
+        uev.subsystem == BLOCK
             && uev.devpath.starts_with(ACPI_DEV_PATH)
             && uev.devpath.ends_with(&self.suffix)
             && !uev.devname.is_empty()
@@ -441,6 +441,48 @@ async fn wait_for_ap_device(sandbox: &Arc<Mutex<Sandbox>>, address: ap::Address)
     Ok(())
 }
 
+#[derive(Debug)]
+struct MmioBlockMatcher {
+    suffix: String,
+}
+
+impl MmioBlockMatcher {
+    fn new(devname: &str) -> MmioBlockMatcher {
+        MmioBlockMatcher {
+            suffix: format!(r"/block/{}", devname),
+        }
+    }
+}
+
+impl UeventMatcher for MmioBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == BLOCK && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty()
+    }
+}
+
+#[instrument]
+pub async fn get_virtio_mmio_device_name(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devpath: &str,
+) -> Result<()> {
+    let devname = devpath
+        .strip_prefix("/dev/")
+        .ok_or_else(|| anyhow!("Storage source '{}' must start with /dev/", devpath))?;
+
+    let matcher = MmioBlockMatcher::new(devname);
+    let uev = wait_for_uevent(sandbox, matcher)
+        .await
+        .context("failed to wait for uevent")?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for mmio device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
+}
+
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
 #[instrument]
 fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
@@ -676,12 +718,18 @@ pub fn update_env_pci(
 #[instrument]
 async fn virtiommio_blk_device_handler(
     device: &Device,
-    _sandbox: &Arc<Mutex<Sandbox>>,
+    sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<SpecUpdate> {
     if device.vm_path.is_empty() {
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
+    if !Path::new(&device.vm_path).exists() {
+        get_virtio_mmio_device_name(sandbox, &device.vm_path.to_string())
+            .await
+            .context("failed to get mmio device name")?;
+    }
+
     Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
 }
 
@@ -1394,7 +1442,7 @@ mod tests {
 
         let mut uev = crate::uevent::Uevent::default();
         uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev.subsystem = "block".to_string();
+        uev.subsystem = BLOCK.to_string();
         uev.devpath = devpath.clone();
         uev.devname = devname.to_string();
 
@@ -1428,7 +1476,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let relpath_a = "/0000:00:0a.0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = "block".to_string();
+        uev_a.subsystem = BLOCK.to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
         let matcher_a = VirtioBlkPciMatcher::new(relpath_a);
@@ -1512,7 +1560,7 @@ mod tests {
         let mut uev_a = crate::uevent::Uevent::default();
         let addr_a = "0:0";
         uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
-        uev_a.subsystem = "block".to_string();
+        uev_a.subsystem = BLOCK.to_string();
         uev_a.devname = devname.to_string();
         uev_a.devpath = format!(
             "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
@@ -1555,6 +1603,33 @@ mod tests {
         assert!(!matcher_a.is_match(&uev_b));
     }
 
+    #[tokio::test]
+    async fn test_mmio_block_matcher() {
+        let devname_a = "vda";
+        let devname_b = "vdb";
+        let mut uev_a = crate::uevent::Uevent::default();
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = BLOCK.to_string();
+        uev_a.devname = devname_a.to_string();
+        uev_a.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.0/virtio0/block/{}",
+            devname_a
+        );
+        let matcher_a = MmioBlockMatcher::new(devname_a);
+
+        let mut uev_b = uev_a.clone();
+        uev_b.devpath = format!(
+            "/sys/devices/virtio-mmio-cmdline/virtio-mmio.4/virtio4/block/{}",
+            devname_b
+        );
+        let matcher_b = MmioBlockMatcher::new(devname_b);
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
     #[test]
     fn test_split_vfio_pci_option() {
         assert_eq!(
diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs
index 0eff266f2..5b0d95c19 100644
--- a/src/agent/src/mount.rs
+++ b/src/agent/src/mount.rs
@@ -21,10 +21,11 @@ use nix::unistd::{Gid, Uid};
 use regex::Regex;
 
 use crate::device::{
-    get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
-    DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE,
-    DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
-    DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB,
+    get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name,
+    online_device, wait_for_pmem_device, DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE,
+    DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE,
+    DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE,
+    FS_TYPE_HUGETLB,
 };
 use crate::linux_abi::*;
 use crate::pci;
@@ -473,8 +474,14 @@ async fn virtiommio_blk_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
+    let storage = storage.clone();
+    if !Path::new(&storage.source).exists() {
+        get_virtio_mmio_device_name(&sandbox, &storage.source)
+            .await
+            .context("failed to get mmio device name")?;
+    }
     //The source path is VmPath
-    common_storage_handler(logger, storage)
+    common_storage_handler(logger, &storage)
 }
 
 // virtiofs_storage_handler handles the storage for virtio-fs.

From a8bfac90b13a83133a4a50461c00e2ee290ae558 Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Wed, 10 May 2023 17:25:27 +0800
Subject: [PATCH 136/150] runtime-rs: support block rootfs

support devmapper for block rootfs

Fixes: #5375

Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 .../hypervisor/src/device/device_manager.rs   |  21 ++-
 .../hypervisor/src/device/driver/vfio.rs      |   9 +-
 .../src/device/driver/vhost_user.rs           |   8 +-
 .../crates/resource/src/manager_inner.rs      |  15 ++-
 .../resource/src/rootfs/block_rootfs.rs       | 126 ++++++++++++++++++
 .../crates/resource/src/rootfs/mod.rs         |  46 ++++---
 .../resource/src/rootfs/nydus_rootfs.rs       |   4 +
 .../resource/src/rootfs/share_fs_rootfs.rs    |   7 +-
 .../crates/resource/src/share_fs/mod.rs       |   4 +-
 .../crates/resource/src/share_fs/utils.rs     |   2 +-
 10 files changed, 204 insertions(+), 38 deletions(-)
 create mode 100644 src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs

diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
index 8d6d06652..cdce8956b 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
@@ -66,7 +66,7 @@ pub struct DeviceManager {
 }
 
 impl DeviceManager {
-    pub async fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
+    pub fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
         let devices = HashMap::<String, ArcMutexBoxDevice>::new();
         Ok(DeviceManager {
             devices,
@@ -75,7 +75,7 @@ impl DeviceManager {
         })
     }
 
-    pub async fn new_device(&mut self, device_config: DeviceConfig) -> Result<String> {
+    pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
         let device_id = if let Some(dev) = self.find_device(&device_config).await {
             dev
         } else {
@@ -86,11 +86,10 @@ impl DeviceManager {
         Ok(device_id)
     }
 
-    pub async fn try_add_device(&mut self, device_id: String) -> Result<()> {
-        // get device
+    pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> {
         let device = self
             .devices
-            .get(&device_id)
+            .get(device_id)
             .context("failed to find device")?;
         // attach device
         let result = device.lock().await.attach(self.hypervisor.as_ref()).await;
@@ -99,12 +98,22 @@ impl DeviceManager {
             if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await {
                 self.shared_info.release_device_index(config.index);
             };
-            self.devices.remove(&device_id);
+            self.devices.remove(device_id);
             return Err(e);
         }
         Ok(())
     }
 
+    pub async fn get_device_info(&self, device_id: &str) -> Result<DeviceConfig> {
+        if let Some(dev) = self.devices.get(device_id) {
+            return Ok(dev.lock().await.get_device_info().await);
+        }
+        Err(anyhow!(
+            "device with specified ID hasn't been created. {}",
+            device_id
+        ))
+    }
+
     async fn find_device(&self, device_config: &DeviceConfig) -> Option<String> {
         for (device_id, dev) in &self.devices {
             match dev.lock().await.get_device_info().await {
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
index 4c43be010..0366257d1 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
@@ -6,14 +6,13 @@
 
 use std::{fs, path::Path, process::Command};
 
-use crate::{driver::hypervisor, DeviceConfig};
+use crate::Hypervisor as hypervisor;
+use crate::{device::Device, DeviceConfig};
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use anyhow::anyhow;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 
-use crate::Device;
-
 fn override_driver(bdf: &str, driver: &str) -> Result<()> {
     let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf);
     fs::write(&driver_override, driver)
@@ -152,11 +151,11 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str
 
 #[async_trait]
 impl Device for VfioConfig {
-    async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
+    async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
         todo!()
     }
 
-    async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
+    async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
         todo!()
     }
 
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
index 53d4a0e01..bfde23239 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
@@ -4,8 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::Device;
-use crate::{driver::hypervisor, DeviceConfig};
+use crate::Hypervisor as hypervisor;
+use crate::{device::Device, DeviceConfig};
 use anyhow::Result;
 use async_trait::async_trait;
 
@@ -31,11 +31,11 @@ pub struct VhostUserConfig {
 
 #[async_trait]
 impl Device for VhostUserConfig {
-    async fn attach(&self, _h: &dyn hypervisor) -> Result<()> {
+    async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
         todo!()
     }
 
-    async fn detach(&self, _h: &dyn hypervisor) -> Result<u64> {
+    async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
         todo!()
     }
 
diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs
index 6c6e4067a..16f88063b 100644
--- a/src/runtime-rs/crates/resource/src/manager_inner.rs
+++ b/src/runtime-rs/crates/resource/src/manager_inner.rs
@@ -10,12 +10,12 @@ use crate::{network::NetworkConfig, resource_persist::ResourceState};
 use agent::{Agent, Storage};
 use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
 use oci::LinuxResources;
 use persist::sandbox_persist::Persist;
-use tokio::runtime;
+use tokio::{runtime, sync::RwLock};
 
 use crate::{
     cgroups::{CgroupArgs, CgroupsResource},
@@ -32,6 +32,7 @@ pub(crate) struct ResourceManagerInner {
     toml_config: Arc<TomlConfig>,
     agent: Arc<dyn Agent>,
     hypervisor: Arc<dyn Hypervisor>,
+    device_manager: Arc<RwLock<DeviceManager>>,
     network: Option<Arc<dyn Network>>,
     share_fs: Option<Arc<dyn ShareFs>>,
 
@@ -48,11 +49,17 @@ impl ResourceManagerInner {
         toml_config: Arc<TomlConfig>,
     ) -> Result<Self> {
         let cgroups_resource = CgroupsResource::new(sid, &toml_config)?;
+
+        // create device manager
+        let dev_manager =
+            DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?;
+
         Ok(Self {
             sid: sid.to_string(),
             toml_config,
             agent,
             hypervisor,
+            device_manager: Arc::new(RwLock::new(dev_manager)),
             network: None,
             share_fs: None,
             rootfs_resource: RootFsResource::new(),
@@ -212,6 +219,7 @@ impl ResourceManagerInner {
         self.rootfs_resource
             .handler_rootfs(
                 &self.share_fs,
+                self.device_manager.as_ref(),
                 self.hypervisor.as_ref(),
                 &self.sid,
                 cid,
@@ -298,7 +306,8 @@ impl Persist for ResourceManagerInner {
         Ok(Self {
             sid: resource_args.sid,
             agent: resource_args.agent,
-            hypervisor: resource_args.hypervisor,
+            hypervisor: resource_args.hypervisor.clone(),
+            device_manager: Arc::new(RwLock::new(DeviceManager::new(resource_args.hypervisor)?)),
             network: None,
             share_fs: None,
             rootfs_resource: RootFsResource::new(),
diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
new file mode 100644
index 000000000..56a2641da
--- /dev/null
+++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
@@ -0,0 +1,126 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use super::{Rootfs, ROOTFS};
+use crate::share_fs::{do_get_guest_path, do_get_host_path};
+use agent::Storage;
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig};
+use kata_types::mount::Mount;
+use nix::sys::stat::{self, SFlag};
+use std::fs;
+use tokio::sync::RwLock;
+
+pub(crate) struct BlockRootfs {
+    guest_path: String,
+    device_id: String,
+    mount: oci::Mount,
+    storage: Option<agent::Storage>,
+}
+
+impl BlockRootfs {
+    pub async fn new(
+        d: &RwLock<DeviceManager>,
+        sid: &str,
+        cid: &str,
+        dev_id: u64,
+        rootfs: &Mount,
+    ) -> Result<Self> {
+        let container_path = do_get_guest_path(ROOTFS, cid, false, false);
+        let host_path = do_get_host_path(ROOTFS, sid, cid, false, false);
+        // Create rootfs dir on host to make sure mount point in guest exists, as readonly dir is
+        // shared to guest via virtiofs, and guest is unable to create rootfs dir.
+        fs::create_dir_all(&host_path)
+            .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?;
+
+        let block_device_config = &mut BlockConfig {
+            major: stat::major(dev_id) as i64,
+            minor: stat::minor(dev_id) as i64,
+            ..Default::default()
+        };
+
+        let device_id = d
+            .write()
+            .await
+            .new_device(&DeviceConfig::Block(block_device_config.clone()))
+            .await
+            .context("failed to create deviec")?;
+
+        d.write()
+            .await
+            .try_add_device(device_id.as_str())
+            .await
+            .context("failed to add deivce")?;
+
+        let mut storage = Storage {
+            fs_type: rootfs.fs_type.clone(),
+            mount_point: container_path.clone(),
+            options: rootfs.options.clone(),
+            ..Default::default()
+        };
+
+        // get complete device information
+        let dev_info = d
+            .read()
+            .await
+            .get_device_info(device_id.as_str())
+            .await
+            .context("failed to get device info")?;
+
+        if let DeviceConfig::Block(config) = dev_info {
+            storage.driver = config.driver_option;
+            storage.source = config.virt_path;
+        }
+
+        Ok(Self {
+            guest_path: container_path.clone(),
+            device_id,
+            mount: oci::Mount {
+                ..Default::default()
+            },
+            storage: Some(storage),
+        })
+    }
+}
+
+#[async_trait]
+impl Rootfs for BlockRootfs {
+    async fn get_guest_rootfs_path(&self) -> Result<String> {
+        Ok(self.guest_path.clone())
+    }
+
+    async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>> {
+        Ok(vec![self.mount.clone()])
+    }
+
+    async fn get_storage(&self) -> Option<Storage> {
+        self.storage.clone()
+    }
+
+    async fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(Some(self.device_id.clone()))
+    }
+    async fn cleanup(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+pub(crate) fn is_block_rootfs(file: &str) -> Option<u64> {
+    if file.is_empty() {
+        return None;
+    }
+    match stat::stat(file) {
+        Ok(fstat) => {
+            if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK {
+                let dev_id = fstat.st_rdev;
+                return Some(dev_id);
+            }
+        }
+        Err(_) => return None,
+    };
+    None
+}
diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs
index 2c9160cec..06487fc43 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs
@@ -6,18 +6,18 @@
 
 mod nydus_rootfs;
 mod share_fs_rootfs;
-
 use agent::Storage;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
 use kata_types::mount::Mount;
+mod block_rootfs;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use std::{sync::Arc, vec::Vec};
 use tokio::sync::RwLock;
 
 use crate::share_fs::ShareFs;
 
-use self::nydus_rootfs::NYDUS_ROOTFS_TYPE;
+use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE};
 
 const ROOTFS: &str = "rootfs";
 const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower";
@@ -28,6 +28,7 @@ pub trait Rootfs: Send + Sync {
     async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>>;
     async fn get_storage(&self) -> Option<Storage>;
     async fn cleanup(&self) -> Result<()>;
+    async fn get_device_id(&self) -> Result<Option<String>>;
 }
 
 #[derive(Default)]
@@ -56,7 +57,8 @@ impl RootFsResource {
     pub async fn handler_rootfs(
         &self,
         share_fs: &Option<Arc<dyn ShareFs>>,
-        hypervisor: &dyn Hypervisor,
+        device_manager: &RwLock<DeviceManager>,
+        h: &dyn Hypervisor,
         sid: &str,
         cid: &str,
         root: &oci::Root,
@@ -67,7 +69,7 @@ impl RootFsResource {
             // if rootfs_mounts is empty
             mounts_vec if mounts_vec.is_empty() => {
                 if let Some(share_fs) = share_fs {
-                    // share fs rootfs
+                    // handle share fs rootfs
                     Ok(Arc::new(
                         share_fs_rootfs::ShareFsRootfs::new(
                             share_fs,
@@ -85,16 +87,27 @@ impl RootFsResource {
             mounts_vec if is_single_layer_rootfs(mounts_vec) => {
                 // Safe as single_layer_rootfs must have one layer
                 let layer = &mounts_vec[0];
-                let rootfs: Arc<dyn Rootfs> = if let Some(share_fs) = share_fs {
-                    // nydus rootfs
-                    if layer.fs_type == NYDUS_ROOTFS_TYPE {
+                let mut inner = self.inner.write().await;
+                let rootfs = if let Some(dev_id) = is_block_rootfs(&layer.source) {
+                    // handle block rootfs
+                    info!(sl!(), "block device: {}", dev_id);
+                    let block_rootfs: Arc<dyn Rootfs> = Arc::new(
+                        block_rootfs::BlockRootfs::new(device_manager, sid, cid, dev_id, layer)
+                            .await
+                            .context("new block rootfs")?,
+                    );
+                    Ok(block_rootfs)
+                } else if let Some(share_fs) = share_fs {
+                    // handle nydus rootfs
+                    let share_rootfs: Arc<dyn Rootfs> = if layer.fs_type == NYDUS_ROOTFS_TYPE {
                         Arc::new(
-                            nydus_rootfs::NydusRootfs::new(share_fs, hypervisor, sid, cid, layer)
+                            nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer)
                                 .await
                                 .context("new nydus rootfs")?,
                         )
-                    } else {
-                        // share fs rootfs
+                    }
+                    // handle sharefs rootfs
+                    else {
                         Arc::new(
                             share_fs_rootfs::ShareFsRootfs::new(
                                 share_fs,
@@ -105,13 +118,12 @@ impl RootFsResource {
                             .await
                             .context("new share fs rootfs")?,
                         )
-                    }
+                    };
+                    Ok(share_rootfs)
                 } else {
-                    return Err(anyhow!("unsupported rootfs {:?}", &layer));
-                };
-
-                let mut inner = self.inner.write().await;
-                inner.rootfs.push(Arc::clone(&rootfs));
+                    Err(anyhow!("unsupported rootfs {:?}", &layer))
+                }?;
+                inner.rootfs.push(rootfs.clone());
                 Ok(rootfs)
             }
             _ => Err(anyhow!(
diff --git a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs
index 008443b87..b8f9ed100 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs
@@ -150,6 +150,10 @@ impl Rootfs for NydusRootfs {
         Some(self.rootfs.clone())
     }
 
+    async fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
+
     async fn cleanup(&self) -> Result<()> {
         // TODO: Clean up NydusRootfs after the container is killed
         warn!(sl!(), "Cleaning up NydusRootfs is still unimplemented.");
diff --git a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs
index b5d4136c1..7a490215a 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs
@@ -4,12 +4,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use std::sync::Arc;
+
 use agent::Storage;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_sys_util::mount::{umount_timeout, Mounter};
 use kata_types::mount::Mount;
-use std::sync::Arc;
 
 use super::{Rootfs, ROOTFS};
 use crate::share_fs::{ShareFs, ShareFsRootfsConfig};
@@ -74,6 +75,10 @@ impl Rootfs for ShareFsRootfs {
         None
     }
 
+    async fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
+
     async fn cleanup(&self) -> Result<()> {
         // Umount the mount point shared to guest
         let share_fs_mount = self.share_fs.get_share_fs_mount();
diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs
index 350c7ea71..12bb64420 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs
@@ -12,7 +12,9 @@ mod share_virtio_fs_standalone;
 use share_virtio_fs_standalone::ShareVirtioFsStandalone;
 mod utils;
 use tokio::sync::Mutex;
-pub use utils::{do_get_guest_path, do_get_guest_share_path, get_host_rw_shared_path};
+pub use utils::{
+    do_get_guest_path, do_get_guest_share_path, do_get_host_path, get_host_rw_shared_path,
+};
 mod virtio_fs_share_mount;
 use virtio_fs_share_mount::VirtiofsShareMount;
 pub use virtio_fs_share_mount::EPHEMERAL_PATH;
diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs
index 47f3df533..c93cbec54 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs
@@ -97,7 +97,7 @@ pub fn do_get_guest_share_path(target: &str, cid: &str, is_rafs: bool) -> String
     do_get_guest_any_path(target, cid, false, is_rafs, true)
 }
 
-pub(crate) fn do_get_host_path(
+pub fn do_get_host_path(
     target: &str,
     sid: &str,
     cid: &str,

From fe9ec67644579b8db872311bf79a666312e07d79 Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Thu, 13 Oct 2022 14:24:41 +0800
Subject: [PATCH 137/150] runtime-rs: block volume

support block volume in runtime-rs

Fixes: #5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 .../crates/resource/src/manager_inner.rs      |   8 +-
 .../resource/src/volume/block_volume.rs       | 127 ++++++++++++++++--
 .../resource/src/volume/default_volume.rs     |   4 +
 .../crates/resource/src/volume/hugepage.rs    |   4 +
 .../crates/resource/src/volume/mod.rs         |  20 ++-
 .../resource/src/volume/share_fs_volume.rs    |   7 +-
 .../crates/resource/src/volume/shm_volume.rs  |   4 +
 7 files changed, 156 insertions(+), 18 deletions(-)

diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs
index 16f88063b..e76fa4eb6 100644
--- a/src/runtime-rs/crates/resource/src/manager_inner.rs
+++ b/src/runtime-rs/crates/resource/src/manager_inner.rs
@@ -236,7 +236,13 @@ impl ResourceManagerInner {
         spec: &oci::Spec,
     ) -> Result<Vec<Arc<dyn Volume>>> {
         self.volume_resource
-            .handler_volumes(&self.share_fs, cid, spec)
+            .handler_volumes(
+                &self.share_fs,
+                cid,
+                spec,
+                self.device_manager.as_ref(),
+                &self.sid,
+            )
             .await
     }
 
diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs
index da8ef03f9..74c75207b 100644
--- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs
@@ -6,27 +6,126 @@
 
 use anyhow::Result;
 use async_trait::async_trait;
+use std::{collections::HashMap, fs, path::Path};
 
-use super::Volume;
+use crate::share_fs::{do_get_guest_path, do_get_host_path};
 
+use super::{share_fs_volume::generate_mount_path, Volume};
+use agent::Storage;
+use anyhow::{anyhow, Context};
+use hypervisor::{device::DeviceManager, BlockConfig, DeviceConfig};
+use nix::sys::stat::{self, SFlag};
+use tokio::sync::RwLock;
 #[derive(Debug)]
-pub(crate) struct BlockVolume {}
+pub(crate) struct BlockVolume {
+    storage: Option<agent::Storage>,
+    mount: oci::Mount,
+    device_id: String,
+}
 
 /// BlockVolume: block device volume
 impl BlockVolume {
-    pub(crate) fn new(_m: &oci::Mount) -> Result<Self> {
-        Ok(Self {})
+    pub(crate) async fn new(
+        d: &RwLock<DeviceManager>,
+        m: &oci::Mount,
+        read_only: bool,
+        cid: &str,
+        sid: &str,
+    ) -> Result<Self> {
+        let fstat = stat::stat(m.source.as_str()).context(format!("stat {}", m.source))?;
+        info!(sl!(), "device stat: {:?}", fstat);
+        let mut options = HashMap::new();
+        if read_only {
+            options.insert("read_only".to_string(), "true".to_string());
+        }
+
+        let block_device_config = &mut BlockConfig {
+            major: stat::major(fstat.st_rdev) as i64,
+            minor: stat::minor(fstat.st_rdev) as i64,
+            ..Default::default()
+        };
+
+        let device_id = d
+            .write()
+            .await
+            .new_device(&DeviceConfig::Block(block_device_config.clone()))
+            .await
+            .context("failed to create deviec")?;
+
+        d.write()
+            .await
+            .try_add_device(device_id.as_str())
+            .await
+            .context("failed to add deivce")?;
+
+        let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap();
+        let file_name = generate_mount_path(cid, file_name);
+        let guest_path = do_get_guest_path(&file_name, cid, true, false);
+        let host_path = do_get_host_path(&file_name, sid, cid, true, read_only);
+        fs::create_dir_all(&host_path)
+            .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?;
+
+        // get complete device information
+        let dev_info = d
+            .read()
+            .await
+            .get_device_info(&device_id)
+            .await
+            .context("failed to get device info")?;
+
+        // storage
+        let mut storage = Storage::default();
+
+        if let DeviceConfig::Block(config) = dev_info {
+            storage.driver = config.driver_option;
+            storage.source = config.virt_path;
+        }
+
+        storage.options = if read_only {
+            vec!["ro".to_string()]
+        } else {
+            Vec::new()
+        };
+
+        storage.mount_point = guest_path.clone();
+
+        // If the volume had specified the filesystem type, use it. Otherwise, set it
+        // to ext4 since but right now we only support it.
+        if m.r#type != "bind" {
+            storage.fs_type = m.r#type.clone();
+        } else {
+            storage.fs_type = "ext4".to_string();
+        }
+
+        // mount
+        let mount = oci::Mount {
+            destination: m.destination.clone(),
+            r#type: m.r#type.clone(),
+            source: guest_path.clone(),
+            options: m.options.clone(),
+        };
+
+        Ok(Self {
+            storage: Some(storage),
+            mount,
+            device_id,
+        })
     }
 }
 
 #[async_trait]
 impl Volume for BlockVolume {
-    fn get_volume_mount(&self) -> anyhow::Result<Vec<oci::Mount>> {
-        todo!()
+    fn get_volume_mount(&self) -> Result<Vec<oci::Mount>> {
+        Ok(vec![self.mount.clone()])
     }
 
     fn get_storage(&self) -> Result<Vec<agent::Storage>> {
-        todo!()
+        let s = if let Some(s) = self.storage.as_ref() {
+            vec![s.clone()]
+        } else {
+            vec![]
+        };
+        Ok(s)
     }
 
     async fn cleanup(&self) -> Result<()> {
@@ -34,9 +133,19 @@ impl Volume for BlockVolume {
         warn!(sl!(), "Cleaning up BlockVolume is still unimplemented.");
         Ok(())
     }
+
+    fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(Some(self.device_id.clone()))
+    }
 }
 
-pub(crate) fn is_block_volume(_m: &oci::Mount) -> bool {
-    // attach block device
+pub(crate) fn is_block_volume(m: &oci::Mount) -> bool {
+    if m.r#type != "bind" {
+        return false;
+    }
+    if let Ok(fstat) = stat::stat(m.source.as_str()).context(format!("stat {}", m.source)) {
+        info!(sl!(), "device stat: {:?}", fstat);
+        return SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK;
+    }
     false
 }
diff --git a/src/runtime-rs/crates/resource/src/volume/default_volume.rs b/src/runtime-rs/crates/resource/src/volume/default_volume.rs
index 8855a8e03..f1e6e7983 100644
--- a/src/runtime-rs/crates/resource/src/volume/default_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/default_volume.rs
@@ -38,4 +38,8 @@ impl Volume for DefaultVolume {
         warn!(sl!(), "Cleaning up DefaultVolume is still unimplemented.");
         Ok(())
     }
+
+    fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
 }
diff --git a/src/runtime-rs/crates/resource/src/volume/hugepage.rs b/src/runtime-rs/crates/resource/src/volume/hugepage.rs
index a827b2657..e8e0548f5 100644
--- a/src/runtime-rs/crates/resource/src/volume/hugepage.rs
+++ b/src/runtime-rs/crates/resource/src/volume/hugepage.rs
@@ -91,6 +91,10 @@ impl Volume for Hugepage {
     async fn cleanup(&self) -> Result<()> {
         Ok(())
     }
+
+    fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
 }
 
 pub(crate) fn get_huge_page_option(m: &oci::Mount) -> Result<Option<Vec<String>>> {
diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs
index 2868ddee3..067ad8f65 100644
--- a/src/runtime-rs/crates/resource/src/volume/mod.rs
+++ b/src/runtime-rs/crates/resource/src/volume/mod.rs
@@ -12,10 +12,11 @@ mod shm_volume;
 use async_trait::async_trait;
 
 use anyhow::{Context, Result};
+use hypervisor::device::DeviceManager;
 use std::{sync::Arc, vec::Vec};
 use tokio::sync::RwLock;
 
-use crate::share_fs::ShareFs;
+use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume};
 
 use self::hugepage::{get_huge_page_limits_map, get_huge_page_option};
 
@@ -25,6 +26,7 @@ const BIND: &str = "bind";
 pub trait Volume: Send + Sync {
     fn get_volume_mount(&self) -> Result<Vec<oci::Mount>>;
     fn get_storage(&self) -> Result<Vec<agent::Storage>>;
+    fn get_device_id(&self) -> Result<Option<String>>;
     async fn cleanup(&self) -> Result<()>;
 }
 
@@ -48,20 +50,25 @@ impl VolumeResource {
         share_fs: &Option<Arc<dyn ShareFs>>,
         cid: &str,
         spec: &oci::Spec,
+        d: &RwLock<DeviceManager>,
+        sid: &str,
     ) -> Result<Vec<Arc<dyn Volume>>> {
         let mut volumes: Vec<Arc<dyn Volume>> = vec![];
         let oci_mounts = &spec.mounts;
+        info!(sl!(), " oci mount is : {:?}", oci_mounts.clone());
         // handle mounts
         for m in oci_mounts {
+            let read_only = m.options.iter().any(|opt| opt == "ro");
             let volume: Arc<dyn Volume> = if shm_volume::is_shim_volume(m) {
                 let shm_size = shm_volume::DEFAULT_SHM_SIZE;
                 Arc::new(
                     shm_volume::ShmVolume::new(m, shm_size)
                         .with_context(|| format!("new shm volume {:?}", m))?,
                 )
-            } else if share_fs_volume::is_share_fs_volume(m) {
+            } else if is_block_volume(m) {
+                // handle block volume
                 Arc::new(
-                    share_fs_volume::ShareFsVolume::new(share_fs, m, cid)
+                    block_volume::BlockVolume::new(d, m, read_only, cid, sid)
                         .await
                         .with_context(|| format!("new share fs volume {:?}", m))?,
                 )
@@ -76,10 +83,11 @@ impl VolumeResource {
                     hugepage::Hugepage::new(m, hugepage_limits, options)
                         .with_context(|| format!("handle hugepages {:?}", m))?,
                 )
-            } else if block_volume::is_block_volume(m) {
+            } else if share_fs_volume::is_share_fs_volume(m) {
                 Arc::new(
-                    block_volume::BlockVolume::new(m)
-                        .with_context(|| format!("new block volume {:?}", m))?,
+                    share_fs_volume::ShareFsVolume::new(share_fs, m, cid, read_only)
+                        .await
+                        .with_context(|| format!("new share fs volume {:?}", m))?,
                 )
             } else if is_skip_volume(m) {
                 info!(sl!(), "skip volume {:?}", m);
diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs
index f3f70424d..9b91856ac 100644
--- a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs
@@ -36,6 +36,7 @@ impl ShareFsVolume {
         share_fs: &Option<Arc<dyn ShareFs>>,
         m: &oci::Mount,
         cid: &str,
+        readonly: bool,
     ) -> Result<Self> {
         // The file_name is in the format of "sandbox-{uuid}-{file_name}"
         let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap();
@@ -69,8 +70,6 @@ impl ShareFsVolume {
                 }
             }
             Some(share_fs) => {
-                let readonly = m.options.iter().any(|opt| opt == "ro");
-
                 let share_fs_mount = share_fs.get_share_fs_mount();
                 let mounted_info_set = share_fs.mounted_info_set();
                 let mut mounted_info_set = mounted_info_set.lock().await;
@@ -226,6 +225,10 @@ impl Volume for ShareFsVolume {
 
         Ok(())
     }
+
+    fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
 }
 
 pub(crate) fn is_share_fs_volume(m: &oci::Mount) -> bool {
diff --git a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs
index 5805106d2..574842b4b 100644
--- a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs
@@ -104,6 +104,10 @@ impl Volume for ShmVolume {
         warn!(sl!(), "Cleaning up ShmVolume is still unimplemented.");
         Ok(())
     }
+
+    fn get_device_id(&self) -> Result<Option<String>> {
+        Ok(None)
+    }
 }
 
 pub(crate) fn is_shim_volume(m: &oci::Mount) -> bool {

From f16012a1eb8665e85b08f129b8fffc14d1ee733b Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Thu, 11 May 2023 10:53:07 +0800
Subject: [PATCH 138/150] runtime-rs: support linux device

support linux device in runtime-rs

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 .../hypervisor/src/device/device_manager.rs   |  4 +-
 src/runtime-rs/crates/resource/src/manager.rs |  8 ++-
 .../crates/resource/src/manager_inner.rs      | 63 +++++++++++++++++--
 .../src/container_manager/container.rs        | 11 +++-
 4 files changed, 78 insertions(+), 8 deletions(-)

diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
index cdce8956b..6d9325d39 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
@@ -76,10 +76,10 @@ impl DeviceManager {
     }
 
     pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
-        let device_id = if let Some(dev) = self.find_device(&device_config).await {
+        let device_id = if let Some(dev) = self.find_device(device_config).await {
             dev
         } else {
-            self.create_device(&device_config)
+            self.create_device(device_config)
                 .await
                 .context("failed to create device")?
         };
diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs
index 9514b6013..34753b7d5 100644
--- a/src/runtime-rs/crates/resource/src/manager.rs
+++ b/src/runtime-rs/crates/resource/src/manager.rs
@@ -7,13 +7,14 @@
 use crate::network::NetworkConfig;
 use crate::resource_persist::ResourceState;
 use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig};
+use agent::types::Device;
 use agent::{Agent, Storage};
 use anyhow::Result;
 use async_trait::async_trait;
 use hypervisor::Hypervisor;
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
-use oci::LinuxResources;
+use oci::{Linux, LinuxResources};
 use persist::sandbox_persist::Persist;
 use std::sync::Arc;
 use tokio::sync::RwLock;
@@ -93,6 +94,11 @@ impl ResourceManager {
         inner.handler_volumes(cid, spec).await
     }
 
+    pub async fn handler_devices(&self, cid: &str, linux: &Linux) -> Result<Vec<Device>> {
+        let inner = self.inner.read().await;
+        inner.handler_devices(cid, linux).await
+    }
+
     pub async fn dump(&self) {
         let inner = self.inner.read().await;
         inner.dump().await
diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs
index e76fa4eb6..169856875 100644
--- a/src/runtime-rs/crates/resource/src/manager_inner.rs
+++ b/src/runtime-rs/crates/resource/src/manager_inner.rs
@@ -4,16 +4,16 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::{sync::Arc, thread};
+use std::{sync::Arc, thread, vec};
 
 use crate::{network::NetworkConfig, resource_persist::ResourceState};
-use agent::{Agent, Storage};
+use agent::{types::Device, Agent, Storage};
 use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
-use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
+use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig, Hypervisor};
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
-use oci::LinuxResources;
+use oci::{Linux, LinuxResources};
 use persist::sandbox_persist::Persist;
 use tokio::{runtime, sync::RwLock};
 
@@ -246,6 +246,61 @@ impl ResourceManagerInner {
             .await
     }
 
+    pub async fn handler_devices(&self, _cid: &str, linux: &Linux) -> Result<Vec<Device>> {
+        let mut devices = vec![];
+        for d in linux.devices.iter() {
+            match d.r#type.as_str() {
+                "b" => {
+                    let device_info = DeviceConfig::Block(BlockConfig {
+                        major: d.major,
+                        minor: d.minor,
+                        ..Default::default()
+                    });
+                    let device_id = self
+                        .device_manager
+                        .write()
+                        .await
+                        .new_device(&device_info)
+                        .await
+                        .context("failed to create deviec")?;
+
+                    self.device_manager
+                        .write()
+                        .await
+                        .try_add_device(&device_id)
+                        .await
+                        .context("failed to add deivce")?;
+
+                    // get complete device information
+                    let dev_info = self
+                        .device_manager
+                        .read()
+                        .await
+                        .get_device_info(&device_id)
+                        .await
+                        .context("failed to get device info")?;
+
+                    // create agent device
+                    if let DeviceConfig::Block(config) = dev_info {
+                        let agent_device = Device {
+                            id: device_id.clone(),
+                            container_path: d.path.clone(),
+                            field_type: config.driver_option,
+                            vm_path: config.virt_path,
+                            ..Default::default()
+                        };
+                        devices.push(agent_device);
+                    }
+                }
+                _ => {
+                    // TODO enable other devices type
+                    continue;
+                }
+            }
+        }
+        Ok(devices)
+    }
+
     pub async fn update_cgroups(
         &self,
         cid: &str,
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
index 2f9f03b0e..94d75b835 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
@@ -140,7 +140,15 @@ impl Container {
         }
         spec.mounts = oci_mounts;
 
-        // TODO: handler device
+        let linux = spec
+            .linux
+            .as_ref()
+            .context("OCI spec missing linux field")?;
+
+        let devices_agent = self
+            .resource_manager
+            .handler_devices(&config.container_id, linux)
+            .await?;
 
         // update cgroups
         self.resource_manager
@@ -158,6 +166,7 @@ impl Container {
             storages,
             oci: Some(spec),
             sandbox_pidns,
+            devices: devices_agent,
             ..Default::default()
         };
 

From 6800d30fdb40f0ceb70bb4b8365adfddf1c44c7f Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Thu, 11 May 2023 11:52:17 +0800
Subject: [PATCH 139/150] runtime-rs: remove device

Support remove device after container stop

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 .../hypervisor/src/device/device_manager.rs   | 49 +++++++++++++++----
 .../src/device/driver/virtio_blk.rs           |  1 +
 src/runtime-rs/crates/resource/src/manager.rs |  6 +++
 .../crates/resource/src/manager_inner.rs      |  4 ++
 .../resource/src/rootfs/block_rootfs.rs       |  9 +++-
 .../crates/resource/src/rootfs/mod.rs         |  2 +-
 .../resource/src/rootfs/nydus_rootfs.rs       |  5 +-
 .../resource/src/rootfs/share_fs_rootfs.rs    |  4 +-
 .../resource/src/volume/block_volume.rs       | 12 +++--
 .../resource/src/volume/default_volume.rs     |  5 +-
 .../crates/resource/src/volume/hugepage.rs    |  5 +-
 .../crates/resource/src/volume/mod.rs         |  4 +-
 .../resource/src/volume/share_fs_volume.rs    |  4 +-
 .../crates/resource/src/volume/shm_volume.rs  |  4 +-
 .../src/container_manager/container.rs        | 14 ++++--
 .../src/container_manager/container_inner.rs  | 21 +++++---
 16 files changed, 110 insertions(+), 39 deletions(-)

diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
index 6d9325d39..45de0e596 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
@@ -6,7 +6,7 @@
 
 use std::{collections::HashMap, sync::Arc};
 
-use anyhow::{anyhow, Context, Ok, Result};
+use anyhow::{anyhow, Context, Result};
 use kata_sys_util::rand::RandomBytes;
 use tokio::sync::Mutex;
 
@@ -19,7 +19,7 @@ use super::{
     util::{get_host_path, get_virt_drive_name},
     Device,
 };
-pub type ArcMutexBoxDevice = Arc<Mutex<dyn Device>>;
+pub type ArcMutexDevice = Arc<Mutex<dyn Device>>;
 
 /// block_index and released_block_index are used to search an available block index
 /// in Sandbox.
@@ -60,14 +60,14 @@ impl SharedInfo {
 
 // Device manager will manage the lifecycle of sandbox device
 pub struct DeviceManager {
-    devices: HashMap<String, ArcMutexBoxDevice>,
+    devices: HashMap<String, ArcMutexDevice>,
     hypervisor: Arc<dyn Hypervisor>,
     shared_info: SharedInfo,
 }
 
 impl DeviceManager {
     pub fn new(hypervisor: Arc<dyn Hypervisor>) -> Result<Self> {
-        let devices = HashMap::<String, ArcMutexBoxDevice>::new();
+        let devices = HashMap::<String, ArcMutexDevice>::new();
         Ok(DeviceManager {
             devices,
             hypervisor,
@@ -87,23 +87,52 @@ impl DeviceManager {
     }
 
     pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> {
+        // find the device
         let device = self
             .devices
             .get(device_id)
             .context("failed to find device")?;
+        let mut device_guard = device.lock().await;
         // attach device
-        let result = device.lock().await.attach(self.hypervisor.as_ref()).await;
+        let result = device_guard.attach(self.hypervisor.as_ref()).await;
         // handle attach error
         if let Err(e) = result {
-            if let DeviceConfig::Block(config) = device.lock().await.get_device_info().await {
+            if let DeviceConfig::Block(config) = device_guard.get_device_info().await {
                 self.shared_info.release_device_index(config.index);
             };
+            drop(device_guard);
             self.devices.remove(device_id);
             return Err(e);
         }
         Ok(())
     }
 
+    pub async fn try_remove_device(&mut self, device_id: &str) -> Result<()> {
+        if let Some(dev) = self.devices.get(device_id) {
+            let mut device_guard = dev.lock().await;
+            let result = match device_guard.detach(self.hypervisor.as_ref()).await {
+                Ok(index) => {
+                    if let Some(i) = index {
+                        // release the declared block device index
+                        self.shared_info.release_device_index(i);
+                    }
+                    Ok(())
+                }
+                Err(e) => Err(e),
+            };
+            if result.is_ok() {
+                drop(device_guard);
+                // if detach success, remove it from device manager
+                self.devices.remove(device_id);
+            }
+            return result;
+        }
+        Err(anyhow!(
+            "device with specified ID hasn't been created. {}",
+            device_id
+        ))
+    }
+
     pub async fn get_device_info(&self, device_id: &str) -> Result<DeviceConfig> {
         if let Some(dev) = self.devices.get(device_id) {
             return Ok(dev.lock().await.get_device_info().await);
@@ -140,7 +169,7 @@ impl DeviceManager {
         // device ID must be generated by manager instead of device itself
         // in case of ID collision
         let device_id = self.new_device_id()?;
-        let dev: ArcMutexBoxDevice = match device_config {
+        let dev: ArcMutexDevice = match device_config {
             DeviceConfig::Block(config) => self
                 .create_block_device(config, device_id.clone())
                 .await
@@ -158,7 +187,7 @@ impl DeviceManager {
         &mut self,
         config: &BlockConfig,
         device_id: String,
-    ) -> Result<ArcMutexBoxDevice> {
+    ) -> Result<ArcMutexDevice> {
         let mut block_config = config.clone();
         block_config.id = device_id.clone();
         // get hypervisor block driver
@@ -181,8 +210,8 @@ impl DeviceManager {
         block_config.index = current_index;
         let drive_name = get_virt_drive_name(current_index as i32)?;
         block_config.virt_path = format!("/dev/{}", drive_name);
-        // if the path on host is empty, we need to get device host path from major and minor
-        // Otherwise, it might be rawfile based block device
+        // if the path on host is empty, we need to get device host path from the device major and minor number
+        // Otherwise, it might be rawfile based block device, the host path is already passed from the runtime, so we don't need to do anything here
         if block_config.path_on_host.is_empty() {
             block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor)
                 .context("failed to get host path")?;
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
index c80be671f..c9f7a2fdf 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
@@ -74,6 +74,7 @@ impl Device for BlockConfig {
     }
 
     async fn detach(&mut self, h: &dyn hypervisor) -> Result<Option<u64>> {
+        // get the count of device detached, skip detach once it reaches the 0
         if self
             .decrease_attach_count()
             .await
diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs
index 34753b7d5..a022f722f 100644
--- a/src/runtime-rs/crates/resource/src/manager.rs
+++ b/src/runtime-rs/crates/resource/src/manager.rs
@@ -11,6 +11,7 @@ use agent::types::Device;
 use agent::{Agent, Storage};
 use anyhow::Result;
 use async_trait::async_trait;
+use hypervisor::device::device_manager::DeviceManager;
 use hypervisor::Hypervisor;
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
@@ -52,6 +53,11 @@ impl ResourceManager {
         inner.config()
     }
 
+    pub async fn get_device_manager(&self) -> Arc<RwLock<DeviceManager>> {
+        let inner = self.inner.read().await;
+        inner.get_device_manager()
+    }
+
     pub async fn prepare_before_start_vm(&self, device_configs: Vec<ResourceConfig>) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.prepare_before_start_vm(device_configs).await
diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs
index 169856875..f422c2056 100644
--- a/src/runtime-rs/crates/resource/src/manager_inner.rs
+++ b/src/runtime-rs/crates/resource/src/manager_inner.rs
@@ -72,6 +72,10 @@ impl ResourceManagerInner {
         self.toml_config.clone()
     }
 
+    pub fn get_device_manager(&self) -> Arc<RwLock<DeviceManager>> {
+        self.device_manager.clone()
+    }
+
     pub async fn prepare_before_start_vm(
         &mut self,
         device_configs: Vec<ResourceConfig>,
diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
index 56a2641da..2c2568500 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
@@ -104,8 +104,13 @@ impl Rootfs for BlockRootfs {
     async fn get_device_id(&self) -> Result<Option<String>> {
         Ok(Some(self.device_id.clone()))
     }
-    async fn cleanup(&self) -> Result<()> {
-        Ok(())
+
+    async fn cleanup(&self, device_manager: &RwLock<DeviceManager>) -> Result<()> {
+        device_manager
+            .write()
+            .await
+            .try_remove_device(&self.device_id)
+            .await
     }
 }
 
diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs
index 06487fc43..a924b021b 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs
@@ -27,7 +27,7 @@ pub trait Rootfs: Send + Sync {
     async fn get_guest_rootfs_path(&self) -> Result<String>;
     async fn get_rootfs_mount(&self) -> Result<Vec<oci::Mount>>;
     async fn get_storage(&self) -> Option<Storage>;
-    async fn cleanup(&self) -> Result<()>;
+    async fn cleanup(&self, device_manager: &RwLock<DeviceManager>) -> Result<()>;
     async fn get_device_id(&self) -> Result<Option<String>>;
 }
 
diff --git a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs
index b8f9ed100..96f29e8d6 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs
@@ -16,8 +16,9 @@ use crate::{
 use agent::Storage;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::Hypervisor;
+use hypervisor::{device::device_manager::DeviceManager, Hypervisor};
 use kata_types::mount::{Mount, NydusExtraOptions};
+use tokio::sync::RwLock;
 
 // Used for nydus rootfs
 pub(crate) const NYDUS_ROOTFS_TYPE: &str = "fuse.nydus-overlayfs";
@@ -154,7 +155,7 @@ impl Rootfs for NydusRootfs {
         Ok(None)
     }
 
-    async fn cleanup(&self) -> Result<()> {
+    async fn cleanup(&self, _device_manager: &RwLock<DeviceManager>) -> Result<()> {
         // TODO: Clean up NydusRootfs after the container is killed
         warn!(sl!(), "Cleaning up NydusRootfs is still unimplemented.");
         Ok(())
diff --git a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs
index 7a490215a..385c058d3 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs
@@ -9,8 +9,10 @@ use std::sync::Arc;
 use agent::Storage;
 use anyhow::{Context, Result};
 use async_trait::async_trait;
+use hypervisor::device::device_manager::DeviceManager;
 use kata_sys_util::mount::{umount_timeout, Mounter};
 use kata_types::mount::Mount;
+use tokio::sync::RwLock;
 
 use super::{Rootfs, ROOTFS};
 use crate::share_fs::{ShareFs, ShareFsRootfsConfig};
@@ -79,7 +81,7 @@ impl Rootfs for ShareFsRootfs {
         Ok(None)
     }
 
-    async fn cleanup(&self) -> Result<()> {
+    async fn cleanup(&self, _device_manager: &RwLock<DeviceManager>) -> Result<()> {
         // Umount the mount point shared to guest
         let share_fs_mount = self.share_fs.get_share_fs_mount();
         share_fs_mount
diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs
index 74c75207b..079465997 100644
--- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs
@@ -13,7 +13,7 @@ use crate::share_fs::{do_get_guest_path, do_get_host_path};
 use super::{share_fs_volume::generate_mount_path, Volume};
 use agent::Storage;
 use anyhow::{anyhow, Context};
-use hypervisor::{device::DeviceManager, BlockConfig, DeviceConfig};
+use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig};
 use nix::sys::stat::{self, SFlag};
 use tokio::sync::RwLock;
 #[derive(Debug)]
@@ -128,10 +128,12 @@ impl Volume for BlockVolume {
         Ok(s)
     }
 
-    async fn cleanup(&self) -> Result<()> {
-        // TODO: Clean up BlockVolume
-        warn!(sl!(), "Cleaning up BlockVolume is still unimplemented.");
-        Ok(())
+    async fn cleanup(&self, device_manager: &RwLock<DeviceManager>) -> Result<()> {
+        device_manager
+            .write()
+            .await
+            .try_remove_device(&self.device_id)
+            .await
     }
 
     fn get_device_id(&self) -> Result<Option<String>> {
diff --git a/src/runtime-rs/crates/resource/src/volume/default_volume.rs b/src/runtime-rs/crates/resource/src/volume/default_volume.rs
index f1e6e7983..827d2b121 100644
--- a/src/runtime-rs/crates/resource/src/volume/default_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/default_volume.rs
@@ -4,6 +4,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use hypervisor::device::device_manager::DeviceManager;
+use tokio::sync::RwLock;
+
 use anyhow::Result;
 use async_trait::async_trait;
 
@@ -33,7 +36,7 @@ impl Volume for DefaultVolume {
         Ok(vec![])
     }
 
-    async fn cleanup(&self) -> Result<()> {
+    async fn cleanup(&self, _device_manager: &RwLock<DeviceManager>) -> Result<()> {
         // TODO: Clean up DefaultVolume
         warn!(sl!(), "Cleaning up DefaultVolume is still unimplemented.");
         Ok(())
diff --git a/src/runtime-rs/crates/resource/src/volume/hugepage.rs b/src/runtime-rs/crates/resource/src/volume/hugepage.rs
index e8e0548f5..ca8502e7e 100644
--- a/src/runtime-rs/crates/resource/src/volume/hugepage.rs
+++ b/src/runtime-rs/crates/resource/src/volume/hugepage.rs
@@ -15,9 +15,10 @@ use agent::Storage;
 use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
 use byte_unit::Byte;
-use hypervisor::HUGETLBFS;
+use hypervisor::{device::device_manager::DeviceManager, HUGETLBFS};
 use kata_sys_util::{fs::get_base_name, mount::PROC_MOUNTS_FILE};
 use kata_types::mount::KATA_EPHEMERAL_VOLUME_TYPE;
+use tokio::sync::RwLock;
 
 use super::{Volume, BIND};
 
@@ -88,7 +89,7 @@ impl Volume for Hugepage {
         Ok(s)
     }
 
-    async fn cleanup(&self) -> Result<()> {
+    async fn cleanup(&self, _device_manager: &RwLock<DeviceManager>) -> Result<()> {
         Ok(())
     }
 
diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs
index 067ad8f65..ea7d4f5d6 100644
--- a/src/runtime-rs/crates/resource/src/volume/mod.rs
+++ b/src/runtime-rs/crates/resource/src/volume/mod.rs
@@ -12,7 +12,7 @@ mod shm_volume;
 use async_trait::async_trait;
 
 use anyhow::{Context, Result};
-use hypervisor::device::DeviceManager;
+use hypervisor::device::device_manager::DeviceManager;
 use std::{sync::Arc, vec::Vec};
 use tokio::sync::RwLock;
 
@@ -27,7 +27,7 @@ pub trait Volume: Send + Sync {
     fn get_volume_mount(&self) -> Result<Vec<oci::Mount>>;
     fn get_storage(&self) -> Result<Vec<agent::Storage>>;
     fn get_device_id(&self) -> Result<Option<String>>;
-    async fn cleanup(&self) -> Result<()>;
+    async fn cleanup(&self, device_manager: &RwLock<DeviceManager>) -> Result<()>;
 }
 
 #[derive(Default)]
diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs
index 9b91856ac..0748e374d 100644
--- a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs
@@ -12,6 +12,8 @@ use std::{
 
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
+use hypervisor::device::device_manager::DeviceManager;
+use tokio::sync::RwLock;
 
 use super::Volume;
 use crate::share_fs::{MountedInfo, ShareFs, ShareFsVolumeConfig};
@@ -158,7 +160,7 @@ impl Volume for ShareFsVolume {
         Ok(self.storages.clone())
     }
 
-    async fn cleanup(&self) -> Result<()> {
+    async fn cleanup(&self, _device_manager: &RwLock<DeviceManager>) -> Result<()> {
         let share_fs = match self.share_fs.as_ref() {
             Some(fs) => fs,
             None => return Ok(()),
diff --git a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs
index 574842b4b..fb12b3614 100644
--- a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs
@@ -8,6 +8,8 @@ use std::path::Path;
 
 use anyhow::Result;
 use async_trait::async_trait;
+use hypervisor::device::device_manager::DeviceManager;
+use tokio::sync::RwLock;
 
 use super::Volume;
 use crate::share_fs::DEFAULT_KATA_GUEST_SANDBOX_DIR;
@@ -99,7 +101,7 @@ impl Volume for ShmVolume {
         Ok(s)
     }
 
-    async fn cleanup(&self) -> Result<()> {
+    async fn cleanup(&self, _device_manager: &RwLock<DeviceManager>) -> Result<()> {
         // TODO: Clean up ShmVolume
         warn!(sl!(), "Cleaning up ShmVolume is still unimplemented.");
         Ok(())
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
index 94d75b835..6dfea9fd0 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs
@@ -183,7 +183,8 @@ impl Container {
         match process.process_type {
             ProcessType::Container => {
                 if let Err(err) = inner.start_container(&process.container_id).await {
-                    let _ = inner.stop_process(process, true).await;
+                    let device_manager = self.resource_manager.get_device_manager().await;
+                    let _ = inner.stop_process(process, true, &device_manager).await;
                     return Err(err);
                 }
 
@@ -195,7 +196,8 @@ impl Container {
             }
             ProcessType::Exec => {
                 if let Err(e) = inner.start_exec_process(process).await {
-                    let _ = inner.stop_process(process, true).await;
+                    let device_manager = self.resource_manager.get_device_manager().await;
+                    let _ = inner.stop_process(process, true, &device_manager).await;
                     return Err(e).context("enter process");
                 }
 
@@ -277,7 +279,10 @@ impl Container {
         all: bool,
     ) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.signal_process(container_process, signal, all).await
+        let device_manager = self.resource_manager.get_device_manager().await;
+        inner
+            .signal_process(container_process, signal, all, &device_manager)
+            .await
     }
 
     pub async fn exec_process(
@@ -314,8 +319,9 @@ impl Container {
 
     pub async fn stop_process(&self, container_process: &ContainerProcess) -> Result<()> {
         let mut inner = self.inner.write().await;
+        let device_manager = self.resource_manager.get_device_manager().await;
         inner
-            .stop_process(container_process, true)
+            .stop_process(container_process, true, &device_manager)
             .await
             .context("stop process")
     }
diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs
index bb6c2ed07..12d4810fb 100644
--- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs
+++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs
@@ -12,6 +12,7 @@ use common::{
     error::Error,
     types::{ContainerID, ContainerProcess, ProcessExitStatus, ProcessStatus, ProcessType},
 };
+use hypervisor::device::device_manager::DeviceManager;
 use nix::sys::signal::Signal;
 use resource::{rootfs::Rootfs, volume::Volume};
 use tokio::sync::RwLock;
@@ -193,6 +194,7 @@ impl ContainerInner {
         &mut self,
         process: &ContainerProcess,
         force: bool,
+        device_manager: &RwLock<DeviceManager>,
     ) -> Result<()> {
         let logger = logger_with_process(process);
         info!(logger, "begin to stop process");
@@ -212,7 +214,7 @@ impl ContainerInner {
             // send kill signal to container
             // ignore the error of sending signal, since the process would
             // have been killed and exited yet.
-            self.signal_process(process, Signal::SIGKILL as u32, false)
+            self.signal_process(process, Signal::SIGKILL as u32, false, device_manager)
                 .await
                 .map_err(|e| {
                     warn!(logger, "failed to signal kill. {:?}", e);
@@ -242,6 +244,7 @@ impl ContainerInner {
         process: &ContainerProcess,
         signal: u32,
         all: bool,
+        device_manager: &RwLock<DeviceManager>,
     ) -> Result<()> {
         let mut process_id: agent::ContainerProcessID = process.clone().into();
         if all {
@@ -253,8 +256,12 @@ impl ContainerInner {
             .signal_process(agent::SignalProcessRequest { process_id, signal })
             .await?;
 
-        self.clean_volumes().await.context("clean volumes")?;
-        self.clean_rootfs().await.context("clean rootfs")?;
+        self.clean_volumes(device_manager)
+            .await
+            .context("clean volumes")?;
+        self.clean_rootfs(device_manager)
+            .await
+            .context("clean rootfs")?;
 
         Ok(())
     }
@@ -278,10 +285,10 @@ impl ContainerInner {
         Ok(())
     }
 
-    async fn clean_volumes(&mut self) -> Result<()> {
+    async fn clean_volumes(&mut self, device_manager: &RwLock<DeviceManager>) -> Result<()> {
         let mut unhandled = Vec::new();
         for v in self.volumes.iter() {
-            if let Err(err) = v.cleanup().await {
+            if let Err(err) = v.cleanup(device_manager).await {
                 unhandled.push(Arc::clone(v));
                 warn!(
                     sl!(),
@@ -297,10 +304,10 @@ impl ContainerInner {
         Ok(())
     }
 
-    async fn clean_rootfs(&mut self) -> Result<()> {
+    async fn clean_rootfs(&mut self, device_manager: &RwLock<DeviceManager>) -> Result<()> {
         let mut unhandled = Vec::new();
         for rootfs in self.rootfs.iter() {
-            if let Err(err) = rootfs.cleanup().await {
+            if let Err(err) = rootfs.cleanup(device_manager).await {
                 unhandled.push(Arc::clone(rootfs));
                 warn!(
                     sl!(),

From f9bded44843e86e632241f353f53bd63d6bf6b9c Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Sat, 20 May 2023 18:07:32 +0800
Subject: [PATCH 140/150] runtime-rs: add devicetype enum

use device type to store the config information for different kind of
devices

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 .../crates/hypervisor/src/ch/inner.rs         |  4 +-
 .../crates/hypervisor/src/ch/inner_device.rs  | 20 ++---
 .../hypervisor/src/ch/inner_hypervisor.rs     | 11 +--
 .../crates/hypervisor/src/ch/mod.rs           |  7 +-
 .../hypervisor/src/device/device_manager.rs   | 18 +++--
 .../hypervisor/src/device/driver/mod.rs       | 32 +++-----
 .../hypervisor/src/device/driver/vfio.rs      | 15 +++-
 .../src/device/driver/vhost_user.rs           | 10 ++-
 .../src/device/driver/virtio_blk.rs           | 33 ++++++---
 .../hypervisor/src/device/driver/virtio_fs.rs | 10 +++
 .../src/device/driver/virtio_net.rs           | 12 ++-
 .../src/device/driver/virtio_vsock.rs         | 30 ++++++--
 .../crates/hypervisor/src/device/mod.rs       | 36 ++++++++-
 .../crates/hypervisor/src/dragonball/inner.rs |  6 +-
 .../hypervisor/src/dragonball/inner_device.rs | 46 ++++++------
 .../src/dragonball/inner_hypervisor.rs        | 12 ++-
 .../crates/hypervisor/src/dragonball/mod.rs   |  6 +-
 src/runtime-rs/crates/hypervisor/src/lib.rs   |  6 +-
 .../crates/hypervisor/src/qemu/inner.rs       |  6 +-
 .../crates/hypervisor/src/qemu/mod.rs         |  6 +-
 .../crates/resource/src/manager_inner.rs      |  9 ++-
 .../src/network/endpoint/ipvlan_endpoint.rs   | 23 ++++--
 .../src/network/endpoint/macvlan_endpoint.rs  | 25 +++++--
 .../src/network/endpoint/physical_endpoint.rs | 14 ++--
 .../src/network/endpoint/veth_endpoint.rs     | 23 ++++--
 .../src/network/endpoint/vlan_endpoint.rs     | 23 ++++--
 .../resource/src/rootfs/block_rootfs.rs       |  9 ++-
 .../resource/src/share_fs/share_virtio_fs.rs  | 74 +++++++++++--------
 .../resource/src/volume/block_volume.rs       |  9 ++-
 29 files changed, 337 insertions(+), 198 deletions(-)

diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
index 33a3fbc56..23b076b16 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::HypervisorState;
-use crate::driver::DeviceConfig;
+use crate::device::DeviceType;
 use crate::VmmState;
 use anyhow::Result;
 use async_trait::async_trait;
@@ -44,7 +44,7 @@ pub struct CloudHypervisorInner {
     pub(crate) jailer_root: String,
 
     /// List of devices that will be added to the VM once it boots
-    pub(crate) pending_devices: Option<Vec<DeviceConfig>>,
+    pub(crate) pending_devices: Option<Vec<DeviceType>>,
 
     pub(crate) _capabilities: Capabilities,
 
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
index ff0f874ec..e06832cbd 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs
@@ -5,8 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::inner::CloudHypervisorInner;
-use crate::driver::{DeviceConfig, ShareFsDeviceConfig};
+use crate::device::DeviceType;
 use crate::HybridVsockConfig;
+use crate::ShareFsDeviceConfig;
 use crate::VmmState;
 use anyhow::{anyhow, Context, Result};
 use ch_config::ch_api::cloud_hypervisor_vm_fs_add;
@@ -18,10 +19,9 @@ use std::path::PathBuf;
 const VIRTIO_FS: &str = "virtio-fs";
 
 impl CloudHypervisorInner {
-    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> {
         if self.state != VmmState::VmRunning {
-            let mut devices: Vec<DeviceConfig> = if let Some(devices) = self.pending_devices.take()
-            {
+            let mut devices: Vec<DeviceType> = if let Some(devices) = self.pending_devices.take() {
                 devices
             } else {
                 vec![]
@@ -39,10 +39,10 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    async fn handle_add_device(&mut self, device: DeviceConfig) -> Result<()> {
+    async fn handle_add_device(&mut self, device: DeviceType) -> Result<()> {
         match device {
-            DeviceConfig::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await,
-            DeviceConfig::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await,
+            DeviceType::ShareFs(sharefs) => self.handle_share_fs_device(sharefs.config).await,
+            DeviceType::HybridVsock(hvsock) => self.handle_hvsock_device(&hvsock.config).await,
             _ => Err(anyhow!("unhandled device: {:?}", device)),
         }
     }
@@ -67,7 +67,7 @@ impl CloudHypervisorInner {
         Ok(())
     }
 
-    pub(crate) async fn remove_device(&mut self, _device: DeviceConfig) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, _device: DeviceType) -> Result<()> {
         Ok(())
     }
 
@@ -133,8 +133,8 @@ impl CloudHypervisorInner {
         if let Some(devices) = pending_root_devices {
             for dev in devices {
                 match dev {
-                    DeviceConfig::ShareFsDevice(dev) => {
-                        let settings = ShareFsSettings::new(dev, self.vm_path.clone());
+                    DeviceType::ShareFs(dev) => {
+                        let settings = ShareFsSettings::new(dev.config, self.vm_path.clone());
 
                         let fs_cfg = FsConfig::try_from(settings)?;
 
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
index 48eeccc83..9cf5bcd8c 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs
@@ -6,9 +6,9 @@
 use super::inner::CloudHypervisorInner;
 use crate::ch::utils::get_api_socket_path;
 use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path};
+use crate::device::DeviceType;
 use crate::kernel_param::KernelParams;
-use crate::DeviceConfig;
-use crate::VsockConfig;
+use crate::VsockDevice;
 use crate::VM_ROOTFS_DRIVER_PMEM;
 use crate::{VcpuThreadIds, VmmState};
 use anyhow::{anyhow, Context, Result};
@@ -417,10 +417,11 @@ impl CloudHypervisorInner {
 
         self.netns = netns;
 
-        let vsock_cfg = VsockConfig::new(self.id.clone()).await?;
+        let vsock_dev = VsockDevice::new(self.id.clone()).await?;
 
-        let dev = DeviceConfig::Vsock(vsock_cfg);
-        self.add_device(dev).await.context("add vsock device")?;
+        self.add_device(DeviceType::Vsock(vsock_dev))
+            .await
+            .context("add vsock device")?;
 
         self.start_hypervisor(self.timeout_secs).await?;
 
diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
index baeadd2d2..a4b8b05ff 100644
--- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs
@@ -4,7 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::HypervisorState;
-use crate::{driver::DeviceConfig, Hypervisor, VcpuThreadIds};
+use crate::device::DeviceType;
+use crate::{Hypervisor, VcpuThreadIds};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use kata_types::capabilities::Capabilities;
@@ -78,12 +79,12 @@ impl Hypervisor for CloudHypervisor {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
+    async fn add_device(&self, device: DeviceType) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
+    async fn remove_device(&self, device: DeviceType) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
index 45de0e596..c633cf918 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs
@@ -11,13 +11,13 @@ use kata_sys_util::rand::RandomBytes;
 use tokio::sync::Mutex;
 
 use crate::{
-    BlockConfig, DeviceConfig, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE,
+    BlockConfig, BlockDevice, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE,
     VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
 };
 
 use super::{
     util::{get_host_path, get_virt_drive_name},
-    Device,
+    Device, DeviceConfig,
 };
 pub type ArcMutexDevice = Arc<Mutex<dyn Device>>;
 
@@ -97,7 +97,7 @@ impl DeviceManager {
         let result = device_guard.attach(self.hypervisor.as_ref()).await;
         // handle attach error
         if let Err(e) = result {
-            if let DeviceConfig::Block(config) = device_guard.get_device_info().await {
+            if let DeviceConfig::BlockCfg(config) = device_guard.get_device_info().await {
                 self.shared_info.release_device_index(config.index);
             };
             drop(device_guard);
@@ -146,8 +146,8 @@ impl DeviceManager {
     async fn find_device(&self, device_config: &DeviceConfig) -> Option<String> {
         for (device_id, dev) in &self.devices {
             match dev.lock().await.get_device_info().await {
-                DeviceConfig::Block(config) => match device_config {
-                    DeviceConfig::Block(ref config_new) => {
+                DeviceConfig::BlockCfg(config) => match device_config {
+                    DeviceConfig::BlockCfg(ref config_new) => {
                         if config_new.path_on_host == config.path_on_host {
                             return Some(device_id.to_string());
                         }
@@ -170,7 +170,7 @@ impl DeviceManager {
         // in case of ID collision
         let device_id = self.new_device_id()?;
         let dev: ArcMutexDevice = match device_config {
-            DeviceConfig::Block(config) => self
+            DeviceConfig::BlockCfg(config) => self
                 .create_block_device(config, device_id.clone())
                 .await
                 .context("failed to create device")?,
@@ -189,7 +189,6 @@ impl DeviceManager {
         device_id: String,
     ) -> Result<ArcMutexDevice> {
         let mut block_config = config.clone();
-        block_config.id = device_id.clone();
         // get hypervisor block driver
         let block_driver = match self
             .hypervisor
@@ -216,7 +215,10 @@ impl DeviceManager {
             block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor)
                 .context("failed to get host path")?;
         }
-        Ok(Arc::new(Mutex::new(BlockConfig::new(block_config))))
+        Ok(Arc::new(Mutex::new(BlockDevice::new(
+            device_id,
+            block_config,
+        ))))
     }
 
     // device ID must be generated by device manager instead of device itself
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
index 71db1e4fd..45487e3b5 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs
@@ -7,31 +7,17 @@
 mod vhost_user;
 mod virtio_blk;
 pub use virtio_blk::{
-    BlockConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI,
+    BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO,
+    VIRTIO_BLOCK_PCI,
 };
 mod virtio_net;
-pub use virtio_net::{Address, NetworkConfig};
+pub use virtio_net::{Address, NetworkConfig, NetworkDevice};
 mod vfio;
-pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig};
+pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig, VfioDevice};
 mod virtio_fs;
-pub use virtio_fs::{ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountType, ShareFsOperation};
+pub use virtio_fs::{
+    ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType,
+    ShareFsOperation,
+};
 mod virtio_vsock;
-use std::fmt;
-pub use virtio_vsock::{HybridVsockConfig, VsockConfig};
-
-#[derive(Debug)]
-pub enum DeviceConfig {
-    Block(BlockConfig),
-    Network(NetworkConfig),
-    ShareFsDevice(ShareFsDeviceConfig),
-    Vfio(VfioConfig),
-    ShareFsMount(ShareFsMountConfig),
-    Vsock(VsockConfig),
-    HybridVsock(HybridVsockConfig),
-}
-
-impl fmt::Display for DeviceConfig {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
+pub use virtio_vsock::{HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice};
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
index 0366257d1..ff3a579a8 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs
@@ -6,8 +6,9 @@
 
 use std::{fs, path::Path, process::Command};
 
+use crate::device::Device;
+use crate::device::DeviceConfig;
 use crate::Hypervisor as hypervisor;
-use crate::{device::Device, DeviceConfig};
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use anyhow::anyhow;
 use anyhow::{Context, Result};
@@ -45,9 +46,6 @@ impl VfioBusMode {
 
 #[derive(Debug, Clone)]
 pub struct VfioConfig {
-    /// Unique identifier of the device
-    pub id: String,
-
     /// Sysfs path for mdev bus type device
     pub sysfs_path: String,
 
@@ -58,6 +56,15 @@ pub struct VfioConfig {
     pub mode: VfioBusMode,
 }
 
+#[derive(Debug, Clone)]
+pub struct VfioDevice {
+    /// Unique identifier of the device
+    pub id: String,
+
+    /// Config info for Vfio Device
+    pub config: VfioConfig,
+}
+
 /// binds the device to vfio driver after unbinding from host.
 /// Will be called by a network interface or a generic pcie device.
 pub fn bind_device_to_vfio(bdf: &str, host_driver: &str, _vendor_device_id: &str) -> Result<()> {
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
index bfde23239..d778c4459 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs
@@ -4,11 +4,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use crate::device::Device;
+use crate::device::DeviceConfig;
 use crate::Hypervisor as hypervisor;
-use crate::{device::Device, DeviceConfig};
 use anyhow::Result;
 use async_trait::async_trait;
 
+#[derive(Debug, Clone, Default)]
 /// VhostUserConfig represents data shared by most vhost-user devices
 pub struct VhostUserConfig {
     /// Device id
@@ -29,6 +31,12 @@ pub struct VhostUserConfig {
     pub queue_siez: u32,
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct VhostUserDevice {
+    pub device_id: String,
+    pub config: VhostUserConfig,
+}
+
 #[async_trait]
 impl Device for VhostUserConfig {
     async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> {
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
index c9f7a2fdf..2ff98a1e7 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs
@@ -5,8 +5,9 @@
 //
 
 pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio";
+use crate::device::Device;
+use crate::device::{DeviceConfig, DeviceType};
 use crate::Hypervisor as hypervisor;
-use crate::{device::Device, DeviceConfig};
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 /// VIRTIO_BLOCK_PCI indicates block driver is virtio-pci based
@@ -16,9 +17,6 @@ pub const KATA_BLK_DEV_TYPE: &str = "blk";
 
 #[derive(Debug, Clone, Default)]
 pub struct BlockConfig {
-    /// Unique identifier of the drive.
-    pub id: String,
-
     /// Path of the drive.
     pub path_on_host: String,
 
@@ -48,15 +46,26 @@ pub struct BlockConfig {
     pub minor: i64,
 }
 
-impl BlockConfig {
+#[derive(Debug, Clone, Default)]
+pub struct BlockDevice {
+    pub device_id: String,
+    pub attach_count: u64,
+    pub config: BlockConfig,
+}
+
+impl BlockDevice {
     // new creates a new VirtioBlkDevice
-    pub fn new(dev_info: BlockConfig) -> Self {
-        dev_info
+    pub fn new(device_id: String, config: BlockConfig) -> Self {
+        BlockDevice {
+            device_id,
+            attach_count: 0,
+            config,
+        }
     }
 }
 
 #[async_trait]
-impl Device for BlockConfig {
+impl Device for BlockDevice {
     async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
         // increase attach count, skip attach the device if the device is already attached
         if self
@@ -66,7 +75,7 @@ impl Device for BlockConfig {
         {
             return Ok(());
         }
-        if let Err(e) = h.add_device(DeviceConfig::Block(self.clone())).await {
+        if let Err(e) = h.add_device(DeviceType::Block(self.clone())).await {
             self.decrease_attach_count().await?;
             return Err(e);
         }
@@ -82,15 +91,15 @@ impl Device for BlockConfig {
         {
             return Ok(None);
         }
-        if let Err(e) = h.remove_device(DeviceConfig::Block(self.clone())).await {
+        if let Err(e) = h.remove_device(DeviceType::Block(self.clone())).await {
             self.increase_attach_count().await?;
             return Err(e);
         }
-        Ok(Some(self.index))
+        Ok(Some(self.config.index))
     }
 
     async fn get_device_info(&self) -> DeviceConfig {
-        DeviceConfig::Block(self.clone())
+        DeviceConfig::BlockCfg(self.config.clone())
     }
 
     async fn increase_attach_count(&mut self) -> Result<bool> {
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs
index b40133c65..d2d3cc762 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs
@@ -42,6 +42,11 @@ pub struct ShareFsMountConfig {
     pub prefetch_list_path: Option<String>,
 }
 
+#[derive(Debug, Clone)]
+pub struct ShareFsMountDevice {
+    pub config: ShareFsMountConfig,
+}
+
 /// ShareFsDeviceConfig: share fs device config
 #[derive(Debug, Clone)]
 pub struct ShareFsDeviceConfig {
@@ -63,3 +68,8 @@ pub struct ShareFsDeviceConfig {
     /// queue_num: queue number
     pub queue_num: u64,
 }
+
+#[derive(Debug, Clone)]
+pub struct ShareFsDevice {
+    pub config: ShareFsDeviceConfig,
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs
index 18b983039..3be861ced 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs
@@ -22,12 +22,18 @@ impl fmt::Debug for Address {
 
 #[derive(Debug, Clone)]
 pub struct NetworkConfig {
-    /// Unique identifier of the device
-    pub id: String,
-
     /// Host level path for the guest network interface.
     pub host_dev_name: String,
 
     /// Guest MAC address.
     pub guest_mac: Option<Address>,
 }
+
+#[derive(Debug, Clone)]
+pub struct NetworkDevice {
+    /// Unique identifier of the device
+    pub id: String,
+
+    /// Network Device config info
+    pub config: NetworkConfig,
+}
diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs
index cd0553adf..d94e8f864 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs
@@ -11,9 +11,6 @@ use tokio::fs::{File, OpenOptions};
 
 #[derive(Debug)]
 pub struct HybridVsockConfig {
-    /// Unique identifier of the device
-    pub id: String,
-
     /// A 32-bit Context Identifier (CID) used to identify the guest.
     pub guest_cid: u32,
 
@@ -22,10 +19,16 @@ pub struct HybridVsockConfig {
 }
 
 #[derive(Debug)]
-pub struct VsockConfig {
+pub struct HybridVsockDevice {
     /// Unique identifier of the device
     pub id: String,
 
+    /// config information for HybridVsockDevice
+    pub config: HybridVsockConfig,
+}
+
+#[derive(Debug)]
+pub struct VsockConfig {
     /// A 32-bit Context Identifier (CID) used to identify the guest.
     pub guest_cid: u32,
 
@@ -33,6 +36,15 @@ pub struct VsockConfig {
     pub vhost_fd: File,
 }
 
+#[derive(Debug)]
+pub struct VsockDevice {
+    /// Unique identifier of the device
+    pub id: String,
+
+    /// config information for VsockDevice
+    pub config: VsockConfig,
+}
+
 const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock";
 
 // From <linux/vhost.h>
@@ -50,7 +62,7 @@ nix::ioctl_write_ptr!(
 
 const CID_RETRY_COUNT: u32 = 50;
 
-impl VsockConfig {
+impl VsockDevice {
     pub async fn new(id: String) -> Result<Self> {
         let vhost_fd = OpenOptions::new()
             .read(true)
@@ -72,10 +84,12 @@ impl VsockConfig {
                 unsafe { vhost_vsock_set_guest_cid(vhost_fd.as_raw_fd(), &(rand_cid as u64)) };
             match guest_cid {
                 Ok(_) => {
-                    return Ok(VsockConfig {
+                    return Ok(VsockDevice {
                         id,
-                        guest_cid: rand_cid,
-                        vhost_fd,
+                        config: VsockConfig {
+                            guest_cid: rand_cid,
+                            vhost_fd,
+                        },
                     });
                 }
                 Err(nix::Error::EADDRINUSE) => {
diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs
index 131c7482c..d341d9a12 100644
--- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs
@@ -4,7 +4,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use crate::{DeviceConfig, Hypervisor as hypervisor};
+use std::fmt;
+
+use crate::{
+    BlockConfig, BlockDevice, HybridVsockConfig, HybridVsockDevice, Hypervisor as hypervisor,
+    NetworkConfig, NetworkDevice, ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig,
+    ShareFsMountDevice, VfioConfig, VfioDevice, VsockConfig, VsockDevice,
+};
 use anyhow::Result;
 use async_trait::async_trait;
 
@@ -12,6 +18,34 @@ pub mod device_manager;
 pub mod driver;
 pub mod util;
 
+#[derive(Debug)]
+pub enum DeviceConfig {
+    BlockCfg(BlockConfig),
+    NetworkCfg(NetworkConfig),
+    ShareFsCfg(ShareFsDeviceConfig),
+    VfioCfg(VfioConfig),
+    ShareFsMountCfg(ShareFsMountConfig),
+    VsockCfg(VsockConfig),
+    HybridVsockCfg(HybridVsockConfig),
+}
+
+#[derive(Debug)]
+pub enum DeviceType {
+    Block(BlockDevice),
+    Vfio(VfioDevice),
+    Network(NetworkDevice),
+    ShareFs(ShareFsDevice),
+    ShareFsMount(ShareFsMountDevice),
+    HybridVsock(HybridVsockDevice),
+    Vsock(VsockDevice),
+}
+
+impl fmt::Display for DeviceType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 #[async_trait]
 pub trait Device: Send + Sync {
     // attach is to plug device into VM
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
index 710a0e50d..998020d21 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
@@ -6,8 +6,8 @@
 
 use super::vmm_instance::VmmInstance;
 use crate::{
-    driver::DeviceConfig, hypervisor_persist::HypervisorState, kernel_param::KernelParams,
-    VmmState, DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
+    device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
+    DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
 };
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
@@ -56,7 +56,7 @@ pub struct DragonballInner {
     pub(crate) run_dir: String,
 
     /// pending device
-    pub(crate) pending_devices: Vec<DeviceConfig>,
+    pub(crate) pending_devices: Vec<DeviceType>,
 
     /// cached block device
     pub(crate) cached_block_devices: HashSet<String>,
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
index 966202d79..94861536c 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs
@@ -15,8 +15,8 @@ use dragonball::api::v1::{
 
 use super::DragonballInner;
 use crate::{
-    driver::DeviceConfig, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig,
-    ShareFsMountConfig, ShareFsMountType, ShareFsOperation, VmmState,
+    device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig,
+    ShareFsMountType, ShareFsOperation, VmmState,
 };
 
 const MB_TO_B: u32 = 1024 * 1024;
@@ -31,7 +31,7 @@ pub(crate) fn drive_index_to_id(index: u64) -> String {
 }
 
 impl DragonballInner {
-    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> {
         if self.state == VmmState::NotReady {
             info!(sl!(), "VMM not ready, queueing device {}", device);
 
@@ -44,41 +44,43 @@ impl DragonballInner {
 
         info!(sl!(), "dragonball add device {:?}", &device);
         match device {
-            DeviceConfig::Network(config) => self.add_net_device(&config).context("add net device"),
-            DeviceConfig::Vfio(_config) => {
+            DeviceType::Network(network) => self
+                .add_net_device(&network.config, network.id)
+                .context("add net device"),
+            DeviceType::Vfio(_) => {
                 todo!()
             }
-            DeviceConfig::Block(config) => self
+            DeviceType::Block(block) => self
                 .add_block_device(
-                    config.path_on_host.as_str(),
-                    config.id.as_str(),
-                    config.is_readonly,
-                    config.no_drop,
+                    block.config.path_on_host.as_str(),
+                    block.device_id.as_str(),
+                    block.config.is_readonly,
+                    block.config.no_drop,
                 )
                 .context("add block device"),
-            DeviceConfig::HybridVsock(config) => self.add_hvsock(&config).context("add vsock"),
-            DeviceConfig::ShareFsDevice(config) => self
-                .add_share_fs_device(&config)
+            DeviceType::HybridVsock(hvsock) => self.add_hvsock(&hvsock.config).context("add vsock"),
+            DeviceType::ShareFs(sharefs) => self
+                .add_share_fs_device(&sharefs.config)
                 .context("add share fs device"),
-            DeviceConfig::ShareFsMount(config) => self
-                .add_share_fs_mount(&config)
+            DeviceType::ShareFsMount(sharefs_mount) => self
+                .add_share_fs_mount(&sharefs_mount.config)
                 .context("add share fs mount"),
-            DeviceConfig::Vsock(_) => {
+            DeviceType::Vsock(_) => {
                 todo!()
             }
         }
     }
 
-    pub(crate) async fn remove_device(&mut self, device: DeviceConfig) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, device: DeviceType) -> Result<()> {
         info!(sl!(), "remove device {} ", device);
 
         match device {
-            DeviceConfig::Block(config) => {
-                let drive_id = drive_index_to_id(config.index);
+            DeviceType::Block(block) => {
+                let drive_id = drive_index_to_id(block.config.index);
                 self.remove_block_drive(drive_id.as_str())
                     .context("remove block drive")
             }
-            DeviceConfig::Vfio(_config) => {
+            DeviceType::Vfio(_config) => {
                 todo!()
             }
             _ => Err(anyhow!("unsupported device {:?}", device)),
@@ -121,9 +123,9 @@ impl DragonballInner {
         Ok(())
     }
 
-    fn add_net_device(&mut self, config: &NetworkConfig) -> Result<()> {
+    fn add_net_device(&mut self, config: &NetworkConfig, device_id: String) -> Result<()> {
         let iface_cfg = VirtioNetDeviceConfigInfo {
-            iface_id: config.id.clone(),
+            iface_id: device_id,
             host_dev_name: config.host_dev_name.clone(),
             guest_mac: match &config.guest_mac {
                 Some(mac) => MacAddr::from_bytes(&mac.0).ok(),
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
index 10640216f..18aa7139d 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs
@@ -13,7 +13,9 @@ use anyhow::{Context, Ok, Result};
 use kata_types::capabilities::Capabilities;
 
 use super::inner::DragonballInner;
-use crate::{utils, VcpuThreadIds, VmmState};
+use crate::{
+    device::DeviceType, utils, HybridVsockConfig, HybridVsockDevice, VcpuThreadIds, VmmState,
+};
 use shim_interface::KATA_PATH;
 const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
 
@@ -32,10 +34,12 @@ impl DragonballInner {
 
         // prepare vsock
         let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/");
-        let d = crate::driver::DeviceConfig::HybridVsock(crate::driver::HybridVsockConfig {
+        let d = DeviceType::HybridVsock(HybridVsockDevice {
             id: format!("vsock-{}", &self.id),
-            guest_cid: 3,
-            uds_path,
+            config: HybridVsockConfig {
+                guest_cid: 3,
+                uds_path,
+            },
         });
 
         self.add_device(d).await.context("add device")?;
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
index c187911b4..c6df95cc9 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs
@@ -20,7 +20,7 @@ use kata_types::capabilities::Capabilities;
 use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 use tokio::sync::RwLock;
 
-use crate::{driver::DeviceConfig, Hypervisor, VcpuThreadIds};
+use crate::{DeviceType, Hypervisor, VcpuThreadIds};
 
 pub struct Dragonball {
     inner: Arc<RwLock<DragonballInner>>,
@@ -77,12 +77,12 @@ impl Hypervisor for Dragonball {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
+    async fn add_device(&self, device: DeviceType) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
+    async fn remove_device(&self, device: DeviceType) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs
index a6e5566bb..9465d9a9d 100644
--- a/src/runtime-rs/crates/hypervisor/src/lib.rs
+++ b/src/runtime-rs/crates/hypervisor/src/lib.rs
@@ -11,8 +11,8 @@ logging::logger_with_subsystem!(sl, "hypervisor");
 
 pub mod device;
 pub mod hypervisor_persist;
-use device::driver;
 pub use device::driver::*;
+use device::DeviceType;
 pub mod dragonball;
 mod kernel_param;
 pub mod qemu;
@@ -79,8 +79,8 @@ pub trait Hypervisor: Send + Sync {
     async fn resume_vm(&self) -> Result<()>;
 
     // device manager
-    async fn add_device(&self, device: driver::DeviceConfig) -> Result<()>;
-    async fn remove_device(&self, device: driver::DeviceConfig) -> Result<()>;
+    async fn add_device(&self, device: DeviceType) -> Result<()>;
+    async fn remove_device(&self, device: DeviceType) -> Result<()>;
 
     // utils
     async fn get_agent_socket(&self) -> Result<String>;
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
index 8043fd3ba..456bf6373 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
@@ -133,16 +133,16 @@ impl QemuInner {
     }
 }
 
-use crate::driver::DeviceConfig;
+use crate::device::DeviceType;
 
 // device manager part of Hypervisor
 impl QemuInner {
-    pub(crate) async fn add_device(&mut self, device: DeviceConfig) -> Result<()> {
+    pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> {
         info!(sl!(), "QemuInner::add_device() {}", device);
         todo!()
     }
 
-    pub(crate) async fn remove_device(&mut self, device: DeviceConfig) -> Result<()> {
+    pub(crate) async fn remove_device(&mut self, device: DeviceType) -> Result<()> {
         info!(sl!(), "QemuInner::remove_device() {} ", device);
         todo!()
     }
diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
index 4f638cfae..77217f153 100644
--- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
+++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs
@@ -5,7 +5,7 @@
 
 mod inner;
 
-use crate::driver::DeviceConfig;
+use crate::device::DeviceType;
 use crate::hypervisor_persist::HypervisorState;
 use crate::Hypervisor;
 use crate::{HypervisorConfig, VcpuThreadIds};
@@ -73,12 +73,12 @@ impl Hypervisor for Qemu {
         inner.save_vm().await
     }
 
-    async fn add_device(&self, device: DeviceConfig) -> Result<()> {
+    async fn add_device(&self, device: DeviceType) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.add_device(device).await
     }
 
-    async fn remove_device(&self, device: DeviceConfig) -> Result<()> {
+    async fn remove_device(&self, device: DeviceType) -> Result<()> {
         let mut inner = self.inner.write().await;
         inner.remove_device(device).await
     }
diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs
index f422c2056..7e50485ae 100644
--- a/src/runtime-rs/crates/resource/src/manager_inner.rs
+++ b/src/runtime-rs/crates/resource/src/manager_inner.rs
@@ -10,7 +10,10 @@ use crate::{network::NetworkConfig, resource_persist::ResourceState};
 use agent::{types::Device, Agent, Storage};
 use anyhow::{anyhow, Context, Ok, Result};
 use async_trait::async_trait;
-use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig, Hypervisor};
+use hypervisor::{
+    device::{device_manager::DeviceManager, DeviceConfig},
+    BlockConfig, Hypervisor,
+};
 use kata_types::config::TomlConfig;
 use kata_types::mount::Mount;
 use oci::{Linux, LinuxResources};
@@ -255,7 +258,7 @@ impl ResourceManagerInner {
         for d in linux.devices.iter() {
             match d.r#type.as_str() {
                 "b" => {
-                    let device_info = DeviceConfig::Block(BlockConfig {
+                    let device_info = DeviceConfig::BlockCfg(BlockConfig {
                         major: d.major,
                         minor: d.minor,
                         ..Default::default()
@@ -285,7 +288,7 @@ impl ResourceManagerInner {
                         .context("failed to get device info")?;
 
                     // create agent device
-                    if let DeviceConfig::Block(config) = dev_info {
+                    if let DeviceConfig::BlockCfg(config) = dev_info {
                         let agent_device = Device {
                             id: device_id.clone(),
                             container_path: d.path.clone(),
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
index cea445e2f..a8f08088b 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs
@@ -9,11 +9,13 @@ use std::io::{self, Error};
 use super::endpoint_persist::{EndpointState, IpVlanEndpointState};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
+use hypervisor::device::DeviceType;
+use hypervisor::NetworkDevice;
 
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
-use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, Hypervisor};
 
 // IPVlanEndpoint is the endpoint bridged to VM
 #[derive(Debug)]
@@ -44,7 +46,6 @@ impl IPVlanEndpoint {
             )
         })?;
         Ok(NetworkConfig {
-            id: self.net_pair.virt_iface.name.clone(),
             host_dev_name: iface.name.clone(),
             guest_mac: Some(guest_mac),
         })
@@ -67,9 +68,12 @@ impl Endpoint for IPVlanEndpoint {
             .await
             .context("error adding network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(DeviceConfig::Network(config))
-            .await
-            .context("error adding device by hypervisor")?;
+        h.add_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error adding device by hypervisor")?;
 
         Ok(())
     }
@@ -82,9 +86,12 @@ impl Endpoint for IPVlanEndpoint {
         let config = self
             .get_network_config()
             .context("error getting network config")?;
-        h.remove_device(DeviceConfig::Network(config))
-            .await
-            .context("error removing device by hypervisor")?;
+        h.remove_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error removing device by hypervisor")?;
 
         Ok(())
     }
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
index 7f31c028b..13716e877 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs
@@ -11,7 +11,9 @@ use super::Endpoint;
 use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
+use hypervisor::device::DeviceType;
+use hypervisor::NetworkDevice;
+use hypervisor::{device::driver::NetworkConfig, Hypervisor};
 
 #[derive(Debug)]
 pub struct MacVlanEndpoint {
@@ -41,7 +43,6 @@ impl MacVlanEndpoint {
             )
         })?;
         Ok(NetworkConfig {
-            id: self.net_pair.virt_iface.name.clone(),
             host_dev_name: iface.name.clone(),
             guest_mac: Some(guest_mac),
         })
@@ -64,9 +65,13 @@ impl Endpoint for MacVlanEndpoint {
             .await
             .context("add network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(DeviceConfig::Network(config))
-            .await
-            .context("Error add device")?;
+        h.add_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error adding device by hypervisor")?;
+
         Ok(())
     }
 
@@ -76,9 +81,13 @@ impl Endpoint for MacVlanEndpoint {
             .await
             .context("del network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.remove_device(DeviceConfig::Network(config))
-            .await
-            .context("remove device")?;
+        h.remove_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error removing device by hypervisor")?;
+
         Ok(())
     }
 
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
index 9f3ccaf90..4db8b865f 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs
@@ -8,7 +8,9 @@ use std::path::Path;
 
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
+use hypervisor::device::DeviceType;
 use hypervisor::{device::driver, Hypervisor};
+use hypervisor::{VfioConfig, VfioDevice};
 
 use super::endpoint_persist::{EndpointState, PhysicalEndpointState};
 use super::Endpoint;
@@ -108,12 +110,14 @@ impl Endpoint for PhysicalEndpoint {
         };
 
         // add vfio device
-        let d = driver::DeviceConfig::Vfio(driver::VfioConfig {
+        let d = DeviceType::Vfio(VfioDevice {
             id: format!("physical_nic_{}", self.name().await),
-            sysfs_path: "".to_string(),
-            bus_slot_func: self.bdf.clone(),
-            mode: driver::VfioBusMode::new(mode)
-                .with_context(|| format!("new vfio bus mode {:?}", mode))?,
+            config: VfioConfig {
+                sysfs_path: "".to_string(),
+                bus_slot_func: self.bdf.clone(),
+                mode: driver::VfioBusMode::new(mode)
+                    .with_context(|| format!("new vfio bus mode {:?}", mode))?,
+            },
         });
         hypervisor.add_device(d).await.context("add device")?;
         Ok(())
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
index 7e822f764..4415e6d29 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs
@@ -11,7 +11,9 @@ use super::Endpoint;
 use crate::network::{utils, NetworkPair};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
+use hypervisor::device::DeviceType;
+use hypervisor::NetworkDevice;
+use hypervisor::{device::driver::NetworkConfig, Hypervisor};
 
 #[derive(Debug)]
 pub struct VethEndpoint {
@@ -41,7 +43,6 @@ impl VethEndpoint {
             )
         })?;
         Ok(NetworkConfig {
-            id: self.net_pair.virt_iface.name.clone(),
             host_dev_name: iface.name.clone(),
             guest_mac: Some(guest_mac),
         })
@@ -64,9 +65,12 @@ impl Endpoint for VethEndpoint {
             .await
             .context("add network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(DeviceConfig::Network(config))
-            .await
-            .context("Error add device")?;
+        h.add_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error adding device by hypervisor")?;
         Ok(())
     }
 
@@ -76,9 +80,12 @@ impl Endpoint for VethEndpoint {
             .await
             .context("del network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.remove_device(DeviceConfig::Network(config))
-            .await
-            .context("remove device")?;
+        h.remove_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error removing device by hypervisor")?;
         Ok(())
     }
     async fn save(&self) -> Option<EndpointState> {
diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
index 7ece7d168..6aca674be 100644
--- a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
+++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs
@@ -8,12 +8,14 @@ use std::io::{self, Error};
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
+use hypervisor::device::DeviceType;
+use hypervisor::NetworkDevice;
 
 use super::endpoint_persist::{EndpointState, VlanEndpointState};
 use super::Endpoint;
 use crate::network::network_model::TC_FILTER_NET_MODEL_STR;
 use crate::network::{utils, NetworkPair};
-use hypervisor::{device::driver::NetworkConfig, DeviceConfig, Hypervisor};
+use hypervisor::{device::driver::NetworkConfig, Hypervisor};
 #[derive(Debug)]
 pub struct VlanEndpoint {
     pub(crate) net_pair: NetworkPair,
@@ -41,7 +43,6 @@ impl VlanEndpoint {
             )
         })?;
         Ok(NetworkConfig {
-            id: self.net_pair.virt_iface.name.clone(),
             host_dev_name: iface.name.clone(),
             guest_mac: Some(guest_mac),
         })
@@ -64,9 +65,12 @@ impl Endpoint for VlanEndpoint {
             .await
             .context("error adding network model")?;
         let config = self.get_network_config().context("get network config")?;
-        h.add_device(DeviceConfig::Network(config))
-            .await
-            .context("error adding device by hypervisor")?;
+        h.add_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error adding device by hypervisor")?;
 
         Ok(())
     }
@@ -79,9 +83,12 @@ impl Endpoint for VlanEndpoint {
         let config = self
             .get_network_config()
             .context("error getting network config")?;
-        h.remove_device(DeviceConfig::Network(config))
-            .await
-            .context("error removing device by hypervisor")?;
+        h.remove_device(DeviceType::Network(NetworkDevice {
+            id: self.net_pair.virt_iface.name.clone(),
+            config,
+        }))
+        .await
+        .context("error removing device by hypervisor")?;
 
         Ok(())
     }
diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
index 2c2568500..0e161fcb3 100644
--- a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
+++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs
@@ -9,7 +9,10 @@ use crate::share_fs::{do_get_guest_path, do_get_host_path};
 use agent::Storage;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
-use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig};
+use hypervisor::{
+    device::{device_manager::DeviceManager, DeviceConfig},
+    BlockConfig,
+};
 use kata_types::mount::Mount;
 use nix::sys::stat::{self, SFlag};
 use std::fs;
@@ -46,7 +49,7 @@ impl BlockRootfs {
         let device_id = d
             .write()
             .await
-            .new_device(&DeviceConfig::Block(block_device_config.clone()))
+            .new_device(&DeviceConfig::BlockCfg(block_device_config.clone()))
             .await
             .context("failed to create deviec")?;
 
@@ -71,7 +74,7 @@ impl BlockRootfs {
             .await
             .context("failed to get device info")?;
 
-        if let DeviceConfig::Block(config) = dev_info {
+        if let DeviceConfig::BlockCfg(config) = dev_info {
             storage.driver = config.driver_option;
             storage.source = config.virt_path;
         }
diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs
index cebce769a..81ab8b7fd 100644
--- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs
+++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs
@@ -8,8 +8,12 @@ use std::path::Path;
 
 use anyhow::{Context, Result};
 use hypervisor::{
-    device::driver::{
-        DeviceConfig as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation,
+    device::{
+        driver::{
+            ShareFsDevice, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType,
+            ShareFsOperation,
+        },
+        DeviceType,
     },
     Hypervisor, ShareFsDeviceConfig,
 };
@@ -44,15 +48,19 @@ pub(crate) async fn prepare_virtiofs(
     mount::bind_mount_unchecked(&host_rw_dest, &host_ro_dest, true)
         .context("bind mount shared_fs directory")?;
 
-    let share_fs_device = HypervisorDevice::ShareFsDevice(ShareFsDeviceConfig {
-        sock_path: generate_sock_path(root),
-        mount_tag: String::from(MOUNT_GUEST_TAG),
-        host_path: String::from(host_ro_dest.to_str().unwrap()),
-        fs_type: fs_type.to_string(),
-        queue_size: 0,
-        queue_num: 0,
-    });
-    h.add_device(share_fs_device).await.context("add device")?;
+    let share_fs_device = ShareFsDevice {
+        config: ShareFsDeviceConfig {
+            sock_path: generate_sock_path(root),
+            mount_tag: String::from(MOUNT_GUEST_TAG),
+            host_path: String::from(host_ro_dest.to_str().unwrap()),
+            fs_type: fs_type.to_string(),
+            queue_size: 0,
+            queue_num: 0,
+        },
+    };
+    h.add_device(DeviceType::ShareFs(share_fs_device))
+        .await
+        .context("add device")?;
     Ok(())
 }
 
@@ -68,16 +76,18 @@ pub(crate) async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Resul
     let ro_source = utils::get_host_ro_shared_path(id).join(PASSTHROUGH_FS_DIR);
     let source = String::from(ro_source.to_str().unwrap());
 
-    let virtio_fs = HypervisorDevice::ShareFsMount(ShareFsMountConfig {
-        source: source.clone(),
-        fstype: ShareFsMountType::PASSTHROUGH,
-        mount_point: mnt,
-        config: None,
-        tag: String::from(MOUNT_GUEST_TAG),
-        op: ShareFsOperation::Mount,
-        prefetch_list_path: None,
-    });
-    h.add_device(virtio_fs)
+    let virtio_fs = ShareFsMountDevice {
+        config: ShareFsMountConfig {
+            source: source.clone(),
+            fstype: ShareFsMountType::PASSTHROUGH,
+            mount_point: mnt,
+            config: None,
+            tag: String::from(MOUNT_GUEST_TAG),
+            op: ShareFsOperation::Mount,
+            prefetch_list_path: None,
+        },
+    };
+    h.add_device(DeviceType::ShareFsMount(virtio_fs))
         .await
         .with_context(|| format!("fail to attach passthrough fs {:?}", source))
 }
@@ -93,16 +103,18 @@ pub async fn rafs_mount(
         sl!(),
         "Attaching rafs meta file {} to virtio-fs device, rafs mount point {}", rafs_meta, rafs_mnt
     );
-    let virtio_fs = HypervisorDevice::ShareFsMount(ShareFsMountConfig {
-        source: rafs_meta.clone(),
-        fstype: ShareFsMountType::RAFS,
-        mount_point: rafs_mnt,
-        config: Some(config_content),
-        tag: String::from(MOUNT_GUEST_TAG),
-        op: ShareFsOperation::Mount,
-        prefetch_list_path,
-    });
-    h.add_device(virtio_fs)
+    let virtio_fs = ShareFsMountDevice {
+        config: ShareFsMountConfig {
+            source: rafs_meta.clone(),
+            fstype: ShareFsMountType::RAFS,
+            mount_point: rafs_mnt,
+            config: Some(config_content),
+            tag: String::from(MOUNT_GUEST_TAG),
+            op: ShareFsOperation::Mount,
+            prefetch_list_path,
+        },
+    };
+    h.add_device(DeviceType::ShareFsMount(virtio_fs))
         .await
         .with_context(|| format!("fail to attach rafs {:?}", rafs_meta))?;
     Ok(())
diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs
index 079465997..14f09fc81 100644
--- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs
+++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs
@@ -13,7 +13,10 @@ use crate::share_fs::{do_get_guest_path, do_get_host_path};
 use super::{share_fs_volume::generate_mount_path, Volume};
 use agent::Storage;
 use anyhow::{anyhow, Context};
-use hypervisor::{device::device_manager::DeviceManager, BlockConfig, DeviceConfig};
+use hypervisor::{
+    device::{device_manager::DeviceManager, DeviceConfig},
+    BlockConfig,
+};
 use nix::sys::stat::{self, SFlag};
 use tokio::sync::RwLock;
 #[derive(Debug)]
@@ -48,7 +51,7 @@ impl BlockVolume {
         let device_id = d
             .write()
             .await
-            .new_device(&DeviceConfig::Block(block_device_config.clone()))
+            .new_device(&DeviceConfig::BlockCfg(block_device_config.clone()))
             .await
             .context("failed to create deviec")?;
 
@@ -76,7 +79,7 @@ impl BlockVolume {
         // storage
         let mut storage = Storage::default();
 
-        if let DeviceConfig::Block(config) = dev_info {
+        if let DeviceConfig::BlockCfg(config) = dev_info {
             storage.driver = config.driver_option;
             storage.source = config.virt_path;
         }

From 4719802c8d9054714d0ccd0caafbf78a4e4730c6 Mon Sep 17 00:00:00 2001
From: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Date: Sat, 20 May 2023 18:38:18 +0800
Subject: [PATCH 141/150] runtime-rs: add virtio-blk-mmio

add virtio-blk-mmio option for dragonball

Fixes:#5375
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>
Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
---
 src/libs/kata-types/src/config/hypervisor/dragonball.rs  | 4 ++--
 src/libs/kata-types/src/config/hypervisor/mod.rs         | 6 +++---
 src/runtime-rs/Makefile                                  | 2 +-
 src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs | 3 ++-
 src/runtime-rs/crates/hypervisor/src/kernel_param.rs     | 4 ++--
 src/runtime-rs/crates/hypervisor/src/lib.rs              | 3 ++-
 6 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs
index bb72944a2..405731945 100644
--- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs
+++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs
@@ -12,7 +12,7 @@ use super::{default, register_hypervisor_plugin};
 use crate::config::default::MAX_DRAGONBALL_VCPUS;
 use crate::config::default::MIN_DRAGONBALL_MEMORY_SIZE_MB;
 use crate::config::hypervisor::{
-    VIRTIO_BLK, VIRTIO_BLK_MMIO, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM,
+    VIRTIO_BLK_MMIO, VIRTIO_BLK_PCI, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM,
 };
 use crate::config::{ConfigPlugin, TomlConfig};
 use crate::{eother, resolve_path, validate_path};
@@ -107,7 +107,7 @@ impl ConfigPlugin for DragonballConfig {
             }
 
             if !db.blockdev_info.disable_block_device_use
-                && db.blockdev_info.block_device_driver != VIRTIO_BLK
+                && db.blockdev_info.block_device_driver != VIRTIO_BLK_PCI
                 && db.blockdev_info.block_device_driver != VIRTIO_BLK_MMIO
                 && db.blockdev_info.block_device_driver != VIRTIO_PMEM
             {
diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs
index 7818b897c..fedbba14b 100644
--- a/src/libs/kata-types/src/config/hypervisor/mod.rs
+++ b/src/libs/kata-types/src/config/hypervisor/mod.rs
@@ -43,8 +43,8 @@ pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU};
 mod ch;
 pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH};
 
-const VIRTIO_BLK: &str = "virtio-blk";
-const VIRTIO_BLK_MMIO: &str = "virtio-mmio";
+const VIRTIO_BLK_PCI: &str = "virtio-blk-pci";
+const VIRTIO_BLK_MMIO: &str = "virtio-blk-mmio";
 const VIRTIO_BLK_CCW: &str = "virtio-blk-ccw";
 const VIRTIO_SCSI: &str = "virtio-scsi";
 const VIRTIO_PMEM: &str = "nvdimm";
@@ -172,7 +172,7 @@ impl BlockDeviceInfo {
             return Ok(());
         }
         let l = [
-            VIRTIO_BLK,
+            VIRTIO_BLK_PCI,
             VIRTIO_BLK_CCW,
             VIRTIO_BLK_MMIO,
             VIRTIO_PMEM,
diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile
index afe974bf9..5304290ca 100644
--- a/src/runtime-rs/Makefile
+++ b/src/runtime-rs/Makefile
@@ -203,7 +203,7 @@ ifneq (,$(DBCMD))
     CONFIGS += $(CONFIG_DB)
     # dragonball-specific options (all should be suffixed by "_DB")
     DEFMAXVCPUS_DB := 1
-    DEFBLOCKSTORAGEDRIVER_DB := virtio-blk
+    DEFBLOCKSTORAGEDRIVER_DB := virtio-blk-mmio
     DEFNETWORKMODEL_DB := tcfilter
     KERNELPARAMS = console=ttyS1 agent.log_vport=1025
 	KERNELTYPE_DB = uncompressed
diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
index 998020d21..45b77f09e 100644
--- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
+++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
@@ -8,6 +8,7 @@ use super::vmm_instance::VmmInstance;
 use crate::{
     device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
     DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK,
+    VM_ROOTFS_DRIVER_MMIO,
 };
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
@@ -265,7 +266,7 @@ impl DragonballInner {
             .get_resource(path, DRAGONBALL_ROOT_FS)
             .context("get resource")?;
 
-        if driver == VM_ROOTFS_DRIVER_BLK {
+        if driver == VM_ROOTFS_DRIVER_BLK || driver == VM_ROOTFS_DRIVER_MMIO {
             let blk_cfg = BlockDeviceConfigInfo {
                 path_on_host: PathBuf::from(jail_drive),
                 drive_id: DRAGONBALL_ROOT_FS.to_string(),
diff --git a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs
index 7ad17cb8f..554d61660 100644
--- a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs
+++ b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs
@@ -7,7 +7,7 @@
 use anyhow::{anyhow, Result};
 
 use crate::{
-    VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_PMEM, VM_ROOTFS_FILESYSTEM_EROFS,
+    VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_MMIO, VM_ROOTFS_DRIVER_PMEM, VM_ROOTFS_FILESYSTEM_EROFS,
     VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS, VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM,
 };
 use kata_types::config::LOG_VPORT_OPTION;
@@ -91,7 +91,7 @@ impl KernelParams {
                     }
                 }
             }
-            VM_ROOTFS_DRIVER_BLK => {
+            VM_ROOTFS_DRIVER_BLK | VM_ROOTFS_DRIVER_MMIO => {
                 params.push(Param::new("root", VM_ROOTFS_ROOT_BLK));
                 match rootfs_type {
                     VM_ROOTFS_FILESYSTEM_EXT4 | VM_ROOTFS_FILESYSTEM_XFS => {
diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs
index 9465d9a9d..2001433e5 100644
--- a/src/runtime-rs/crates/hypervisor/src/lib.rs
+++ b/src/runtime-rs/crates/hypervisor/src/lib.rs
@@ -32,8 +32,9 @@ use kata_types::config::hypervisor::Hypervisor as HypervisorConfig;
 pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH;
 
 // Config which driver to use as vm root dev
-const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk";
+const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk-pci";
 const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem";
+const VM_ROOTFS_DRIVER_MMIO: &str = "virtio-blk-mmio";
 
 //Configure the root corresponding to the driver
 const VM_ROOTFS_ROOT_BLK: &str = "/dev/vda1";

From fc09d0f5dd1d2f83ec75532e7a3534579e8337fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 23 May 2023 09:06:44 +0200
Subject: [PATCH 142/150] release: Kata Containers 3.2.0-alpha2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Fix cache for OVMF and rootfs-initrd (both x86_64)
- Upgrade to Cloud Hypervisor v32.0
- osbuilder: Bump fedora image version
- local-build: Standardise what's set for the local build scripts
- gha: aks: Wait a little bit more before run the tests
- docs: Update container network model url
- gha: release: Fix s390x worklow
- cache: Fix OVMF caching
- gha: payload-after-push: Pass secrets down
- tools: Fix arch bug

22154e0a3 cache: Fix OVMF tarball name for different flavours
b7341cd96 cache: Use "initrd" as `initrd_type` to build rootfs-initrd
b8ffcd1b9 osbuilder: Bump fedora image version
636539bf0 kata-deploy: Use apt-key.gpg from k8s.io
ae24dc73c local-build: Standardise what's set for the local build scripts
35c3d7b4b runtime: clh: Re-generate the client code
cfee99c57 versions: Upgrade to Cloud Hypervisor v32.0
ad324adf1 gha: aks: Wait a little bit more before run the tests
191b6dd9d gha: release: Fix s390x worklow
cfd8f4ff7 gha: payload-after-push: Pass secrets down
75330ab3f cache: Fix OVMF caching
a89b44aab tools: Fix arch bug
11a34a72e docs: Update container network model url

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index dcd25b873..b79892313 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.2.0-alpha1
+3.2.0-alpha2

From f3702268d1adaf342f4c79e70d49ea6044971152 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 23 May 2023 14:11:03 +0200
Subject: [PATCH 143/150] release: Fix `docker/login-action` version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`docker/login-action@v3` does *not* exist and `docker/login-action@v2`
should be used instead.

Fixes: #6934

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/release-amd64.yaml | 4 ++--
 .github/workflows/release-arm64.yaml | 4 ++--
 .github/workflows/release-s390x.yaml | 4 ++--
 .github/workflows/release.yaml       | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml
index 3e897d051..358921cef 100644
--- a/.github/workflows/release-amd64.yaml
+++ b/.github/workflows/release-amd64.yaml
@@ -15,14 +15,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml
index 2d54d0b62..fdafd1eb5 100644
--- a/.github/workflows/release-arm64.yaml
+++ b/.github/workflows/release-arm64.yaml
@@ -15,14 +15,14 @@ jobs:
     runs-on: arm64
     steps:
       - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml
index e9fc5fa7e..4238fa36b 100644
--- a/.github/workflows/release-s390x.yaml
+++ b/.github/workflows/release-s390x.yaml
@@ -15,14 +15,14 @@ jobs:
     runs-on: s390x
     steps:
       - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 020804beb..292ac9b29 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -31,14 +31,14 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Login to Kata Containers docker.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}

From 0b1c5ea5bb34b3afa159e20dc457842badbf8577 Mon Sep 17 00:00:00 2001
From: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
Date: Tue, 23 May 2023 15:39:04 +0000
Subject: [PATCH 144/150] versions: Update nydus version to 2.2.1

This PR updates the nydus version to 2.2.1. This change includes:
nydus-image: fix a underflow issue in get_compressed_size()
backport fix/feature to stable 2.2
[backport] contrib: upgrade runc to v1.1.5
service: add README for nydus-service
nydus: fix a possible panic caused by SubCmdArgs::is_present
Backports two bugfixes from master into stable/v2.2
[backport stable/v2.2] action: upgrade golangci-lint to v1.51.2
[backport] action: fix smoke test for branch pattern

Fixes #6938

Signed-off-by: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
---
 versions.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.yaml b/versions.yaml
index 9642541dd..38a6fa397 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -261,7 +261,7 @@ externals:
   nydus:
     description: "Nydus image acceleration service"
     url: "https://github.com/dragonflyoss/image-service"
-    version: "v2.2.0"
+    version: "v2.2.1"
 
   nydus-snapshotter:
     description: "Snapshotter for Nydus image acceleration service"

From c54363114d5fe01fe7d1cf410324eaf940c766a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 23 May 2023 18:39:16 +0200
Subject: [PATCH 145/150] release: Kata Containers 3.2.0-alpha3
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- release: Fix `docker/login-action` version

f3702268d release: Fix `docker/login-action` version

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index b79892313..bb48c8b0a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.2.0-alpha2
+3.2.0-alpha3

From d10c9be6031474b46809d7c5a490037fa0ec7561 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 23 May 2023 22:38:12 +0200
Subject: [PATCH 146/150] gha: release: login-action: Don't specify docker.io
 registry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For some bizarre reason, the login-action will simply fail to
authenticate to docker.io in it's specified as a registry.  The way to
proceed, instead, is to *not* specify any registry as it'd be used by
default.

Fixes: #6943

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/release-amd64.yaml | 1 -
 .github/workflows/release-arm64.yaml | 1 -
 .github/workflows/release-s390x.yaml | 1 -
 .github/workflows/release.yaml       | 1 -
 4 files changed, 4 deletions(-)

diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml
index 358921cef..8545f7c0d 100644
--- a/.github/workflows/release-amd64.yaml
+++ b/.github/workflows/release-amd64.yaml
@@ -17,7 +17,6 @@ jobs:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
         with:
-          registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml
index fdafd1eb5..a16b070e4 100644
--- a/.github/workflows/release-arm64.yaml
+++ b/.github/workflows/release-arm64.yaml
@@ -17,7 +17,6 @@ jobs:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
         with:
-          registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml
index 4238fa36b..66da2f975 100644
--- a/.github/workflows/release-s390x.yaml
+++ b/.github/workflows/release-s390x.yaml
@@ -17,7 +17,6 @@ jobs:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
         with:
-          registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 292ac9b29..af9b93132 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -33,7 +33,6 @@ jobs:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
         with:
-          registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 

From f636c1f8a49095026f634699fac76257eb6be1ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 24 May 2023 08:54:43 +0200
Subject: [PATCH 147/150] gha: release: Simplify the process for tagging the
 payload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We previously were doing:
* Create a new image on kata-deploy-ci using the commit hash of the
  latest tag
  * This was used to test on AKS, which is no longer needed as we test
    on AKS on every PR
* Create a new image on kata-deploy using the release tag and "latest"
  or "stable", by tagging the kata-deploy-ci image accordingly

As part of cfe63527c5328611cac647d0c10916bef6cf0408, we broke the
workflow described above, as in the first step we would save the PKG_SHA
to be used in the second step, but that part ended up being removed.

Anyways, this back and forth is not needed anymore and we can simplify
the process by doing:
* Create a new image on kata-deploy, using:
  - The tag received as ref from the event that triggered this worklow
  - "latest" or "stable" tag, depending on whether it's a stable release
    or not

Fixes: #6946

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .github/workflows/release-amd64.yaml | 27 ++++++++-------------------
 .github/workflows/release-arm64.yaml | 27 ++++++++-------------------
 .github/workflows/release-s390x.yaml | 27 ++++++++-------------------
 3 files changed, 24 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml
index 8545f7c0d..8d48b956b 100644
--- a/.github/workflows/release-amd64.yaml
+++ b/.github/workflows/release-amd64.yaml
@@ -36,27 +36,16 @@ jobs:
       - name: build-and-push-kata-deploy-ci-amd64
         id: build-and-push-kata-deploy-ci-amd64
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
-            "${pkg_sha}-${{ inputs.target-arch }}"
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
-            "${pkg_sha}-${{ inputs.target-arch }}"
-
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
           for tag in ${tags[@]}; do
-            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-amd64.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-amd64.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
           done
diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml
index a16b070e4..2b5e810a3 100644
--- a/.github/workflows/release-arm64.yaml
+++ b/.github/workflows/release-arm64.yaml
@@ -36,27 +36,16 @@ jobs:
       - name: build-and-push-kata-deploy-ci-arm64
         id: build-and-push-kata-deploy-ci-arm64
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
-            "${pkg_sha}-${{ inputs.target-arch }}"
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
-            "${pkg_sha}-${{ inputs.target-arch }}"
-
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
           for tag in ${tags[@]}; do
-            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-arm64.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-arm64.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
           done
diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml
index 66da2f975..ef436b7b8 100644
--- a/.github/workflows/release-s390x.yaml
+++ b/.github/workflows/release-s390x.yaml
@@ -36,27 +36,16 @@ jobs:
       - name: build-and-push-kata-deploy-ci-s390x
         id: build-and-push-kata-deploy-ci-s390x
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-            $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy-ci" \
-            "${pkg_sha}-${{ inputs.target-arch }}"
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-            $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy-ci" \
-            "${pkg_sha}-${{ inputs.target-arch }}"
-
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
+          # We need to do such trick here as the format of the $GITHUB_REF 
+          # is "refs/tags/<tag>"
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tags=($tag)
           tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
           for tag in ${tags[@]}; do
-            docker tag docker.io/katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-s390x.outputs.PKG_SHA}}-${{ inputs.target-arch }} docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci-s390x.outputs.PKG_SHA}}-${{ inputs.target-arch }} quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker push docker.io/katadocker/kata-deploy:${tag}-${{ inputs.target-arch }}
-            docker push quay.io/kata-containers/kata-deploy:${tag}-${{ inputs.target-arch }}
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
           done

From 6a0035e419253cea937cafc43bbdf18a4b4ddc79 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Wed, 24 May 2023 13:16:48 +0100
Subject: [PATCH 148/150] doc: Update git commands

Fix bad migrations from `go get` to `git clone` and update the cloned
directory path

Fixes: #6951
Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 tools/packaging/kata-deploy/README.md | 8 ++++----
 tools/packaging/kernel/README.md      | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md
index 5a8b9cc1a..9e546e4f5 100644
--- a/tools/packaging/kata-deploy/README.md
+++ b/tools/packaging/kata-deploy/README.md
@@ -16,13 +16,13 @@ be utilized to install Kata Containers on a running Kubernetes cluster.
 For your [k3s](https://k3s.io/) cluster, run:
 
 ```sh
-$ git clone github.com/kata-containers/kata-containers
+$ git clone https://github.com/kata-containers/kata-containers.git
 ```
 
 Check and switch to the stable branch of your choice, if wanted, and then run:
 
 ```bash
-$ cd kata-containers/kata-containers/tools/packaging/kata-deploy
+$ cd kata-containers/tools/packaging/kata-deploy
 $ kubectl apply -f kata-rbac/base/kata-rbac.yaml
 $ kubectl apply -k kata-deploy/overlays/k3s
 ```
@@ -32,13 +32,13 @@ $ kubectl apply -k kata-deploy/overlays/k3s
 For your [RKE2](https://docs.rke2.io/) cluster, run:
 
 ```sh
-$ git clone github.com/kata-containers/kata-containers
+$ git clone https://github.com/kata-containers/kata-containers.git
 ```
 
 Check and switch to the stable branch of your choice, if wanted, and then run:
 
 ```bash
-$ cd kata-containers/kata-containers/tools/packaging/kata-deploy
+$ cd kata-containers/tools/packaging/kata-deploy
 $ kubectl apply -f kata-rbac/base/kata-rbac.yaml
 $ kubectl apply -k kata-deploy/overlays/rke2
 ```
diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md
index d9b78480a..5c8580a68 100644
--- a/tools/packaging/kernel/README.md
+++ b/tools/packaging/kernel/README.md
@@ -70,7 +70,7 @@ $ ./build-kernel.sh -v 5.10.25 -g nvidia -f -d setup
 ## Setup kernel source code
 
 ```bash
-$ git clone github.com/kata-containers/kata-containers
+$ git clone https://github.com/kata-containers/kata-containers.git
 $ cd kata-containers/tools/packaging/kernel
 $ ./build-kernel.sh setup
 ```

From 14c3f1e9f5bb051c5945f7d9b32f642ad216f306 Mon Sep 17 00:00:00 2001
From: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
Date: Mon, 22 May 2023 15:43:25 +0000
Subject: [PATCH 149/150] kata-deploy: Fix indentation on kata deploy merge
 script

This PR fixes the indentation on the kata deploy merge script
that instead of single spaces uses a tap.

Fixes #6925

Signed-off-by: Gabriela Cervantes <gabriela.cervantes.tellez@intel.com>
---
 .../kata-deploy/local-build/kata-deploy-merge-builds.sh       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh
index dea0f3b89..b2d02b43f 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh
@@ -20,8 +20,8 @@ mkdir "${tarball_content_dir}"
 
 for c in kata-static-*.tar.xz
 do
-    echo "untarring tarball "${c}" into ${tarball_content_dir}"
-    tar -xvf "${c}" -C "${tarball_content_dir}"
+	echo "untarring tarball "${c}" into ${tarball_content_dir}"
+	tar -xvf "${c}" -C "${tarball_content_dir}"
 done
 
 echo "create ${tar_path}"

From 428041624ae44b4cbfd24a37870201b911519630 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Wed, 24 May 2023 18:39:27 +0200
Subject: [PATCH 150/150] kata-deploy: Improve shim backup / restore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We're currently backing up and restoring all the possible shim files,
but the default one ("containerd-shim-kata-v2").

Let's ensure this is also backed up and restored.

Fixes: #6957

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../kata-deploy/scripts/kata-deploy.sh        | 49 +++++++++++++------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 820ac9d5d..6bb660198 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -92,6 +92,20 @@ function configure_cri_runtime() {
 	wait_till_node_is_ready
 }
 
+function backup_shim() {
+	local shim_file="$1"
+	local shim_backup="${shim_file}.bak"
+
+	if [ -f "${shim_file}" ]; then
+		echo "warning: ${shim_file} already exists" >&2
+		if [ ! -f "${shim_backup}" ]; then
+			mv "${shim_file}" "${shim_backup}"
+		else
+			rm "${shim_file}"
+		fi
+	fi
+}
+
 function configure_different_shims_base() {
 	# Currently containerd has an assumption on the location of the shimv2 implementation
 	# This forces kata-deploy to create files in a well-defined location that's part of
@@ -100,21 +114,15 @@ function configure_different_shims_base() {
 	#   https://github.com/containerd/containerd/issues/3073
 	#   https://github.com/containerd/containerd/issues/5006
 
+	local default_shim_file="/usr/local/bin/containerd-shim-kata-v2"
+
 	mkdir -p /usr/local/bin
 
 	for shim in "${shims[@]}"; do
 		local shim_binary="containerd-shim-kata-${shim}-v2"
 		local shim_file="/usr/local/bin/${shim_binary}"
-		local shim_backup="/usr/local/bin/${shim_binary}.bak"
 
-		if [ -f "${shim_file}" ]; then
-			echo "warning: ${shim_binary} already exists" >&2
-			if [ ! -f "${shim_backup}" ]; then
-				mv "${shim_file}" "${shim_backup}"
-			else
-				rm "${shim_file}"
-			fi
-		fi
+		backup_shim "${shim_file}"
 
 		if [[ "${shim}" == "dragonball" ]]; then
 			ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}"
@@ -124,26 +132,37 @@ function configure_different_shims_base() {
 		chmod +x "$shim_file"
 
 		if [ "${shim}" == "${default_shim}" ]; then
+			backup_shim "${default_shim_file}"
+
 			echo "Creating the default shim-v2 binary"
-			ln -sf "${shim_file}" /usr/local/bin/containerd-shim-kata-v2
+			ln -sf "${shim_file}" "${default_shim_file}"
 		fi
 	done
 }
 
+function restore_shim() {
+	local shim_file="$1"
+	local shim_backup="${shim_file}.bak"
+
+	if [ -f "${shim_backup}" ]; then
+		mv "$shim_backup" "$shim_file"
+	fi
+}
+
 function cleanup_different_shims_base() {
+	local default_shim_file="/usr/local/bin/containerd-shim-kata-v2"
+
 	for shim in "${shims[@]}"; do
 		local shim_binary="containerd-shim-kata-${shim}-v2"
 		local shim_file="/usr/local/bin/${shim_binary}"
-		local shim_backup="/usr/local/bin/${shim_binary}.bak"
 
 		rm "${shim_file}" || true
 
-		if [ -f "${shim_backup}" ]; then
-			mv "$shim_backup" "$shim_file"
-		fi
+		restore_shim "${shim_file}"
 	done
 
-	rm /usr/local/bin/containerd-shim-kata-v2
+	rm "${default_shim_file}" || true
+	restore_shim "${default_shim_file}"
 }
 
 function configure_crio_runtime() {