CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #6558
Signed-off-by: stevenhorsman <steven@uk.ibm.com>

src/runtime/pkg/containerd-shim-v2/shim_management.go

@@ -243,7 +243,7 @@ func (s *service) genericIPTablesHandler(w http.ResponseWriter, r *http.Request,
 
 func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) {
 	// metrics socket will under sandbox's bundle path
-	metricsAddress := SocketAddress(s.id)
+	metricsAddress := ServerSocketAddress(s.id)
 
 	listener, err := cdshim.NewSocket(metricsAddress)
 	if err != nil {
@@ -312,14 +312,38 @@ func GetSandboxesStoragePathRust() string {
 	return "/run/kata"
 }
 
-// SocketAddress returns the address of the unix domain socket for communicating with the
+// SocketPath returns the path of the socket using the given storagePath
+func SocketPath(id string, storagePath string) string {
+	return filepath.Join(string(filepath.Separator), storagePath, id, "shim-monitor.sock")
+}
+
+// SocketPathGo returns the path of the socket to be used with the go runtime
+func SocketPathGo(id string) string {
+	return SocketPath(id, GetSandboxesStoragePath())
+}
+
+// SocketPathRust returns the path of the socket to be used with the rust runtime
+func SocketPathRust(id string) string {
+	return SocketPath(id, GetSandboxesStoragePathRust())
+}
+
+// ServerSocketAddress returns the address of the unix domain socket the shim management endpoint
+// should listen.
+// NOTE: this code is only called by the go shim management implementation.
+func ServerSocketAddress(id string) string {
+	return fmt.Sprintf("unix://%s", SocketPathGo(id))
+}
+
+// ClientSocketAddress returns the address of the unix domain socket for communicating with the
 // shim management endpoint
-func SocketAddress(id string) string {
+// NOTE: this code allows various go clients, e.g. kata-runtime or kata-monitor commands, to
+// connect to the rust shim management implementation.
+func ClientSocketAddress(id string) string {
 	// get the go runtime uds path
-	socketPath := filepath.Join(string(filepath.Separator), GetSandboxesStoragePath(), id, "shim-monitor.sock")
+	socketPath := SocketPathGo(id)
 	// if the path not exist, use the rust runtime uds path instead
 	if _, err := os.Stat(socketPath); err != nil {
-		return fmt.Sprintf("unix://%s", filepath.Join(string(filepath.Separator), GetSandboxesStoragePathRust(), id, "shim-monitor.sock"))
+		socketPath = SocketPathRust(id)
 	}
 	return fmt.Sprintf("unix://%s", socketPath)
 }

src/runtime/pkg/device/drivers/vfio.go

@@ -54,6 +54,25 @@ func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice {
 	}
 }
 
+// Ignore specific PCI devices, supply the pciClass and the bitmask to check
+// against the device class, deviceBDF for meaningfull info message
+func (device *VFIODevice) checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (bool, error) {
+	if pciClass == "" {
+		return false, nil
+	}
+	pciClassID, err := strconv.ParseUint(pciClass, 0, 32)
+	if err != nil {
+		return false, err
+	}
+	// ClassID is 16 bits, remove the two trailing zeros
+	pciClassID = pciClassID >> 8
+	if pciClassID&bitmask == bitmask {
+		deviceLogger().Infof("Ignoring PCI (Host) Bridge deviceBDF %v Class %x", deviceBDF, pciClassID)
+		return true, nil
+	}
+	return false, nil
+}
+
 // Attach is standard interface of api.Device, it's used to add device to some
 // DeviceReceiver
 func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceReceiver) (retErr error) {
@@ -88,6 +107,18 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		}
 		id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize)
 
+		pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
+		// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
+		// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
+		// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
+		ignorePCIDevice, err := device.checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
+		if err != nil {
+			return err
+		}
+		if ignorePCIDevice {
+			continue
+		}
+
 		var vfio config.VFIODev
 
 		switch vfioDeviceType {
@@ -100,7 +131,7 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 				BDF:      deviceBDF,
 				SysfsDev: deviceSysfsDev,
 				IsPCIe:   isPCIe,
-				Class:    getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass),
+				Class:    pciClass,
 			}
 			if isPCIe {
 				vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
@@ -121,6 +152,7 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		default:
 			return fmt.Errorf("Failed to append device: VFIO device type unrecognized")
 		}
+
 		device.VfioDevs = append(device.VfioDevs, &vfio)
 	}
 

src/runtime/pkg/govmm/qemu/qemu.go

@@ -363,16 +363,12 @@ func (object Object) QemuParams(config *Config) []string {
 
 	case TDXGuest:
 		objectParams = append(objectParams, string(object.Type))
+		objectParams = append(objectParams, "sept-ve-disable=on")
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		if object.Debug {
 			objectParams = append(objectParams, "debug=on")
 		}
-		deviceParams = append(deviceParams, string(object.Driver))
-		deviceParams = append(deviceParams, fmt.Sprintf("id=%s", object.DeviceID))
-		deviceParams = append(deviceParams, fmt.Sprintf("file=%s", object.File))
-		if object.FirmwareVolume != "" {
-			deviceParams = append(deviceParams, fmt.Sprintf("config-firmware-volume=%s", object.FirmwareVolume))
-		}
+		config.Bios = object.File
 	case SEVGuest:
 		objectParams = append(objectParams, string(object.Type))
 		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))

src/runtime/pkg/kata-monitor/metrics.go

@@ -114,25 +114,32 @@ func (km *KataMonitor) ProcessMetricsRequest(w http.ResponseWriter, r *http.Requ
 		writer = gz
 	}
 
-	// create encoder to encode metrics.
-	encoder := expfmt.NewEncoder(writer, contentType)
-
-	// gather metrics collected for management agent.
-	mfs, err := prometheus.DefaultGatherer.Gather()
+	filterFamilies, err := getFilterFamilyFromReq(r)
 	if err != nil {
-		monitorLog.WithError(err).Error("failed to Gather metrics from prometheus.DefaultGatherer")
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte(err.Error()))
 		return
 	}
 
-	// encode metric gathered in current process
-	if err := encodeMetricFamily(mfs, encoder); err != nil {
-		monitorLog.WithError(err).Warnf("failed to encode metrics")
+	// create encoder to encode metrics.
+	encoder := expfmt.NewEncoder(writer, contentType)
+
+	if len(filterFamilies) == 0 {
+		// gather metrics collected for management agent.
+		mfs, err := prometheus.DefaultGatherer.Gather()
+		if err != nil {
+			monitorLog.WithError(err).Error("failed to Gather metrics from prometheus.DefaultGatherer")
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return
+		}
+
+		// encode metric gathered in current process
+		if err := encodeMetricFamily(mfs, encoder); err != nil {
+			monitorLog.WithError(err).Warnf("failed to encode metrics")
+		}
 	}
 
 	// aggregate sandboxes metrics and write to response by encoder
-	if err := km.aggregateSandboxMetrics(encoder); err != nil {
+	if err := km.aggregateSandboxMetrics(encoder, filterFamilies); err != nil {
 		monitorLog.WithError(err).Errorf("failed aggregateSandboxMetrics")
 		scrapeFailedCount.Inc()
 	}
@@ -155,7 +162,7 @@ func encodeMetricFamily(mfs []*dto.MetricFamily, encoder expfmt.Encoder) error {
 }
 
 // aggregateSandboxMetrics will get metrics from one sandbox and do some process
-func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
+func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder, filterFamilies []string) error {
 	// get all kata sandboxes from cache
 	sandboxes := km.sandboxCache.getSandboxList()
 	// save running kata pods as a metrics.
@@ -230,9 +237,21 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
 	}
 
 	// write metrics to response.
-	for _, mf := range metricsMap {
-		if err := encoder.Encode(mf); err != nil {
-			return err
+	if len(filterFamilies) > 0 {
+		for _, filterName := range filterFamilies {
+			for fullName, mf := range metricsMap {
+				if strings.HasPrefix(fullName, filterName) {
+					if err := encoder.Encode(mf); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	} else {
+		for _, mf := range metricsMap {
+			if err := encoder.Encode(mf); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

src/runtime/pkg/kata-monitor/pprof.go

@@ -32,7 +32,7 @@ func (km *KataMonitor) composeSocketAddress(r *http.Request) (string, error) {
 		return "", err
 	}
 
-	return shim.SocketAddress(sandbox), nil
+	return shim.ClientSocketAddress(sandbox), nil
 }
 
 func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request,

src/runtime/pkg/kata-monitor/shim_client.go

@@ -8,6 +8,7 @@ package katamonitor
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
@@ -36,3 +37,11 @@ func getSandboxIDFromReq(r *http.Request) (string, error) {
 func getSandboxFS() string {
 	return shim.GetSandboxesStoragePath()
 }
+
+func getFilterFamilyFromReq(r *http.Request) ([]string, error) {
+	filterFamilies := r.URL.Query().Get("filter_family")
+	if filterFamilies != "" {
+		return strings.Split(filterFamilies, ","), nil
+	}
+	return nil, nil
+}

src/runtime/pkg/katautils/config.go

@@ -786,6 +786,16 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		kataUtilsLogger.Info("Setting 'disable_image_nvdimm = true' as microvm does not support NVDIMM")
 	}
 
+	// Nvdimm can only be support when UEFI/ACPI is enabled on arm64, otherwise disable it.
+	if goruntime.GOARCH == "arm64" && firmware == "" {
+		if p, err := h.PFlash(); err == nil {
+			if len(p) == 0 {
+				h.DisableImageNvdimm = true
+				kataUtilsLogger.Info("Setting 'disable_image_nvdimm = true' if there is no firmware specified")
+			}
+		}
+	}
+
 	blockDriver, err := h.blockDeviceDriver()
 	if err != nil {
 		return vc.HypervisorConfig{}, err
@@ -1071,6 +1081,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableAnnotations:              h.EnableAnnotations,
 		DisableSeccomp:                 h.DisableSeccomp,
 		ConfidentialGuest:              h.ConfidentialGuest,
+		Rootless:                       h.Rootless,
 		DisableSeLinux:                 h.DisableSeLinux,
 		DisableGuestSeLinux:            h.DisableGuestSeLinux,
 		NetRateLimiterBwMaxRate:        h.getNetRateLimiterBwMaxRate(),

src/runtime/pkg/katautils/config_test.go

@@ -13,6 +13,7 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
+	goruntime "runtime"
 	"strings"
 	"syscall"
 	"testing"
@@ -182,6 +183,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		SNPGuestPolicy:        defaultSNPGuestPolicy,
 	}
 
+	if goruntime.GOARCH == "arm64" && len(hypervisorConfig.PFlash) == 0 && hypervisorConfig.FirmwarePath == "" {
+		hypervisorConfig.DisableImageNvdimm = true
+	}
+
 	agentConfig := vc.KataAgentConfig{
 		LongLiveConn: true,
 	}

src/runtime/pkg/utils/shimclient/shim_management_client.go

@@ -19,7 +19,7 @@ import (
 
 // BuildShimClient builds and returns an http client for communicating with the provided sandbox
 func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) {
-	return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout)
+	return buildUnixSocketClient(shim.ClientSocketAddress(sandboxID), timeout)
 }
 
 // buildUnixSocketClient build http client for Unix socket