CCv0: Merge from main -- August 1st

Conflicts:
	src/runtime/pkg/katautils/config.go
	src/runtime/virtcontainers/container.go
	src/runtime/virtcontainers/hypervisor.go
	src/runtime/virtcontainers/qemu_arch_base.go
	src/runtime/virtcontainers/sandbox.go
	tests/integration/kubernetes/gha-run.sh
	tests/integration/kubernetes/setup.sh
	tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
	tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
	tools/packaging/kata-deploy/scripts/kata-deploy.sh
	tools/packaging/kernel/kata_config_version
	versions.yaml

Fixes: #7433

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -330,31 +330,29 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string,
 	disableBlockDevice := true
 	blockDeviceDriver := "virtio-scsi"
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
 	hotPlugVFIO = config.BridgePort
-	coldPlugVFIO = config.RootPort
+	coldPlugVFIO = config.NoPort
 
 	configFileOptions := ktu.RuntimeConfigOptions{
-		Hypervisor:           "qemu",
-		HypervisorPath:       hypervisorPath,
-		KernelPath:           kernelPath,
-		ImagePath:            imagePath,
-		RootfsType:           rootfsType,
-		KernelParams:         kernelParams,
-		MachineType:          machineType,
-		LogPath:              logPath,
-		DisableBlock:         disableBlockDevice,
-		BlockDeviceDriver:    blockDeviceDriver,
-		EnableIOThreads:      enableIOThreads,
-		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
-		DisableNewNetNs:      disableNewNetNs,
-		SharedFS:             sharedFS,
-		VirtioFSDaemon:       virtioFSdaemon,
-		HotPlugVFIO:          hotPlugVFIO,
-		ColdPlugVFIO:         coldPlugVFIO,
+		Hypervisor:        "qemu",
+		HypervisorPath:    hypervisorPath,
+		KernelPath:        kernelPath,
+		ImagePath:         imagePath,
+		RootfsType:        rootfsType,
+		KernelParams:      kernelParams,
+		MachineType:       machineType,
+		LogPath:           logPath,
+		DisableBlock:      disableBlockDevice,
+		BlockDeviceDriver: blockDeviceDriver,
+		EnableIOThreads:   enableIOThreads,
+		DisableNewNetNs:   disableNewNetNs,
+		SharedFS:          sharedFS,
+		VirtioFSDaemon:    virtioFSdaemon,
+		HotPlugVFIO:       hotPlugVFIO,
+		ColdPlugVFIO:      coldPlugVFIO,
 	}
 
 	runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions)

src/runtime/pkg/device/drivers/utils.go

@@ -111,7 +111,7 @@ func GetVFIODeviceType(deviceFilePath string) (config.VFIODeviceType, error) {
 		return config.VFIODeviceErrorType, err
 	}
 
-	if strings.HasPrefix(deviceSysfsDev, vfioAPSysfsDir) {
+	if strings.Contains(deviceSysfsDev, vfioAPSysfsDir) {
 		return config.VFIOAPDeviceMediatedType, nil
 	}
 
@@ -178,22 +178,22 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 		}
 		id := utils.MakeNameID("vfio", device.ID+strconv.Itoa(i), maxDevIDSize)
 
-		pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
-		// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
-		// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
-		// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
-		ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
-		if err != nil {
-			return nil, err
-		}
-		if ignorePCIDevice {
-			continue
-		}
-
 		var vfio config.VFIODev
 
 		switch vfioDeviceType {
 		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			// This is vfio-pci and vfio-mdev specific
+			pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
+			// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
+			// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
+			// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
+			ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
+			if err != nil {
+				return nil, err
+			}
+			if ignorePCIDevice {
+				continue
+			}
 			// Do not directly assign to `vfio` -- need to access field still
 			vfio = config.VFIODev{
 				ID:       id,
@@ -216,6 +216,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 				SysfsDev:  deviceSysfsDev,
 				Type:      config.VFIOAPDeviceMediatedType,
 				APDevices: devices,
+				Port:      device.Port,
 			}
 		default:
 			return nil, fmt.Errorf("Failed to append device: VFIO device type unrecognized")

src/runtime/pkg/device/drivers/vfio.go

@@ -69,7 +69,14 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 	if err != nil {
 		return err
 	}
+
 	for _, vfio := range device.VfioDevs {
+		// If vfio.Port is not set we bail out, users should set
+		// explicitly the port in the config file
+		if vfio.Port == "" {
+			return fmt.Errorf("cold_plug_vfio= or hot_plug_vfio= port is not set for device %s (BridgePort | RootPort | SwitchPort)", vfio.BDF)
+		}
+
 		if vfio.IsPCIe {
 			busIndex := len(config.PCIeDevices[vfio.Port])
 			vfio.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[vfio.Port], busIndex)

src/runtime/pkg/device/manager/manager.go

@@ -120,7 +120,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
 	if devInfo.ID, err = dm.newDeviceID(); err != nil {
 		return nil, err
 	}
-	if IsVFIO(devInfo.HostPath) {
+	if IsVFIODevice(devInfo.HostPath) {
 		return drivers.NewVFIODevice(&devInfo), nil
 	} else if IsVhostUserBlk(devInfo) {
 		if devInfo.DriverOptions == nil {
@@ -191,12 +191,12 @@ func (dm *deviceManager) AttachDevice(ctx context.Context, id string, dr api.Dev
 	dm.Lock()
 	defer dm.Unlock()
 
-	d, ok := dm.devices[id]
+	dev, ok := dm.devices[id]
 	if !ok {
 		return ErrDeviceNotExist
 	}
 
-	if err := d.Attach(ctx, dr); err != nil {
+	if err := dev.Attach(ctx, dr); err != nil {
 		return err
 	}
 	return nil

src/runtime/pkg/device/manager/manager_test.go

@@ -90,6 +90,100 @@ func TestNewDevice(t *testing.T) {
 	assert.Equal(t, vfioDev.DeviceInfo.GID, uint32(2))
 }
 
+func TestAttachVFIOAPDevice(t *testing.T) {
+
+	var err error
+	var ok bool
+
+	dm := &deviceManager{
+		devices: make(map[string]api.Device),
+	}
+
+	tmpDir := t.TempDir()
+	// sys/devices/vfio_ap/matrix/f94290f8-78ac-45fb-bb22-e55e519fa64f
+	testSysfsAP := "/sys/devices/vfio_ap/"
+	testDeviceAP := "f94290f8-78ac-45fb-bb22-e55e519fa64f"
+	testVFIOGroup := "42"
+
+	matrixDir := filepath.Join(tmpDir, testSysfsAP, "matrix")
+	err = os.MkdirAll(matrixDir, dirMode)
+	assert.Nil(t, err)
+
+	deviceAPFile := filepath.Join(matrixDir, testDeviceAP)
+	err = os.MkdirAll(deviceAPFile, dirMode)
+	assert.Nil(t, err)
+
+	matrixDeviceAPFile := filepath.Join(deviceAPFile, "matrix")
+	_, err = os.Create(matrixDeviceAPFile)
+	assert.Nil(t, err)
+	// create AP devices in the matrix file
+	APDevices := []byte("05.001f\n")
+	err = os.WriteFile(matrixDeviceAPFile, APDevices, 0644)
+	assert.Nil(t, err)
+
+	devicesVFIOGroupDir := filepath.Join(tmpDir, testVFIOGroup, "devices")
+	err = os.MkdirAll(devicesVFIOGroupDir, dirMode)
+	assert.Nil(t, err)
+
+	deviceAPSymlink := filepath.Join(devicesVFIOGroupDir, testDeviceAP)
+	err = os.Symlink(deviceAPFile, deviceAPSymlink)
+	assert.Nil(t, err)
+
+	savedIOMMUPath := config.SysIOMMUGroupPath
+	config.SysIOMMUGroupPath = tmpDir
+
+	savedSysBusPciDevicesPath := config.SysBusPciDevicesPath
+	config.SysBusPciDevicesPath = devicesVFIOGroupDir
+
+	defer func() {
+		config.SysIOMMUGroupPath = savedIOMMUPath
+		config.SysBusPciDevicesPath = savedSysBusPciDevicesPath
+	}()
+
+	path := filepath.Join(vfioPath, testVFIOGroup)
+	deviceInfo := config.DeviceInfo{
+		HostPath:      path,
+		ContainerPath: path,
+		DevType:       "c",
+		ColdPlug:      false,
+		Port:          config.RootPort,
+	}
+
+	device, err := dm.NewDevice(deviceInfo)
+	assert.Nil(t, err)
+	_, ok = device.(*drivers.VFIODevice)
+	assert.True(t, ok)
+
+	devReceiver := &api.MockDeviceReceiver{}
+	err = device.Attach(context.Background(), devReceiver)
+	assert.Nil(t, err)
+
+	err = device.Detach(context.Background(), devReceiver)
+	assert.Nil(t, err)
+
+	// If we omit the port setting we should fail
+	failDm := &deviceManager{
+		devices: make(map[string]api.Device),
+	}
+
+	failDeviceInfo := config.DeviceInfo{
+		HostPath:      path,
+		ContainerPath: path,
+		DevType:       "c",
+		ColdPlug:      false,
+	}
+
+	failDevice, err := failDm.NewDevice(failDeviceInfo)
+	assert.Nil(t, err)
+	_, ok = failDevice.(*drivers.VFIODevice)
+	assert.True(t, ok)
+
+	failDevReceiver := &api.MockDeviceReceiver{}
+	err = failDevice.Attach(context.Background(), failDevReceiver)
+	assert.Error(t, err)
+
+}
+
 func TestAttachVFIODevice(t *testing.T) {
 	dm := &deviceManager{
 		blockDriver: config.VirtioBlock,
@@ -132,6 +226,8 @@ func TestAttachVFIODevice(t *testing.T) {
 		HostPath:      path,
 		ContainerPath: path,
 		DevType:       "c",
+		ColdPlug:      false,
+		Port:          config.RootPort,
 	}
 
 	device, err := dm.NewDevice(deviceInfo)

src/runtime/pkg/device/manager/utils.go

@@ -17,8 +17,15 @@ const (
 	vfioPath = "/dev/vfio/"
 )
 
+// IsVFIOControlDevice checks if the device provided is a vfio control device.
+// Depending no the vfio_mode we need to know if a device is a VFIO device
+// or the VFIO control device
+func IsVFIOControlDevice(path string) bool {
+	return path == filepath.Join(vfioPath, "vfio")
+}
+
 // IsVFIO checks if the device provided is a vfio group.
-func IsVFIO(hostPath string) bool {
+func IsVFIODevice(hostPath string) bool {
 	// Ignore /dev/vfio/vfio character device
 	if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) {
 		return false

src/runtime/pkg/device/manager/utils_test.go

@@ -31,7 +31,7 @@ func TestIsVFIO(t *testing.T) {
 	}
 
 	for _, d := range data {
-		isVFIO := IsVFIO(d.path)
+		isVFIO := IsVFIODevice(d.path)
 		assert.Equal(t, d.expected, isVFIO)
 	}
 }

src/runtime/pkg/govmm/qemu/qemu.go

@@ -163,6 +163,9 @@ const (
 
 	// TransportMMIO is the MMIO transport for virtio devices.
 	TransportMMIO VirtioTransport = "mmio"
+
+	// TransportAP is the AP transport for virtio devices.
+	TransportAP VirtioTransport = "ap"
 )
 
 // defaultTransport returns the default transport for the current combination
@@ -199,6 +202,14 @@ func (transport VirtioTransport) isVirtioCCW(config *Config) bool {
 	return transport == TransportCCW
 }
 
+func (transport VirtioTransport) isVirtioAP(config *Config) bool {
+	if transport == "" {
+		transport = transport.defaultTransport(config)
+	}
+
+	return transport == TransportAP
+}
+
 // getName returns the name of the current transport.
 func (transport VirtioTransport) getName(config *Config) string {
 	if transport == "" {
@@ -1852,6 +1863,9 @@ type VFIODevice struct {
 
 	// Transport is the virtio transport for this device.
 	Transport VirtioTransport
+
+	// SysfsDev specifies the sysfs matrix entry for the AP device
+	SysfsDev string
 }
 
 // VFIODeviceTransport is a map of the vfio device name that corresponds to
@@ -1860,11 +1874,13 @@ var VFIODeviceTransport = map[VirtioTransport]string{
 	TransportPCI:  "vfio-pci",
 	TransportCCW:  "vfio-ccw",
 	TransportMMIO: "vfio-device",
+	TransportAP:   "vfio-ap",
 }
 
 // Valid returns true if the VFIODevice structure is valid and complete.
+// s390x architecture requires SysfsDev to be set.
 func (vfioDev VFIODevice) Valid() bool {
-	return vfioDev.BDF != ""
+	return vfioDev.BDF != "" || vfioDev.SysfsDev != ""
 }
 
 // QemuParams returns the qemu parameters built out of this vfio device.
@@ -1874,6 +1890,15 @@ func (vfioDev VFIODevice) QemuParams(config *Config) []string {
 
 	driver := vfioDev.deviceName(config)
 
+	if vfioDev.Transport.isVirtioAP(config) {
+		deviceParams = append(deviceParams, fmt.Sprintf("%s,sysfsdev=%s", driver, vfioDev.SysfsDev))
+
+		qemuParams = append(qemuParams, "-device")
+		qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
+
+		return qemuParams
+	}
+
 	deviceParams = append(deviceParams, fmt.Sprintf("%s,host=%s", driver, vfioDev.BDF))
 	if vfioDev.Transport.isVirtioPCI(config) {
 		if vfioDev.VendorID != "" {
@@ -2878,10 +2903,9 @@ func (config *Config) appendDevices(logger QMPLog) {
 
 	for _, d := range config.Devices {
 		if !d.Valid() {
-			logger.Errorf("vm device is not valid: %+v", config.Devices)
+			logger.Errorf("vm device is not valid: %+v", d)
 			continue
 		}
-
 		config.qemuParams = append(config.qemuParams, d.QemuParams(config)...)
 	}
 }

src/runtime/pkg/govmm/qemu/qmp.go

@@ -1233,10 +1233,11 @@ func (q *QMP) ExecutePCIVFIOMediatedDeviceAdd(ctx context.Context, devID, sysfsd
 }
 
 // ExecuteAPVFIOMediatedDeviceAdd adds a VFIO mediated AP device to a QEMU instance using the device_add command.
-func (q *QMP) ExecuteAPVFIOMediatedDeviceAdd(ctx context.Context, sysfsdev string) error {
+func (q *QMP) ExecuteAPVFIOMediatedDeviceAdd(ctx context.Context, sysfsdev string, devID string) error {
 	args := map[string]interface{}{
 		"driver":   VfioAP,
 		"sysfsdev": sysfsdev,
+		"id":       devID,
 	}
 	return q.executeCommand(ctx, "device_add", args, nil)
 }

src/runtime/pkg/govmm/qemu/qmp_test.go

@@ -1128,7 +1128,7 @@ func TestQMPAPVFIOMediatedDeviceAdd(t *testing.T) {
 	q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh)
 	checkVersion(t, connectedCh)
 	sysfsDev := "/sys/devices/vfio_ap/matrix/a297db4a-f4c2-11e6-90f6-d3b88d6c9525"
-	err := q.ExecuteAPVFIOMediatedDeviceAdd(context.Background(), sysfsDev)
+	err := q.ExecuteAPVFIOMediatedDeviceAdd(context.Background(), sysfsDev, "test-id")
 	if err != nil {
 		t.Fatalf("Unexpected error %v", err)
 	}

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -42,10 +42,9 @@ type HypervisorState struct {
 	// HotpluggedCPUs is the list of CPUs that were hot-added
 	HotpluggedVCPUs []CPUDevice
 
-	HotpluggedMemory     int
-	VirtiofsDaemonPid    int
-	Pid                  int
-	HotPlugVFIO          config.PCIePort
-	ColdPlugVFIO         config.PCIePort
-	HotplugVFIOOnRootBus bool
+	HotpluggedMemory  int
+	VirtiofsDaemonPid int
+	Pid               int
+	HotPlugVFIO       config.PCIePort
+	ColdPlugVFIO      config.PCIePort
 }

src/runtime/pkg/katatestutils/utils.go

@@ -233,7 +233,6 @@ type RuntimeConfigOptions struct {
 	DefaultMsize9p       uint32
 	DisableBlock         bool
 	EnableIOThreads      bool
-	HotplugVFIOOnRootBus bool
 	DisableNewNetNs      bool
 	HypervisorDebug      bool
 	RuntimeDebug         bool
@@ -317,8 +316,8 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	default_memory = ` + strconv.FormatUint(uint64(config.DefaultMemSize), 10) + `
 	disable_block_device_use =  ` + strconv.FormatBool(config.DisableBlock) + `
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
-	hotplug_vfio_on_root_bus =  ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
 	cold_plug_vfio =  "` + config.ColdPlugVFIO.String() + `"
+	hot_plug_vfio =   "` + config.HotPlugVFIO.String() + `"
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"

src/runtime/pkg/katautils/config-settings.go.in

@@ -81,7 +81,6 @@ const defaultFileBackedMemRootDir string = ""
 const defaultEnableDebug bool = false
 const defaultDisableNestingChecks bool = false
 const defaultMsize9p uint32 = 8192
-const defaultHotplugVFIOOnRootBus bool = false
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "never"

src/runtime/pkg/katautils/config.go

@@ -22,6 +22,7 @@ import (
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
@@ -157,7 +158,6 @@ type hypervisor struct {
 	DisableNestingChecks           bool            `toml:"disable_nesting_checks"`
 	EnableIOThreads                bool            `toml:"enable_iothreads"`
 	DisableImageNvdimm             bool            `toml:"disable_image_nvdimm"`
-	HotplugVFIOOnRootBus           bool            `toml:"hotplug_vfio_on_root_bus"`
 	HotPlugVFIO                    config.PCIePort `toml:"hot_plug_vfio"`
 	ColdPlugVFIO                   config.PCIePort `toml:"cold_plug_vfio"`
 	DisableVhostNet                bool            `toml:"disable_vhost_net"`
@@ -886,7 +886,6 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:           h.EnableIOThreads,
 		Msize9p:                   h.msize9p(),
 		DisableImageNvdimm:        h.DisableImageNvdimm,
-		HotplugVFIOOnRootBus:      h.HotplugVFIOOnRootBus,
 		HotPlugVFIO:               h.hotPlugVFIO(),
 		ColdPlugVFIO:              h.coldPlugVFIO(),
 		DisableVhostNet:           h.DisableVhostNet,
@@ -1089,7 +1088,6 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		BlockDeviceCacheNoflush:        h.BlockDeviceCacheNoflush,
 		EnableIOThreads:                h.EnableIOThreads,
 		Msize9p:                        h.msize9p(),
-		HotplugVFIOOnRootBus:           h.HotplugVFIOOnRootBus,
 		ColdPlugVFIO:                   h.coldPlugVFIO(),
 		HotPlugVFIO:                    h.hotPlugVFIO(),
 		DisableVhostNet:                true,
@@ -1336,7 +1334,6 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		BlockDeviceCacheNoflush:   defaultBlockDeviceCacheNoflush,
 		EnableIOThreads:           defaultEnableIOThreads,
 		Msize9p:                   defaultMsize9p,
-		HotplugVFIOOnRootBus:      defaultHotplugVFIOOnRootBus,
 		ColdPlugVFIO:              defaultColdPlugVFIO,
 		HotPlugVFIO:               defaultHotPlugVFIO,
 		GuestHookPath:             defaultGuestHookPath,
@@ -1722,7 +1719,8 @@ func checkConfig(config oci.RuntimeConfig) error {
 	hotPlugVFIO := config.HypervisorConfig.HotPlugVFIO
 	coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO
 	machineType := config.HypervisorConfig.HypervisorMachineType
-	if err := checkPCIeConfig(coldPlugVFIO, hotPlugVFIO, machineType); err != nil {
+	hypervisorType := config.HypervisorType
+	if err := checkPCIeConfig(coldPlugVFIO, hotPlugVFIO, machineType, hypervisorType); err != nil {
 		return err
 	}
 
@@ -1732,10 +1730,9 @@ func checkConfig(config oci.RuntimeConfig) error {
 // checkPCIeConfig ensures the PCIe configuration is valid.
 // Only allow one of the following settings for cold-plug:
 // no-port, root-port, switch-port
-func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string) error {
-	// Currently only QEMU q35 supports advanced PCIe topologies
-	// firecracker, dragonball do not have right now any PCIe support
-	if machineType != "q35" {
+func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string, hypervisorType virtcontainers.HypervisorType) error {
+	if hypervisorType != virtcontainers.QemuHypervisor {
+		kataUtilsLogger.Warn("Advanced PCIe Topology only available for QEMU hypervisor, ignoring hot(cold)_vfio_port setting")
 		return nil
 	}
 
@@ -1745,6 +1742,12 @@ func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineT
 	if coldPlug == config.NoPort && hotPlug == config.NoPort {
 		return nil
 	}
+	// Currently only QEMU q35,virt support advanced PCIe topologies
+	// firecracker, dragonball do not have right now any PCIe support
+	if machineType != "q35" && machineType != "virt" {
+		return nil
+	}
+
 	var port config.PCIePort
 	if coldPlug != config.NoPort {
 		port = coldPlug
@@ -1752,10 +1755,13 @@ func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineT
 	if hotPlug != config.NoPort {
 		port = hotPlug
 	}
-	if port == config.NoPort || port == config.BridgePort || port == config.RootPort || port == config.SwitchPort {
+	if port == config.NoPort {
+		return fmt.Errorf("invalid vfio_port=%s setting, use on of %s, %s, %s",
+			port, config.BridgePort, config.RootPort, config.SwitchPort)
+	}
+	if port == config.BridgePort || port == config.RootPort || port == config.SwitchPort {
 		return nil
 	}
-
 	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s",
 		coldPlug, config.NoPort, config.BridgePort, config.RootPort, config.SwitchPort)
 }

src/runtime/pkg/katautils/config_test.go

@@ -85,7 +85,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 	blockDeviceDriver := "virtio-scsi"
 	blockDeviceAIO := "io_uring"
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	hotPlugVFIO = config.NoPort
 	coldPlugVFIO = config.BridgePort
 	disableNewNetNs := false
@@ -108,7 +107,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		BlockDeviceDriver:    blockDeviceDriver,
 		BlockDeviceAIO:       blockDeviceAIO,
 		EnableIOThreads:      enableIOThreads,
-		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 		DisableNewNetNs:      disableNewNetNs,
@@ -172,7 +170,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		BlockDeviceAIO:        defaultBlockDeviceAIO,
 		DefaultBridges:        defaultBridgesCount,
 		EnableIOThreads:       enableIOThreads,
-		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		HotPlugVFIO:           hotPlugVFIO,
 		ColdPlugVFIO:          coldPlugVFIO,
 		Msize9p:               defaultMsize9p,
@@ -613,7 +610,6 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	machineType := "machineType"
 	disableBlock := true
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	coldPlugVFIO = config.BridgePort
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
 	blockDeviceAIO := "io_uring"
@@ -632,7 +628,6 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		MachineType:           machineType,
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
-		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		ColdPlugVFIO:          coldPlugVFIO,
 		RxRateLimiterMaxRate:  rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:  txRateLimiterMaxRate,
@@ -684,10 +679,6 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		t.Errorf("Expected value for enable IOThreads  %v, got %v", enableIOThreads, config.EnableIOThreads)
 	}
 
-	if config.HotplugVFIOOnRootBus != hotplugVFIOOnRootBus {
-		t.Errorf("Expected value for HotplugVFIOOnRootBus %v, got %v", hotplugVFIOOnRootBus, config.HotplugVFIOOnRootBus)
-	}
-
 	if config.RxRateLimiterMaxRate != rxRateLimiterMaxRate {
 		t.Errorf("Expected value for rx rate limiter %v, got %v", rxRateLimiterMaxRate, config.RxRateLimiterMaxRate)
 	}
@@ -809,7 +800,6 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
 	machineType := "machineType"
 	disableBlock := true
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 
 	hypervisor := hypervisor{
 		Path:                  hypervisorPath,
@@ -819,7 +809,6 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
 		MachineType:           machineType,
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
-		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 	}
 
 	_, err := newQemuHypervisorConfig(hypervisor)

src/runtime/pkg/oci/utils.go

@@ -511,12 +511,6 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		return err
 	}
 
-	if err := newAnnotationConfiguration(ocispec, vcAnnotations.HotplugVFIOOnRootBus).setBool(func(hotplugVFIOOnRootBus bool) {
-		config.HypervisorConfig.HotplugVFIOOnRootBus = hotplugVFIOOnRootBus
-	}); err != nil {
-		return err
-	}
-
 	if err := newAnnotationConfiguration(ocispec, vcAnnotations.UseLegacySerial).setBool(func(useLegacySerial bool) {
 		config.HypervisorConfig.LegacySerial = useLegacySerial
 	}); err != nil {

src/runtime/pkg/oci/utils_test.go

@@ -659,7 +659,6 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.DisableVhostNet] = "true"
 	ocispec.Annotations[vcAnnotations.GuestHookPath] = "/usr/bin/"
 	ocispec.Annotations[vcAnnotations.DisableImageNvdimm] = "true"
-	ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus] = "true"
 	ocispec.Annotations[vcAnnotations.ColdPlugVFIO] = config.BridgePort
 	ocispec.Annotations[vcAnnotations.HotPlugVFIO] = config.NoPort
 	ocispec.Annotations[vcAnnotations.IOMMUPlatform] = "true"
@@ -700,7 +699,6 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(sbConfig.HypervisorConfig.DisableVhostNet, true)
 	assert.Equal(sbConfig.HypervisorConfig.GuestHookPath, "/usr/bin/")
 	assert.Equal(sbConfig.HypervisorConfig.DisableImageNvdimm, true)
-	assert.Equal(sbConfig.HypervisorConfig.HotplugVFIOOnRootBus, true)
 	assert.Equal(string(sbConfig.HypervisorConfig.ColdPlugVFIO), string(config.BridgePort))
 	assert.Equal(string(sbConfig.HypervisorConfig.HotPlugVFIO), string(config.NoPort))
 	assert.Equal(sbConfig.HypervisorConfig.IOMMUPlatform, true)