CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #6504
Signed-off-by: Megan Wright <megan.wright@ibm.com>

src/runtime/pkg/device/config/config.go

@@ -258,15 +258,24 @@ const (
 	// VFIODeviceErrorType is the error type of VFIO device
 	VFIODeviceErrorType VFIODeviceType = iota
 
-	// VFIODeviceNormalType is a normal VFIO device type
-	VFIODeviceNormalType
+	// VFIOPCIDeviceNormalType is a normal VFIO PCI device type
+	VFIOPCIDeviceNormalType
 
-	// VFIODeviceMediatedType is a VFIO mediated device type
-	VFIODeviceMediatedType
+	// VFIOPCIDeviceMediatedType is a VFIO PCI mediated device type
+	VFIOPCIDeviceMediatedType
+
+	// VFIOAPDeviceMediatedType is a VFIO AP mediated device type
+	VFIOAPDeviceMediatedType
 )
 
-// VFIODev represents a VFIO drive used for hotplugging
-type VFIODev struct {
+type VFIODev interface {
+	GetID() *string
+	GetType() VFIODeviceType
+	GetSysfsDev() *string
+}
+
+// VFIOPCIDev represents a VFIO PCI device used for hotplugging
+type VFIOPCIDev struct {
 	// ID is used to identify this drive in the hypervisor options.
 	ID string
 
@@ -298,6 +307,44 @@ type VFIODev struct {
 	IsPCIe bool
 }
 
+func (d VFIOPCIDev) GetID() *string {
+	return &d.ID
+}
+
+func (d VFIOPCIDev) GetType() VFIODeviceType {
+	return d.Type
+}
+
+func (d VFIOPCIDev) GetSysfsDev() *string {
+	return &d.SysfsDev
+}
+
+type VFIOAPDev struct {
+	// ID is used to identify this drive in the hypervisor options.
+	ID string
+
+	// sysfsdev of VFIO mediated device
+	SysfsDev string
+
+	// APDevices are the Adjunct Processor devices assigned to the mdev
+	APDevices []string
+
+	// Type of VFIO device
+	Type VFIODeviceType
+}
+
+func (d VFIOAPDev) GetID() *string {
+	return &d.ID
+}
+
+func (d VFIOAPDev) GetType() VFIODeviceType {
+	return d.Type
+}
+
+func (d VFIOAPDev) GetSysfsDev() *string {
+	return &d.SysfsDev
+}
+
 // RNGDev represents a random number generator device
 type RNGDev struct {
 	// ID is used to identify the device in the hypervisor options.

src/runtime/pkg/device/drivers/utils.go

@@ -89,18 +89,47 @@ func readPCIProperty(propertyPath string) (string, error) {
 	return strings.Split(string(buf), "\n")[0], nil
 }
 
-func GetVFIODeviceType(deviceFileName string) config.VFIODeviceType {
+func GetVFIODeviceType(deviceFilePath string) (config.VFIODeviceType, error) {
+	deviceFileName := filepath.Base(deviceFilePath)
+
 	//For example, 0000:04:00.0
 	tokens := strings.Split(deviceFileName, ":")
-	vfioDeviceType := config.VFIODeviceErrorType
 	if len(tokens) == 3 {
-		vfioDeviceType = config.VFIODeviceNormalType
-	} else {
-		//For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
-		tokens = strings.Split(deviceFileName, "-")
-		if len(tokens) == 5 {
-			vfioDeviceType = config.VFIODeviceMediatedType
-		}
+		return config.VFIOPCIDeviceNormalType, nil
 	}
-	return vfioDeviceType
+
+	//For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
+	tokens = strings.Split(deviceFileName, "-")
+	if len(tokens) != 5 {
+		return config.VFIODeviceErrorType, fmt.Errorf("Incorrect tokens found while parsing VFIO details: %s", deviceFileName)
+	}
+
+	deviceSysfsDev, err := GetSysfsDev(deviceFilePath)
+	if err != nil {
+		return config.VFIODeviceErrorType, err
+	}
+
+	if strings.HasPrefix(deviceSysfsDev, vfioAPSysfsDir) {
+		return config.VFIOAPDeviceMediatedType, nil
+	}
+
+	return config.VFIOPCIDeviceMediatedType, nil
+}
+
+// GetSysfsDev returns the sysfsdev of mediated device
+// Expected input string format is absolute path to the sysfs dev node
+// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4
+func GetSysfsDev(sysfsDevStr string) (string, error) {
+	return filepath.EvalSymlinks(sysfsDevStr)
+}
+
+// GetAPVFIODevices retrieves all APQNs associated with a mediated VFIO-AP
+// device
+func GetAPVFIODevices(sysfsdev string) ([]string, error) {
+	data, err := os.ReadFile(filepath.Join(sysfsdev, "matrix"))
+	if err != nil {
+		return []string{}, err
+	}
+	// Split by newlines, omitting final newline
+	return strings.Split(string(data[:len(data)-1]), "\n"), nil
 }

src/runtime/pkg/device/drivers/vfio.go

@@ -30,6 +30,7 @@ const (
 	iommuGroupPath      = "/sys/bus/pci/devices/%s/iommu_group"
 	vfioDevPath         = "/dev/vfio/%s"
 	pcieRootPortPrefix  = "rp"
+	vfioAPSysfsDir      = "/sys/devices/vfio_ap"
 )
 
 var (
@@ -85,19 +86,42 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 		if err != nil {
 			return err
 		}
-		vfio := &config.VFIODev{
-			ID:       utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize),
-			Type:     vfioDeviceType,
-			BDF:      deviceBDF,
-			SysfsDev: deviceSysfsDev,
-			IsPCIe:   isPCIeDevice(deviceBDF),
-			Class:    getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass),
-		}
-		device.VfioDevs = append(device.VfioDevs, vfio)
-		if vfio.IsPCIe {
-			vfio.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
-			AllPCIeDevs[vfio.BDF] = true
+		id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize)
+
+		var vfio config.VFIODev
+
+		switch vfioDeviceType {
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			isPCIe := isPCIeDevice(deviceBDF)
+			// Do not directly assign to `vfio` -- need to access field still
+			vfioPCI := config.VFIOPCIDev{
+				ID:       id,
+				Type:     vfioDeviceType,
+				BDF:      deviceBDF,
+				SysfsDev: deviceSysfsDev,
+				IsPCIe:   isPCIe,
+				Class:    getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass),
+			}
+			if isPCIe {
+				vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
+				AllPCIeDevs[deviceBDF] = true
+			}
+			vfio = vfioPCI
+		case config.VFIOAPDeviceMediatedType:
+			devices, err := GetAPVFIODevices(deviceSysfsDev)
+			if err != nil {
+				return err
+			}
+			vfio = config.VFIOAPDev{
+				ID:        id,
+				SysfsDev:  deviceSysfsDev,
+				Type:      config.VFIOAPDeviceMediatedType,
+				APDevices: devices,
+			}
+		default:
+			return fmt.Errorf("Failed to append device: VFIO device type unrecognized")
 		}
+		device.VfioDevs = append(device.VfioDevs, &vfio)
 	}
 
 	coldPlug := device.DeviceInfo.ColdPlug
@@ -192,31 +216,60 @@ func (device *VFIODevice) Load(ds config.DeviceState) {
 	device.GenericDevice.Load(ds)
 
 	for _, dev := range ds.VFIODevs {
-		device.VfioDevs = append(device.VfioDevs, &config.VFIODev{
-			ID:       dev.ID,
-			Type:     config.VFIODeviceType(dev.Type),
-			BDF:      dev.BDF,
-			SysfsDev: dev.SysfsDev,
-		})
+		var vfio config.VFIODev
+
+		vfioDeviceType := (*device.VfioDevs[0]).GetType()
+		switch vfioDeviceType {
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
+			bdf := ""
+			if pciDev, ok := (*dev).(config.VFIOPCIDev); ok {
+				bdf = pciDev.BDF
+			}
+			vfio = config.VFIOPCIDev{
+				ID:       *(*dev).GetID(),
+				Type:     config.VFIODeviceType((*dev).GetType()),
+				BDF:      bdf,
+				SysfsDev: *(*dev).GetSysfsDev(),
+			}
+		case config.VFIOAPDeviceMediatedType:
+			vfio = config.VFIOAPDev{
+				ID:       *(*dev).GetID(),
+				SysfsDev: *(*dev).GetSysfsDev(),
+			}
+		default:
+			deviceLogger().WithError(
+				fmt.Errorf("VFIO device type unrecognized"),
+			).Error("Failed to append device")
+			return
+		}
+
+		device.VfioDevs = append(device.VfioDevs, &vfio)
 	}
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) {
-	vfioDeviceType = GetVFIODeviceType(deviceFileName)
+	sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
+	vfioDeviceType, err = GetVFIODeviceType(sysfsDevStr)
+	if err != nil {
+		return deviceBDF, deviceSysfsDev, vfioDeviceType, err
+	}
 
 	switch vfioDeviceType {
-	case config.VFIODeviceNormalType:
+	case config.VFIOPCIDeviceNormalType:
 		// Get bdf of device eg. 0000:00:1c.0
 		deviceBDF = getBDF(deviceFileName)
 		// Get sysfs path used by cloud-hypervisor
 		deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName)
-	case config.VFIODeviceMediatedType:
+	case config.VFIOPCIDeviceMediatedType:
 		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
 		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
-		deviceSysfsDev, err = getSysfsDev(sysfsDevStr)
+		deviceSysfsDev, err = GetSysfsDev(sysfsDevStr)
 		deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev))
+	case config.VFIOAPDeviceMediatedType:
+		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
+		deviceSysfsDev, err = GetSysfsDev(sysfsDevStr)
 	default:
 		err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName)
 	}
@@ -244,13 +297,6 @@ func getBDF(deviceSysStr string) string {
 	return tokens[1]
 }
 
-// getSysfsDev returns the sysfsdev of mediated device
-// Expected input string format is absolute path to the sysfs dev node
-// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4
-func getSysfsDev(sysfsDevStr string) (string, error) {
-	return filepath.EvalSymlinks(sysfsDevStr)
-}
-
 // BindDevicetoVFIO binds the device to vfio driver after unbinding from host.
 // Will be called by a network interface or a generic pcie device.
 func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) {

src/runtime/pkg/device/drivers/vfio_test.go

@@ -32,9 +32,9 @@ func TestGetVFIODetails(t *testing.T) {
 		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "")
 
 		switch vfioDeviceType {
-		case config.VFIODeviceNormalType:
+		case config.VFIOPCIDeviceNormalType:
 			assert.Equal(t, d.expectedStr, deviceBDF)
-		case config.VFIODeviceMediatedType:
+		case config.VFIOPCIDeviceMediatedType, config.VFIOAPDeviceMediatedType:
 			assert.Equal(t, d.expectedStr, deviceSysfsDev)
 		default:
 			assert.NotNil(t, err)

src/runtime/pkg/resourcecontrol/cgroups.go

@@ -67,7 +67,8 @@ func sandboxDevices() []specs.LinuxDeviceCgroup {
 	// In order to run Virtual Machines and create virtqueues, hypervisors
 	// need access to certain character devices in the host, like kvm and vhost-net.
 	hypervisorDevices := []string{
-		"/dev/kvm",         // To run virtual machines
+		"/dev/kvm",         // To run virtual machines with KVM
+		"/dev/mshv",        // To run virtual machines with Hyper-V
 		"/dev/vhost-net",   // To create virtqueues
 		"/dev/vfio/vfio",   // To access VFIO devices
 		"/dev/vhost-vsock", // To interact with vsock if