From 2a830177cab9575863fbaf4ddadba25b5e07b982 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 21 Apr 2023 10:43:56 +0000 Subject: [PATCH] gpu: Add fwcfg helper function Added driver util function for easier handling of VFIO devices outside of the VFIO module. At the sandbox level we may need to set options depending if we have a VFIO/PCIe device, like the fwCfg for confiential guests. Signed-off-by: Zvonko Kaiser --- src/runtime/cmd/kata-runtime/kata-env.go | 22 ++++---- src/runtime/pkg/device/drivers/utils.go | 8 +-- src/runtime/pkg/device/drivers/vfio.go | 66 +----------------------- src/runtime/pkg/katatestutils/utils.go | 1 + src/runtime/virtcontainers/qemu.go | 44 +++++++++++++--- src/runtime/virtcontainers/sandbox.go | 2 +- 6 files changed, 56 insertions(+), 87 deletions(-) diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index c7e919d62..f17480aba 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -307,17 +307,17 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) { } return HypervisorInfo{ - Debug: config.HypervisorConfig.Debug, - MachineType: config.HypervisorConfig.HypervisorMachineType, - Version: version, - Path: hypervisorPath, - BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, - Msize9p: config.HypervisorConfig.Msize9p, - MemorySlots: config.HypervisorConfig.MemSlots, - EntropySource: config.HypervisorConfig.EntropySource, - SharedFS: config.HypervisorConfig.SharedFS, - VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, - + Debug: config.HypervisorConfig.Debug, + MachineType: config.HypervisorConfig.HypervisorMachineType, + Version: version, + Path: hypervisorPath, + BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, + Msize9p: config.HypervisorConfig.Msize9p, + MemorySlots: config.HypervisorConfig.MemSlots, + EntropySource: config.HypervisorConfig.EntropySource, + SharedFS: config.HypervisorConfig.SharedFS, + VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, + ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO, HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus, PCIeRootPort: config.HypervisorConfig.PCIeRootPort, SocketPath: socketPath, diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index 7c87e6a59..bfffa31a2 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -156,8 +156,10 @@ func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (boo } // GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group -// We can reuse this function at various leverls, sandbox, container. -func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIODev, error) { +// We can reuse this function at various levels, sandbox, container. +// Only the VFIO module is allowed to do bus assignments, all other modules need to +// ignore it if used as helper function to get VFIO information. +func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignment bool) ([]*config.VFIODev, error) { vfioDevs := []*config.VFIODev{} @@ -204,7 +206,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIOD IsPCIe: isPCIe, Class: pciClass, } - if isPCIe { + if isPCIe && !ignoreBusAssignment { vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) AllPCIeDevs[deviceBDF] = true } diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 4d06240b4..106220dcf 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "github.com/sirupsen/logrus" @@ -71,74 +70,11 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece } }() - device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(device.DeviceInfo) - - vfioGroup := filepath.Base(device.DeviceInfo.HostPath) - iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices") - - deviceFiles, err := os.ReadDir(iommuDevicesPath) + device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo, false) if err != nil { return err } - // Pass all devices in iommu group - for i, deviceFile := range deviceFiles { - //Get bdf of device eg 0000:00:1c.0 - deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath) - if err != nil { - return err - } - id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize) - - pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass) - // We need to ignore Host or PCI Bridges that are in the same IOMMU group as the - // passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge. - // Class 0x0604 is PCI bridge, 0x0600 is Host bridge - ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600) - if err != nil { - return err - } - if ignorePCIDevice { - continue - } - - var vfio config.VFIODev - - switch vfioDeviceType { - case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: - isPCIe := isPCIeDevice(deviceBDF) - // Do not directly assign to `vfio` -- need to access field still - vfioPCI := config.VFIOPCIDev{ - ID: id, - Type: vfioDeviceType, - BDF: deviceBDF, - SysfsDev: deviceSysfsDev, - IsPCIe: isPCIe, - Class: pciClass, - } - if isPCIe { - vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) - AllPCIeDevs[deviceBDF] = true - } - vfio = vfioPCI - case config.VFIOAPDeviceMediatedType: - devices, err := GetAPVFIODevices(deviceSysfsDev) - if err != nil { - return err - } - vfio = config.VFIOAPDev{ - ID: id, - SysfsDev: deviceSysfsDev, - Type: config.VFIOAPDeviceMediatedType, - APDevices: devices, - } - default: - return fmt.Errorf("Failed to append device: VFIO device type unrecognized") - } - - device.VfioDevs = append(device.VfioDevs, &vfio) - } - coldPlug := device.DeviceInfo.ColdPlug deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device") diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index b973063e8..bec0ed70d 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -319,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + ` hotplug_vfio_on_root_bus = ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + ` pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + ` + cold_plug_vfio = "` + config.ColdPlugVFIO + `" msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + ` enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + ` guest_hook_path = "` + config.DefaultGuestHookPath + `" diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index d65c93553..064aa72c7 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -718,20 +718,50 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi // At the sandbox level we alreaady checked that we have a // VFIO device, pass-through of a PCIe device needs allocated // mmemory in the firmware otherwise BARs cannot be mapped - if len(hypervisorConfig.VFIODevices) > 0 { - fwCfg := govmmQemu.FwCfg{ - Name: "opt/ovmf/X-PciMmio64Mb", - Str: "262144", - } - qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg) + // First check if we have a PCIe devices, otherwise ignore + err, fwCfg := q.appendFwCfgForConfidentialGuest(hypervisorConfig.VFIODevices) + if err != nil { + return err + } + if fwCfg != nil { + qemuConfig.FwCfg = append(qemuConfig.FwCfg, *fwCfg) } } - q.qemuConfig = qemuConfig return err } +// appendFwCfgForConfidentialGuest appends the firmware configuration for a +// VFIO and PCIe device, otherwise it will be ignored. +func (q *qemu) appendFwCfgForConfidentialGuest(vfioDevices []config.DeviceInfo) (error, *govmmQemu.FwCfg) { + var err error + for _, dev := range vfioDevices { + dev.HostPath, err = config.GetHostPath(dev, false, "") + if err != nil { + return err, nil + } + vfioDevs, err := drivers.GetAllVFIODevicesFromIOMMUGroup(dev, true) + if err != nil { + return err, nil + } + fwCfg := govmmQemu.FwCfg{} + for _, vfioDev := range vfioDevs { + switch (*vfioDev).GetType() { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + if (*vfioDev).(config.VFIOPCIDev).IsPCIe { + fwCfg = govmmQemu.FwCfg{ + Name: "opt/ovmf/X-PciMmio64Mb", + Str: "262144", + } + return nil, &fwCfg + } + } + } + } + return nil, nil +} + func (q *qemu) checkBpfEnabled() { if q.config.SeccompSandbox != "" { out, err := os.ReadFile("/proc/sys/net/core/bpf_jit_enable") diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 35b319394..ff3f7fe74 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -623,7 +623,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor // If we have a confidential guest we need to cold-plug the PCIe VFIO devices // until we have TDISP/IDE PCIe support. - coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort) + coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO == hv.RootPort) var devs []config.DeviceInfo for cnt, containers := range sandboxConfig.Containers { for dev, device := range containers.DeviceInfos {