diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index c7e919d62..f17480aba 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -307,17 +307,17 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) { } return HypervisorInfo{ - Debug: config.HypervisorConfig.Debug, - MachineType: config.HypervisorConfig.HypervisorMachineType, - Version: version, - Path: hypervisorPath, - BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, - Msize9p: config.HypervisorConfig.Msize9p, - MemorySlots: config.HypervisorConfig.MemSlots, - EntropySource: config.HypervisorConfig.EntropySource, - SharedFS: config.HypervisorConfig.SharedFS, - VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, - + Debug: config.HypervisorConfig.Debug, + MachineType: config.HypervisorConfig.HypervisorMachineType, + Version: version, + Path: hypervisorPath, + BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, + Msize9p: config.HypervisorConfig.Msize9p, + MemorySlots: config.HypervisorConfig.MemSlots, + EntropySource: config.HypervisorConfig.EntropySource, + SharedFS: config.HypervisorConfig.SharedFS, + VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, + ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO, HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus, PCIeRootPort: config.HypervisorConfig.PCIeRootPort, SocketPath: socketPath, diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index 7c87e6a59..bfffa31a2 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -156,8 +156,10 @@ func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (boo } // GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group -// We can reuse this function at various leverls, sandbox, container. -func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIODev, error) { +// We can reuse this function at various levels, sandbox, container. +// Only the VFIO module is allowed to do bus assignments, all other modules need to +// ignore it if used as helper function to get VFIO information. +func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignment bool) ([]*config.VFIODev, error) { vfioDevs := []*config.VFIODev{} @@ -204,7 +206,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIOD IsPCIe: isPCIe, Class: pciClass, } - if isPCIe { + if isPCIe && !ignoreBusAssignment { vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) AllPCIeDevs[deviceBDF] = true } diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 4d06240b4..106220dcf 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "github.com/sirupsen/logrus" @@ -71,74 +70,11 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece } }() - device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(device.DeviceInfo) - - vfioGroup := filepath.Base(device.DeviceInfo.HostPath) - iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices") - - deviceFiles, err := os.ReadDir(iommuDevicesPath) + device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo, false) if err != nil { return err } - // Pass all devices in iommu group - for i, deviceFile := range deviceFiles { - //Get bdf of device eg 0000:00:1c.0 - deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath) - if err != nil { - return err - } - id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize) - - pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass) - // We need to ignore Host or PCI Bridges that are in the same IOMMU group as the - // passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge. - // Class 0x0604 is PCI bridge, 0x0600 is Host bridge - ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600) - if err != nil { - return err - } - if ignorePCIDevice { - continue - } - - var vfio config.VFIODev - - switch vfioDeviceType { - case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: - isPCIe := isPCIeDevice(deviceBDF) - // Do not directly assign to `vfio` -- need to access field still - vfioPCI := config.VFIOPCIDev{ - ID: id, - Type: vfioDeviceType, - BDF: deviceBDF, - SysfsDev: deviceSysfsDev, - IsPCIe: isPCIe, - Class: pciClass, - } - if isPCIe { - vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) - AllPCIeDevs[deviceBDF] = true - } - vfio = vfioPCI - case config.VFIOAPDeviceMediatedType: - devices, err := GetAPVFIODevices(deviceSysfsDev) - if err != nil { - return err - } - vfio = config.VFIOAPDev{ - ID: id, - SysfsDev: deviceSysfsDev, - Type: config.VFIOAPDeviceMediatedType, - APDevices: devices, - } - default: - return fmt.Errorf("Failed to append device: VFIO device type unrecognized") - } - - device.VfioDevs = append(device.VfioDevs, &vfio) - } - coldPlug := device.DeviceInfo.ColdPlug deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device") diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index b973063e8..bec0ed70d 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -319,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + ` hotplug_vfio_on_root_bus = ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + ` pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + ` + cold_plug_vfio = "` + config.ColdPlugVFIO + `" msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + ` enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + ` guest_hook_path = "` + config.DefaultGuestHookPath + `" diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index d65c93553..064aa72c7 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -718,20 +718,50 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi // At the sandbox level we alreaady checked that we have a // VFIO device, pass-through of a PCIe device needs allocated // mmemory in the firmware otherwise BARs cannot be mapped - if len(hypervisorConfig.VFIODevices) > 0 { - fwCfg := govmmQemu.FwCfg{ - Name: "opt/ovmf/X-PciMmio64Mb", - Str: "262144", - } - qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg) + // First check if we have a PCIe devices, otherwise ignore + err, fwCfg := q.appendFwCfgForConfidentialGuest(hypervisorConfig.VFIODevices) + if err != nil { + return err + } + if fwCfg != nil { + qemuConfig.FwCfg = append(qemuConfig.FwCfg, *fwCfg) } } - q.qemuConfig = qemuConfig return err } +// appendFwCfgForConfidentialGuest appends the firmware configuration for a +// VFIO and PCIe device, otherwise it will be ignored. +func (q *qemu) appendFwCfgForConfidentialGuest(vfioDevices []config.DeviceInfo) (error, *govmmQemu.FwCfg) { + var err error + for _, dev := range vfioDevices { + dev.HostPath, err = config.GetHostPath(dev, false, "") + if err != nil { + return err, nil + } + vfioDevs, err := drivers.GetAllVFIODevicesFromIOMMUGroup(dev, true) + if err != nil { + return err, nil + } + fwCfg := govmmQemu.FwCfg{} + for _, vfioDev := range vfioDevs { + switch (*vfioDev).GetType() { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + if (*vfioDev).(config.VFIOPCIDev).IsPCIe { + fwCfg = govmmQemu.FwCfg{ + Name: "opt/ovmf/X-PciMmio64Mb", + Str: "262144", + } + return nil, &fwCfg + } + } + } + } + return nil, nil +} + func (q *qemu) checkBpfEnabled() { if q.config.SeccompSandbox != "" { out, err := os.ReadFile("/proc/sys/net/core/bpf_jit_enable") diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 35b319394..ff3f7fe74 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -623,7 +623,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor // If we have a confidential guest we need to cold-plug the PCIe VFIO devices // until we have TDISP/IDE PCIe support. - coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort) + coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO == hv.RootPort) var devs []config.DeviceInfo for cnt, containers := range sandboxConfig.Containers { for dev, device := range containers.DeviceInfos {