From 62aa6750eca8985503f43dd1377a3483253a2d78 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Thu, 20 Jul 2023 08:46:09 +0000 Subject: [PATCH] vfio: Added better handling of VFIO Control Devices Depending on the vfio_mode we need to mount the VFIO control device additionally into the container. Signed-off-by: Zvonko Kaiser --- src/runtime/pkg/device/drivers/utils.go | 1 + src/runtime/pkg/device/manager/manager.go | 6 ++-- src/runtime/pkg/device/manager/utils.go | 9 +++++- src/runtime/pkg/device/manager/utils_test.go | 2 +- src/runtime/virtcontainers/container.go | 8 ++++++ src/runtime/virtcontainers/sandbox.go | 29 +++++++++++++------- 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index 9ce16040b..4fa0fdd0e 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -216,6 +216,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe SysfsDev: deviceSysfsDev, Type: config.VFIOAPDeviceMediatedType, APDevices: devices, + Port: device.Port, } default: return nil, fmt.Errorf("Failed to append device: VFIO device type unrecognized") diff --git a/src/runtime/pkg/device/manager/manager.go b/src/runtime/pkg/device/manager/manager.go index 735061d9e..cb5e86a04 100644 --- a/src/runtime/pkg/device/manager/manager.go +++ b/src/runtime/pkg/device/manager/manager.go @@ -120,7 +120,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device if devInfo.ID, err = dm.newDeviceID(); err != nil { return nil, err } - if IsVFIO(devInfo.HostPath) { + if IsVFIODevice(devInfo.HostPath) { return drivers.NewVFIODevice(&devInfo), nil } else if IsVhostUserBlk(devInfo) { if devInfo.DriverOptions == nil { @@ -191,12 +191,12 @@ func (dm *deviceManager) AttachDevice(ctx context.Context, id string, dr api.Dev dm.Lock() defer dm.Unlock() - d, ok := dm.devices[id] + dev, ok := dm.devices[id] if !ok { return ErrDeviceNotExist } - if err := d.Attach(ctx, dr); err != nil { + if err := dev.Attach(ctx, dr); err != nil { return err } return nil diff --git a/src/runtime/pkg/device/manager/utils.go b/src/runtime/pkg/device/manager/utils.go index a9e4ee8c6..6658b19a4 100644 --- a/src/runtime/pkg/device/manager/utils.go +++ b/src/runtime/pkg/device/manager/utils.go @@ -17,8 +17,15 @@ const ( vfioPath = "/dev/vfio/" ) +// IsVFIOControlDevice checks if the device provided is a vfio control device. +// Depending no the vfio_mode we need to know if a device is a VFIO device +// or the VFIO control device +func IsVFIOControlDevice(path string) bool { + return path == filepath.Join(vfioPath, "vfio") +} + // IsVFIO checks if the device provided is a vfio group. -func IsVFIO(hostPath string) bool { +func IsVFIODevice(hostPath string) bool { // Ignore /dev/vfio/vfio character device if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) { return false diff --git a/src/runtime/pkg/device/manager/utils_test.go b/src/runtime/pkg/device/manager/utils_test.go index 6752719dd..9fbc829d7 100644 --- a/src/runtime/pkg/device/manager/utils_test.go +++ b/src/runtime/pkg/device/manager/utils_test.go @@ -31,7 +31,7 @@ func TestIsVFIO(t *testing.T) { } for _, d := range data { - isVFIO := IsVFIO(d.path) + isVFIO := IsVFIODevice(d.path) assert.Equal(t, d.expected, isVFIO) } } diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 1e2316236..4927bdab3 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -18,6 +18,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" + deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" @@ -872,9 +873,16 @@ func (c *Container) create(ctx context.Context) (err error) { // If cold-plug we've attached the devices already, do not try to // attach them a second time. coldPlugVFIO := (c.sandbox.config.HypervisorConfig.ColdPlugVFIO != config.NoPort) + modeVFIO := (c.sandbox.config.VfioMode == config.VFIOModeVFIO) + if coldPlugVFIO { var cntDevices []ContainerDevice for _, dev := range c.devices { + isVFIOControlDevice := deviceManager.IsVFIOControlDevice(dev.ContainerPath) + if isVFIOControlDevice && modeVFIO { + cntDevices = append(cntDevices, dev) + } + if strings.HasPrefix(dev.ContainerPath, vfioPath) { c.Logger().WithFields(logrus.Fields{ "device": dev, diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 52ed76567..9f18b5c2c 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -619,7 +619,9 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor // Aggregate all the containner devices for hot-plug and use them to dedcue // the correct amount of ports to reserve for the hypervisor. hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != config.NoPort) - stripVFIO := sandboxConfig.VfioMode == config.VFIOModeGuestKernel + + modeGK := (sandboxConfig.VfioMode == config.VFIOModeGuestKernel) + modeVFIO := (sandboxConfig.VfioMode == config.VFIOModeVFIO) var vfioDevices []config.DeviceInfo // vhost-user-block device is a PCIe device in Virt, keep track of it @@ -633,19 +635,26 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor vhostUserBlkDevices = append(vhostUserBlkDevices, device) continue } - isVFIO := deviceManager.IsVFIO(device.ContainerPath) - if hotPlugVFIO && isVFIO { + isVFIODevice := deviceManager.IsVFIODevice(device.ContainerPath) + isVFIOControlDevice := deviceManager.IsVFIOControlDevice(device.ContainerPath) + // vfio_mode=vfio needs the VFIO control device add it to the list + // of devices to be added to the VM. + if modeVFIO && isVFIOControlDevice { + vfioDevices = append(vfioDevices, device) + } + + if hotPlugVFIO && isVFIODevice { vfioDevices = append(vfioDevices, device) sandboxConfig.Containers[cnt].DeviceInfos[dev].Port = sandboxConfig.HypervisorConfig.HotPlugVFIO } - if coldPlugVFIO && isVFIO { + if coldPlugVFIO && isVFIODevice { device.ColdPlug = true device.Port = sandboxConfig.HypervisorConfig.ColdPlugVFIO vfioDevices = append(vfioDevices, device) // We need to remove the devices marked for cold-plug // otherwise at the container level the kata-agent // will try to hot-plug them. - if stripVFIO { + if modeGK { sandboxConfig.Containers[cnt].DeviceInfos[dev].ID = "remove-we-are-cold-plugging" } } @@ -2053,26 +2062,26 @@ func (s *Sandbox) AddDevice(ctx context.Context, info config.DeviceInfo) (api.De } var err error - b, err := s.devManager.NewDevice(info) + add, err := s.devManager.NewDevice(info) if err != nil { return nil, err } defer func() { if err != nil { - s.devManager.RemoveDevice(b.DeviceID()) + s.devManager.RemoveDevice(add.DeviceID()) } }() - if err = s.devManager.AttachDevice(ctx, b.DeviceID(), s); err != nil { + if err = s.devManager.AttachDevice(ctx, add.DeviceID(), s); err != nil { return nil, err } defer func() { if err != nil { - s.devManager.DetachDevice(ctx, b.DeviceID(), s) + s.devManager.DetachDevice(ctx, add.DeviceID(), s) } }() - return b, nil + return add, nil } // updateResources will: