From 2216d528f6bca0badb13027be7b310f8c03999af Mon Sep 17 00:00:00 2001 From: Zhao Xinda Date: Thu, 20 Sep 2018 14:55:20 +0800 Subject: [PATCH 1/2] vendor: Update govmm to add vfio mediated device support on root bus In addition to supporting hotplug for VFIO mediated device on PCI bridge, this patch adds hotplug functionality on root bus. When parameter bus and addr are set to be empty, the system will pick up an empty slot on root bus. Fixes #542 Signed-off-by: Zhao Xinda --- Gopkg.lock | 2 +- Gopkg.toml | 2 +- vendor/github.com/intel/govmm/qemu/qmp.go | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 96a608433..d95c83259 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -127,7 +127,7 @@ name = "github.com/intel/govmm" packages = ["qemu"] pruneopts = "NUT" - revision = "032705ba6aae05a9bf41e296cf89c8529cffb822" + revision = "9905ae92c5915c07abeb669eaa4d7f7408834b51" [[projects]] digest = "1:672470f31bc4e50f9ba09a1af7ab6035bf8b1452db64dfd79b1a22614bb30710" diff --git a/Gopkg.toml b/Gopkg.toml index 013fd3fb6..eb4723c04 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -52,7 +52,7 @@ [[constraint]] name = "github.com/intel/govmm" - revision = "032705ba6aae05a9bf41e296cf89c8529cffb822" + revision = "9905ae92c5915c07abeb669eaa4d7f7408834b51" [[constraint]] name = "github.com/kata-containers/agent" diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go index e9fa65e52..4f981f6bb 100644 --- a/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/vendor/github.com/intel/govmm/qemu/qmp.go @@ -946,11 +946,13 @@ func (q *QMP) ExecutePCIVFIOMediatedDeviceAdd(ctx context.Context, devID, sysfsd "id": devID, "driver": "vfio-pci", "sysfsdev": sysfsdev, - "addr": addr, } if bus != "" { args["bus"] = bus } + if addr != "" { + args["addr"] = addr + } return q.executeCommand(ctx, "device_add", args, nil) } From 37b83c89236131fba265dc5975d017a3c84099c9 Mon Sep 17 00:00:00 2001 From: Zhao Xinda Date: Thu, 20 Sep 2018 15:27:34 +0800 Subject: [PATCH 2/2] device: Add GPU device support Enable GPU device support in kata runtime, including GVT-g and GVT-d. GVT-g: graphic virtualization technology with mediated pass through GVT-d: graphic virtualization technology with direct pass through BDF of device eg "0000:00:1c.0" is used to distinguish GPU device in GVT-d, while sysfsdev of device eg "f79944e4-5a3d-11e8-99ce-479cbab002e4" is used in GVT-g. Fixes #542 Signed-off-by: Zhao Xinda --- virtcontainers/device/config/config.go | 21 +++++++++ virtcontainers/device/drivers/vfio.go | 54 +++++++++++++++++----- virtcontainers/device/drivers/vfio_test.go | 22 +++++++-- virtcontainers/qemu.go | 18 ++++++-- 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go index 280a3e7d3..5f06d10c5 100644 --- a/virtcontainers/device/config/config.go +++ b/virtcontainers/device/config/config.go @@ -109,12 +109,33 @@ type BlockDrive struct { VirtPath string } +// VFIODeviceType indicates VFIO device type +type VFIODeviceType uint32 + +const ( + // VFIODeviceErrorType is the error type of VFIO device + VFIODeviceErrorType VFIODeviceType = iota + + // VFIODeviceNormalType is a normal VFIO device type + VFIODeviceNormalType + + // VFIODeviceMediatedType is a VFIO mediated device type + VFIODeviceMediatedType +) + // VFIODev represents a VFIO drive used for hotplugging type VFIODev struct { // ID is used to identify this drive in the hypervisor options. ID string + + // Type of VFIO device + Type VFIODeviceType + // BDF (Bus:Device.Function) of the PCI address BDF string + + // sysfsdev of VFIO mediated device + SysfsDev string } // RNGDev represents a random number generator device diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index 7b58eb781..06568c07c 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -67,13 +67,15 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error { // Pass all devices in iommu group for i, deviceFile := range deviceFiles { //Get bdf of device eg 0000:00:1c.0 - deviceBDF, err := getBDF(deviceFile.Name()) + deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath) if err != nil { return err } vfio := &config.VFIODev{ - ID: utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize), - BDF: deviceBDF, + ID: utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize), + Type: vfioDeviceType, + BDF: deviceBDF, + SysfsDev: deviceSysfsDev, } device.vfioDevs = append(device.vfioDevs, vfio) } @@ -130,17 +132,45 @@ func (device *VFIODevice) GetDeviceInfo() interface{} { // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes -// getBDF returns the BDF of pci device -// Expected input strng format is []:[][].[] eg. 0000:02:10.0 -func getBDF(deviceSysStr string) (string, error) { - tokens := strings.Split(deviceSysStr, ":") - - if len(tokens) != 3 { - return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr) +func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) { + tokens := strings.Split(deviceFileName, ":") + vfioDeviceType = config.VFIODeviceErrorType + if len(tokens) == 3 { + vfioDeviceType = config.VFIODeviceNormalType + } else { + tokens = strings.Split(deviceFileName, "-") + if len(tokens) == 5 { + vfioDeviceType = config.VFIODeviceMediatedType + } } - tokens = strings.SplitN(deviceSysStr, ":", 2) - return tokens[1], nil + switch vfioDeviceType { + case config.VFIODeviceNormalType: + // Get bdf of device eg. 0000:00:1c.0 + deviceBDF = getBDF(deviceFileName) + case config.VFIODeviceMediatedType: + // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + deviceSysfsDev, err = getSysfsDev(sysfsDevStr) + default: + err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName) + } + + return deviceBDF, deviceSysfsDev, vfioDeviceType, err +} + +// getBDF returns the BDF of pci device +// Expected input string format is []:[][].[] eg. 0000:02:10.0 +func getBDF(deviceSysStr string) string { + tokens := strings.SplitN(deviceSysStr, ":", 2) + return tokens[1] +} + +// getSysfsDev returns the sysfsdev of mediated device +// Expected input string format is absolute path to the sysfs dev node +// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 +func getSysfsDev(sysfsDevStr string) (string, error) { + return filepath.EvalSymlinks(sysfsDevStr) } // BindDevicetoVFIO binds the device to vfio driver after unbinding from host. diff --git a/virtcontainers/device/drivers/vfio_test.go b/virtcontainers/device/drivers/vfio_test.go index cf60098ad..59a441fc1 100644 --- a/virtcontainers/device/drivers/vfio_test.go +++ b/virtcontainers/device/drivers/vfio_test.go @@ -9,26 +9,38 @@ package drivers import ( "testing" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/stretchr/testify/assert" ) -func TestGetBDF(t *testing.T) { +func TestGetVFIODetails(t *testing.T) { type testData struct { deviceStr string - expectedBDF string + expectedStr string } data := []testData{ {"0000:02:10.0", "02:10.0"}, {"0000:0210.0", ""}, + {"f79944e4-5a3d-11e8-99ce-", ""}, + {"f79944e4-5a3d-11e8-99ce", ""}, {"test", ""}, {"", ""}, } for _, d := range data { - deviceBDF, err := getBDF(d.deviceStr) - assert.Equal(t, d.expectedBDF, deviceBDF) - if d.expectedBDF == "" { + deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "") + + switch vfioDeviceType { + case config.VFIODeviceNormalType: + assert.Equal(t, d.expectedStr, deviceBDF) + case config.VFIODeviceMediatedType: + assert.Equal(t, d.expectedStr, deviceSysfsDev) + default: + assert.NotNil(t, err) + } + + if d.expectedStr == "" { assert.NotNil(t, err) } else { assert.Nil(t, err) diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 337cc68b8..e2c236ccc 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -795,7 +795,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. if q.state.HotplugVFIOOnRootBus { - return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF) + switch device.Type { + case config.VFIODeviceNormalType: + return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF) + case config.VFIODeviceMediatedType: + return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", "") + default: + return fmt.Errorf("Incorrect VFIO device type found") + } } addr, bridge, err := q.addDeviceToBridge(devID) @@ -803,8 +810,13 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { return err } - if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID); err != nil { - return err + switch device.Type { + case config.VFIODeviceNormalType: + return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID) + case config.VFIODeviceMediatedType: + return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID) + default: + return fmt.Errorf("Incorrect VFIO device type found") } } else { if !q.state.HotplugVFIOOnRootBus {