Merge pull request #304 from fidencio/wip/forward_port_2703

[foward port] Add vIOMMU support to qemu q35

src/runtime/virtcontainers/hypervisor.go

@@ -358,6 +358,9 @@ type HypervisorConfig struct {
 	// VirtioMem is used to enable/disable virtio-mem
 	VirtioMem bool
 
+	// IOMMU specifies if the VM should have a vIOMMU
+	IOMMU bool
+
 	// Realtime Used to enable/disable realtime
 	Realtime bool
 

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -148,6 +148,9 @@ const (
 	// HugePages is a sandbox annotation to specify if the memory should be pre-allocated from huge pages
 	HugePages = kataAnnotHypervisorPrefix + "enable_hugepages"
 
+	// Iommu is a sandbox annotation to specify if the VM should have a vIOMMU device
+	IOMMU = kataAnnotHypervisorPrefix + "enable_iommu"
+
 	// FileBackedMemRootDir is a sandbox annotation to soecify file based memory backend root directory
 	FileBackedMemRootDir = kataAnnotHypervisorPrefix + "file_mem_backend"
 

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -539,6 +539,15 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig
 
 		sbConfig.HypervisorConfig.HugePages = hugePages
 	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.IOMMU]; ok {
+		iommu, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("Error parsing annotation for iommu: Please specify boolean value 'true|false'")
+		}
+
+		sbConfig.HypervisorConfig.IOMMU = iommu
+	}
 	return nil
 }
 

src/runtime/virtcontainers/pkg/oci/utils_test.go

@@ -771,6 +771,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.EnableSwap] = "true"
 	ocispec.Annotations[vcAnnotations.FileBackedMemRootDir] = "/dev/shm"
 	ocispec.Annotations[vcAnnotations.HugePages] = "true"
+	ocispec.Annotations[vcAnnotations.IOMMU] = "true"
 	ocispec.Annotations[vcAnnotations.BlockDeviceDriver] = "virtio-scsi"
 	ocispec.Annotations[vcAnnotations.DisableBlockDeviceUse] = "true"
 	ocispec.Annotations[vcAnnotations.EnableIOThreads] = "true"
@@ -802,6 +803,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(config.HypervisorConfig.Mlock, false)
 	assert.Equal(config.HypervisorConfig.FileBackedMemRootDir, "/dev/shm")
 	assert.Equal(config.HypervisorConfig.HugePages, true)
+	assert.Equal(config.HypervisorConfig.IOMMU, true)
 	assert.Equal(config.HypervisorConfig.BlockDeviceDriver, "virtio-scsi")
 	assert.Equal(config.HypervisorConfig.DisableBlockDeviceUse, true)
 	assert.Equal(config.HypervisorConfig.EnableIOThreads, true)

src/runtime/virtcontainers/qemu.go

@@ -409,6 +409,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
 		}
 	}
 
+	if q.config.IOMMU {
+		devices, err = q.arch.appendIOMMU(devices)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
 	var ioThread *govmmQemu.IOThread
 	if q.config.BlockDeviceDriver == config.VirtioSCSI {
 		return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)

src/runtime/virtcontainers/qemu_amd64.go

@@ -48,7 +48,6 @@ var kernelParams = []Param{
 	{"reboot", "k"},
 	{"console", "hvc0"},
 	{"console", "hvc1"},
-	{"iommu", "off"},
 	{"cryptomgr.notests", ""},
 	{"net.ifnames", "0"},
 	{"pci", "lastbus=0"},
@@ -89,12 +88,31 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 		factory = true
 	}
 
+	var qemuMachines = supportedQemuMachines
+	if config.IOMMU {
+		var q35QemuIOMMUOptions = "accel=kvm,kernel_irqchip=split"
+
+		kernelParams = append(kernelParams,
+			Param{"intel_iommu", "on"})
+		kernelParams = append(kernelParams,
+			Param{"iommu", "pt"})
+
+		for i, m := range qemuMachines {
+			if m.Type == QemuQ35 {
+				qemuMachines[i].Options = q35QemuIOMMUOptions
+			}
+		}
+	} else {
+		kernelParams = append(kernelParams,
+			Param{"iommu", "off"})
+	}
+
 	q := &qemuAmd64{
 		qemuArchBase: qemuArchBase{
 			machineType:           machineType,
 			memoryOffset:          config.MemOffset,
 			qemuPaths:             qemuPaths,
-			supportedQemuMachines: supportedQemuMachines,
+			supportedQemuMachines: qemuMachines,
 			kernelParamsNonDebug:  kernelParamsNonDebug,
 			kernelParamsDebug:     kernelParamsDebug,
 			kernelParams:          kernelParams,

src/runtime/virtcontainers/qemu_amd64_test.go

@@ -239,3 +239,19 @@ func TestQemuAmd64WithInitrd(t *testing.T) {
 		assert.NotContains(m.Options, qemuNvdimmOption)
 	}
 }
+
+func TestQemuAmd64Iommu(t *testing.T) {
+	assert := assert.New(t)
+
+	config := qemuConfig(QemuQ35)
+	config.IOMMU = true
+	qemu := newQemuArch(config)
+
+	p := qemu.kernelParameters(false)
+	assert.Contains(p, Param{"intel_iommu", "on"})
+
+	m, err := qemu.machine()
+
+	assert.NoError(err)
+	assert.Contains(m.Options, "kernel_irqchip=split")
+}

src/runtime/virtcontainers/qemu_arch_base.go

@@ -130,6 +130,9 @@ type qemuArch interface {
 
 	// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
 	appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
+
+	// append vIOMMU device
+	appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error)
 }
 
 type qemuArchBase struct {
@@ -765,4 +768,22 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
 // appendPCIeRootPortDevice appends to devices the given pcie-root-port
 func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
 	return genericAppendPCIeRootPort(devices, number, q.machineType)
+
+}
+
+// appendIOMMU appends a virtual IOMMU device
+func (q *qemuArchBase) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
+	switch q.machineType {
+	case QemuQ35:
+		iommu := govmmQemu.IommuDev{
+			Intremap:    true,
+			DeviceIotlb: true,
+			CachingMode: true,
+		}
+
+		devices = append(devices, iommu)
+		return devices, nil
+	default:
+		return devices, fmt.Errorf("Machine Type %s does not support vIOMMU", q.machineType)
+	}
 }

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -566,3 +566,27 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(expectedOut, devices)
 }
+
+func TestQemuArchBaseAppendIOMMU(t *testing.T) {
+	var devices []govmmQemu.Device
+	var err error
+	assert := assert.New(t)
+	qemuArchBase := newQemuArchBase()
+
+	expectedOut := []govmmQemu.Device{
+		govmmQemu.IommuDev{
+			Intremap:    true,
+			DeviceIotlb: true,
+			CachingMode: true,
+		},
+	}
+	// Test IOMMU is not appended to PC machine type
+	qemuArchBase.machineType = QemuPC
+	devices, err = qemuArchBase.appendIOMMU(devices)
+	assert.Error(err)
+
+	qemuArchBase.machineType = QemuQ35
+	devices, err = qemuArchBase.appendIOMMU(devices)
+	assert.NoError(err)
+	assert.Equal(expectedOut, devices)
+}

src/runtime/virtcontainers/qemu_arm64.go

@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"runtime"
 	"strings"
@@ -168,3 +169,7 @@ func (q *qemuArm64) setIgnoreSharedMemoryMigrationCaps(_ context.Context, _ *gov
 	// x-ignore-shared not support in arm64 for now
 	return nil
 }
+
+func (q *qemuArm64) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
+	return devices, fmt.Errorf("Arm64 architecture does not support vIOMMU")
+}

src/runtime/virtcontainers/qemu_ppc64le.go

@@ -6,6 +6,7 @@
 package virtcontainers
 
 import (
+	"fmt"
 	"time"
 
 	govmmQemu "github.com/intel/govmm/qemu"
@@ -121,3 +122,7 @@ func (q *qemuPPC64le) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8)
 func (q *qemuPPC64le) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device {
 	return genericAppendBridges(devices, q.Bridges, q.machineType)
 }
+
+func (q *qemuPPC64le) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
+	return devices, fmt.Errorf("PPC64le does not support appending a vIOMMU")
+}

src/runtime/virtcontainers/qemu_s390x.go

@@ -268,3 +268,7 @@ func (q *qemuS390x) appendVSock(devices []govmmQemu.Device, vsock types.VSock) (
 	return devices, nil
 
 }
+
+func (q *qemuS390x) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
+	return devices, fmt.Errorf("S390x does not support appending a vIOMMU")
+}