From c2e9801a97e0b664581f35869738f8a210d825c9 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 9 Mar 2018 10:07:57 -0800 Subject: [PATCH 1/4] virtcontainers: kata_agent: Rely on Storage instead of Device for rootfs The Kata agent has been reworked regarding Storage and Device structures recently. It now expect that Storage is going to be used for both volumes/mounts and rootfs, while Device will be exclusive to devices passed through the VM that should appear inside the container. This commit implements this new way of handling volumes and devices. Fixes #56 Signed-off-by: Sebastien Boeuf --- virtcontainers/kata_agent.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index be12106fd..a8017fa58 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -49,6 +49,7 @@ var ( type9pFs = "9p" devPath = "/dev" vsockSocketScheme = "vsock" + kata9pDevType = "9p" kataBlkDevType = "blk" ) @@ -471,6 +472,7 @@ func (k *kataAgent) startPod(pod Pod) error { // (resolv.conf, etc...) and potentially all container // rootfs will reside. sharedVolume := &grpc.Storage{ + Driver: kata9pDevType, Source: mountGuest9pTag, MountPoint: kataGuestSharedDir, Fstype: type9pFs, @@ -611,14 +613,7 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (*Process, error) { // without trying to match and update it into the OCI spec list // of actual devices. The device corresponding to the rootfs is // a very specific case. - rootfsDevice := &grpc.Device{ - Type: kataBlkDevType, - VmPath: virtPath, - ContainerPath: "", - } - - ctrDevices = append(ctrDevices, rootfsDevice) - + rootfs.Driver = kataBlkDevType rootfs.Source = virtPath rootfs.MountPoint = rootPathParent rootfs.Fstype = c.state.Fstype From 8152e15a61f42356af178a0722120ff2626f2916 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 9 Mar 2018 10:22:55 -0800 Subject: [PATCH 2/4] virtcontainers: kata_agent: Factorize appending devices This commit factorizes the code appending devices to the device list provided to the Kata agent, in order to reduce the complexity of the function createContainer(). Fixes #56 Signed-off-by: Sebastien Boeuf --- virtcontainers/kata_agent.go | 37 ++++++++++++++----------- virtcontainers/kata_agent_test.go | 45 ++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index a8017fa58..7e39b7678 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -579,6 +579,25 @@ func constraintGRPCSpec(grpcSpec *grpc.Spec) { } } +func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []Device) []*grpc.Device { + for _, device := range devices { + d, ok := device.(*BlockDevice) + if !ok { + continue + } + + kataDevice := &grpc.Device{ + Type: kataBlkDevType, + VmPath: d.VirtPath, + ContainerPath: d.DeviceInfo.ContainerPath, + } + + deviceList = append(deviceList, kataDevice) + } + + return deviceList +} + func (k *kataAgent) createContainer(pod *Pod, c *Container) (*Process, error) { ociSpecJSON, ok := c.config.Annotations[vcAnnotations.ConfigJSONKey] if !ok { @@ -673,22 +692,8 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (*Process, error) { // irrelevant information to the agent. constraintGRPCSpec(grpcSpec) - // Append container mounts for block devices passed with --device. - for _, device := range c.devices { - d, ok := device.(*BlockDevice) - - if !ok { - continue - } - - kataDevice := &grpc.Device{ - Type: kataBlkDevType, - VmPath: d.VirtPath, - ContainerPath: d.DeviceInfo.ContainerPath, - } - - ctrDevices = append(ctrDevices, kataDevice) - } + // Append container devices for block devices passed with --device. + ctrDevices = k.appendDevices(ctrDevices, c.devices) req := &grpc.CreateContainerRequest{ ContainerId: c.id, diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 8440636e1..da2e83824 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -33,7 +33,11 @@ import ( "google.golang.org/grpc" ) -var testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock" +var ( + testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock" + testBlockDeviceVirtPath = "testBlockDeviceVirtPath" + testBlockDeviceCtrPath = "testBlockDeviceCtrPath" +) func proxyHandlerDiscard(c net.Conn) { buf := make([]byte, 1024) @@ -336,3 +340,42 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) } + +func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { + k := kataAgent{} + + devList := []*pb.Device{} + expected := []*pb.Device{} + ctrDevices := []Device{} + + updatedDevList := k.appendDevices(devList, ctrDevices) + assert.True(t, reflect.DeepEqual(updatedDevList, expected), + "Device lists didn't match: got %+v, expecting %+v", + updatedDevList, expected) +} + +func TestAppendDevices(t *testing.T) { + k := kataAgent{} + + devList := []*pb.Device{} + expected := []*pb.Device{ + { + Type: kataBlkDevType, + VmPath: testBlockDeviceVirtPath, + ContainerPath: testBlockDeviceCtrPath, + }, + } + ctrDevices := []Device{ + &BlockDevice{ + VirtPath: testBlockDeviceVirtPath, + DeviceInfo: DeviceInfo{ + ContainerPath: testBlockDeviceCtrPath, + }, + }, + } + + updatedDevList := k.appendDevices(devList, ctrDevices) + assert.True(t, reflect.DeepEqual(updatedDevList, expected), + "Device lists didn't match: got %+v, expecting %+v", + updatedDevList, expected) +} From 7e85032aa7d3953d83e9dbd82e6ed2ca2e3f73a3 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 9 Mar 2018 10:26:57 -0800 Subject: [PATCH 3/4] virtcontainers: kata_agent: Use 9p2000.L version for 9p There is a shared directory shared through virtio-9p between the host and guest OS. The version of the driver used matters as it may improve a few things. In this case, using the specific version 9p2000.L does not result in any regression, and it fixes failures related to symlinks being passed through 9p. Fixes #56 Signed-off-by: Sebastien Boeuf --- virtcontainers/kata_agent.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 7e39b7678..5a2a88f36 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -51,6 +51,7 @@ var ( vsockSocketScheme = "vsock" kata9pDevType = "9p" kataBlkDevType = "blk" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L", "nodev"} ) // KataAgentConfig is a structure storing information needed @@ -476,7 +477,7 @@ func (k *kataAgent) startPod(pod Pod) error { Source: mountGuest9pTag, MountPoint: kataGuestSharedDir, Fstype: type9pFs, - Options: []string{"trans=virtio", "nodev"}, + Options: sharedDir9pOptions, } req := &grpc.CreateSandboxRequest{ From 21f89119605bed6b8692e681e904f70760eda4ec Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 9 Mar 2018 10:33:38 -0800 Subject: [PATCH 4/4] virtcontainers: kata_agent: Add virtio-scsi support This commit enables the support of virtio-scsi for block devices used as root filesystem or as devices passed through the VM to be used directly from the container. Fixes #56 Signed-off-by: Sebastien Boeuf --- virtcontainers/kata_agent.go | 47 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 5a2a88f36..6a8f593d6 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -51,6 +51,7 @@ var ( vsockSocketScheme = "vsock" kata9pDevType = "9p" kataBlkDevType = "blk" + kataSCSIDevType = "scsi" sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L", "nodev"} ) @@ -588,11 +589,17 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []Device) [ } kataDevice := &grpc.Device{ - Type: kataBlkDevType, - VmPath: d.VirtPath, ContainerPath: d.DeviceInfo.ContainerPath, } + if d.SCSIAddr == "" { + kataDevice.Type = kataBlkDevType + kataDevice.VmPath = d.VirtPath + } else { + kataDevice.Type = kataSCSIDevType + kataDevice.Id = d.SCSIAddr + } + deviceList = append(deviceList, kataDevice) } @@ -621,20 +628,30 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (*Process, error) { if c.state.Fstype != "" { // This is a block based device rootfs. - // driveName is the predicted virtio-block guest name (the vd* in /dev/vd*). - driveName, err := getVirtDriveName(c.state.BlockIndex) - if err != nil { - return nil, err - } - virtPath := filepath.Join(devPath, driveName) - // Create a new device with empty ContainerPath so that we get - // the device being waited for by the agent inside the VM, - // without trying to match and update it into the OCI spec list - // of actual devices. The device corresponding to the rootfs is - // a very specific case. - rootfs.Driver = kataBlkDevType - rootfs.Source = virtPath + // Pass a drive name only in case of virtio-blk driver. + // If virtio-scsi driver, the agent will be able to find the + // device based on the provided address. + if pod.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock { + // driveName is the predicted virtio-block guest name (the vd* in /dev/vd*). + driveName, err := getVirtDriveName(c.state.BlockIndex) + if err != nil { + return nil, err + } + virtPath := filepath.Join(devPath, driveName) + + rootfs.Driver = kataBlkDevType + rootfs.Source = virtPath + } else { + scsiAddr, err := getSCSIAddress(c.state.BlockIndex) + if err != nil { + return nil, err + } + + rootfs.Driver = kataSCSIDevType + rootfs.Source = scsiAddr + } + rootfs.MountPoint = rootPathParent rootfs.Fstype = c.state.Fstype