From 8ae28888e0350613d55db4cff4f3b73301564618 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 25 Jul 2018 07:27:05 -0500 Subject: [PATCH 01/10] vendor: update govmm add vhostfd and disable-modern to vhost-vsock-pci shortlog: 3830b44 qemu: add vhostfd and disable-modern to vhost-vsock-pci f700a97 qemu/qmp: implement function to hotplug vsock-pci Signed-off-by: Julio Montes --- Gopkg.lock | 4 ++-- Gopkg.toml | 2 +- vendor/github.com/intel/govmm/qemu/qemu.go | 23 ++++++++++++++++++++-- vendor/github.com/intel/govmm/qemu/qmp.go | 10 ++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f9b877531..c5dc1fc7f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -92,7 +92,7 @@ [[projects]] name = "github.com/intel/govmm" packages = ["qemu"] - revision = "ff2401825e0930811919c86c36d64b113aa00083" + revision = "6ff20ae2f409df976574d0139b5ec2fa3e314769" [[projects]] name = "github.com/kata-containers/agent" @@ -264,6 +264,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "ea3d6532c4375832a1c79d70af45e6722e526bde97f6caf23d90b91267a3cf0b" + inputs-digest = "f57c595789df5fbc01e237cc52c7454911dbc37b9282304b6b1f076606e4c157" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 2264af592..9fe693641 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -56,7 +56,7 @@ [[constraint]] name = "github.com/intel/govmm" - revision = "ff2401825e0930811919c86c36d64b113aa00083" + revision = "6ff20ae2f409df976574d0139b5ec2fa3e314769" [[constraint]] name = "github.com/kata-containers/agent" diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go index 41381d80e..4f7a383ee 100644 --- a/vendor/github.com/intel/govmm/qemu/qemu.go +++ b/vendor/github.com/intel/govmm/qemu/qemu.go @@ -82,6 +82,9 @@ const ( // VirtioSerialPort is the serial port device driver. VirtioSerialPort = "virtserialport" + + // VHostVSockPCI is the vhost vsock pci driver. + VHostVSockPCI = "vhost-vsock-pci" ) // ObjectType is a string representing a qemu object type. @@ -962,6 +965,12 @@ type VSOCKDevice struct { ID string ContextID uint32 + + // VHostFD vhost file descriptor that holds the ContextID + VHostFD *os.File + + // DisableModern prevents qemu from relying on fast MMIO. + DisableModern bool } const ( @@ -986,12 +995,22 @@ func (vsock VSOCKDevice) Valid() bool { // QemuParams returns the qemu parameters built out of the VSOCK device. func (vsock VSOCKDevice) QemuParams(config *Config) []string { + var deviceParams []string var qemuParams []string - deviceParam := fmt.Sprintf("%s,id=%s,%s=%d", VhostVSOCKPCI, vsock.ID, VSOCKGuestCID, vsock.ContextID) + deviceParams = append(deviceParams, fmt.Sprintf("%s", VhostVSOCKPCI)) + if vsock.DisableModern { + deviceParams = append(deviceParams, ",disable-modern=true") + } + if vsock.VHostFD != nil { + qemuFDs := config.appendFDs([]*os.File{vsock.VHostFD}) + deviceParams = append(deviceParams, fmt.Sprintf(",vhostfd=%d", qemuFDs[0])) + } + deviceParams = append(deviceParams, fmt.Sprintf(",id=%s", vsock.ID)) + deviceParams = append(deviceParams, fmt.Sprintf(",%s=%d", VSOCKGuestCID, vsock.ContextID)) qemuParams = append(qemuParams, "-device") - qemuParams = append(qemuParams, deviceParam) + qemuParams = append(qemuParams, strings.Join(deviceParams, "")) return qemuParams } diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go index 37334e99e..32a59ef04 100644 --- a/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/vendor/github.com/intel/govmm/qemu/qmp.go @@ -881,3 +881,13 @@ func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string return err } + +// ExecutePCIVSockAdd adds a vhost-vsock-pci bus +func (q *QMP) ExecutePCIVSockAdd(ctx context.Context, id, guestCID string) error { + args := map[string]interface{}{ + "driver": VHostVSockPCI, + "id": id, + "guest-cid": guestCID, + } + return q.executeCommand(ctx, "device_add", args, nil) +} From 2339ac3f93ddaa2a60d47386df32563ac7cc2b2c Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Thu, 19 Jul 2018 08:44:55 -0500 Subject: [PATCH 02/10] virtcontainers/utils: Implement function to check vsocks support Implement function to check if the system has support for vsocks. This function looks for vsock and vhost-vsock devices returning true if those exist, otherwise false. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Julio Montes --- virtcontainers/utils/utils.go | 19 ++++++++++++++++++ virtcontainers/utils/utils_test.go | 31 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go index 15eab237d..8c2814b64 100644 --- a/virtcontainers/utils/utils.go +++ b/virtcontainers/utils/utils.go @@ -23,6 +23,12 @@ const fileMode0755 = os.FileMode(0755) // See unix(7). const MaxSocketPathLen = 107 +// VSockDevicePath path to vsock device +var VSockDevicePath = "/dev/vsock" + +// VHostVSockDevicePath path to vhost-vsock device +var VHostVSockDevicePath = "/dev/vhost-vsock" + // FileCopy copys files from srcPath to dstPath func FileCopy(srcPath, dstPath string) error { if srcPath == "" { @@ -200,3 +206,16 @@ func BuildSocketPath(elements ...string) (string, error) { return result, nil } + +// SupportsVsocks returns true if vsocks are supported, otherwise false +func SupportsVsocks() bool { + if _, err := os.Stat(VSockDevicePath); err != nil { + return false + } + + if _, err := os.Stat(VHostVSockDevicePath); err != nil { + return false + } + + return true +} diff --git a/virtcontainers/utils/utils_test.go b/virtcontainers/utils/utils_test.go index f738c235c..51375ebc6 100644 --- a/virtcontainers/utils/utils_test.go +++ b/virtcontainers/utils/utils_test.go @@ -294,3 +294,34 @@ func TestBuildSocketPath(t *testing.T) { assert.Equal(d.expected, result) } } + +func TestSupportsVsocks(t *testing.T) { + assert := assert.New(t) + + orgVSockDevicePath := VSockDevicePath + orgVHostVSockDevicePath := VHostVSockDevicePath + defer func() { + VSockDevicePath = orgVSockDevicePath + VHostVSockDevicePath = orgVHostVSockDevicePath + }() + + VSockDevicePath = "/abc/xyz/123" + VHostVSockDevicePath = "/abc/xyz/123" + assert.False(SupportsVsocks()) + + vSockDeviceFile, err := ioutil.TempFile("", "vsock") + assert.NoError(err) + defer os.Remove(vSockDeviceFile.Name()) + defer vSockDeviceFile.Close() + VSockDevicePath = vSockDeviceFile.Name() + + assert.False(SupportsVsocks()) + + vHostVSockFile, err := ioutil.TempFile("", "vhost-vsock") + assert.NoError(err) + defer os.Remove(vHostVSockFile.Name()) + defer vHostVSockFile.Close() + VHostVSockDevicePath = vHostVSockFile.Name() + + assert.True(SupportsVsocks()) +} From 9b283254c3f0cfed20c1fccb087ca98dd61277ae Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 23 Jul 2018 17:51:56 -0500 Subject: [PATCH 03/10] virtcontainers: Implement function to get a free context ID FindContextID generates a random number between 3 and max uint32 and uses it as context ID. Using ioctl findContextID checks if the context ID is free, if the context ID is being used by other process, this function iterates from over all valid context IDs until one is available. `/dev/vhost-vsock` is used to check what context IDs are free, we need it to ensure we are using a unique context ID to create the vsocks. Signed-off-by: Julio Montes --- virtcontainers/utils/utils_linux.go | 89 ++++++++++++++++++++++++ virtcontainers/utils/utils_linux_test.go | 35 ++++++++++ 2 files changed, 124 insertions(+) create mode 100644 virtcontainers/utils/utils_linux.go create mode 100644 virtcontainers/utils/utils_linux_test.go diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go new file mode 100644 index 000000000..5d73c2ad1 --- /dev/null +++ b/virtcontainers/utils/utils_linux.go @@ -0,0 +1,89 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package utils + +import ( + "crypto/rand" + "fmt" + "math/big" + "os" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +// from +// VHOST_VSOCK_SET_GUEST_CID = _IOW(VHOST_VIRTIO, 0x60, __u64) +const ioctlVhostVsockSetGuestCid = 0x4008AF60 + +var ioctlFunc = ioctl + +var maxUInt uint32 = 1<<32 - 1 + +func ioctl(fd uintptr, request int, arg1 uint32) error { + if _, _, errno := unix.Syscall( + unix.SYS_IOCTL, + fd, + uintptr(request), + uintptr(unsafe.Pointer(&arg1)), + ); errno != 0 { + return os.NewSyscallError("ioctl", fmt.Errorf("%d", int(errno))) + } + + return nil +} + +// FindContextID finds a unique context ID by generating a random number between 3 and max unsigned int (maxUint). +// Using the ioctl VHOST_VSOCK_SET_GUEST_CID, findContextID asks to the kernel if the given +// context ID (N) is available, when the context ID is not available, incrementing by 1 findContextID +// iterates from N to maxUint until an available context ID is found, otherwise decrementing by 1 +// findContextID iterates from N to 3 until an available context ID is found, this is the last chance +// to find a context ID available. +// On success vhost file and a context ID greater or equal than 3 are returned, otherwise 0 and an error are returned. +// vhost file can be used to send vhost file decriptor to QEMU. It's the caller's responsibility to +// close vhost file descriptor. +// +// Benefits of using random context IDs: +// - Reduce the probability of a *DoS attack*, since other processes don't know whatis the initial context ID +// used by findContextID to find a context ID available +// +func FindContextID() (*os.File, uint32, error) { + // context IDs 0x0, 0x1 and 0x2 are reserved, 0x3 is the first context ID usable. + var firstContextID uint32 = 0x3 + var contextID = firstContextID + + // Generate a random number + n, err := rand.Int(rand.Reader, big.NewInt(int64(maxUInt))) + if err == nil && n.Int64() >= int64(firstContextID) { + contextID = uint32(n.Int64()) + } + + // Open vhost-vsock device to check what context ID is available. + // This file descriptor holds/locks the context ID and it should be + // inherited by QEMU process. + vsockFd, err := os.OpenFile(VHostVSockDevicePath, syscall.O_RDWR, 0666) + if err != nil { + return nil, 0, err + } + + // Looking for the first available context ID. + for cid := contextID; cid <= maxUInt; cid++ { + if err := ioctlFunc(vsockFd.Fd(), ioctlVhostVsockSetGuestCid, cid); err == nil { + return vsockFd, cid, nil + } + } + + // Last chance to get a free context ID. + for cid := contextID - 1; cid >= firstContextID; cid-- { + if err := ioctlFunc(vsockFd.Fd(), ioctlVhostVsockSetGuestCid, cid); err == nil { + return vsockFd, cid, nil + } + } + + vsockFd.Close() + return nil, 0, fmt.Errorf("Could not get a unique context ID for the vsock") +} diff --git a/virtcontainers/utils/utils_linux_test.go b/virtcontainers/utils/utils_linux_test.go new file mode 100644 index 000000000..758f9609d --- /dev/null +++ b/virtcontainers/utils/utils_linux_test.go @@ -0,0 +1,35 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package utils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFindContextID(t *testing.T) { + assert := assert.New(t) + + ioctlFunc = func(fd uintptr, request int, arg1 uint32) error { + return errors.New("ioctl") + } + + orgVHostVSockDevicePath := VHostVSockDevicePath + orgMaxUInt := maxUInt + defer func() { + VHostVSockDevicePath = orgVHostVSockDevicePath + maxUInt = orgMaxUInt + }() + VHostVSockDevicePath = "/dev/null" + maxUInt = uint32(1000000) + + f, cid, err := FindContextID() + assert.Nil(f) + assert.Zero(cid) + assert.Error(err) +} From f389b94d8a14133c4eeb72abd7962ba08d5605d4 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Thu, 19 Jul 2018 14:53:53 -0500 Subject: [PATCH 04/10] kata-agent: Remove GRPCSock unused variable. We already save the URL used to connect to the agent in the `state.URL` this variable is the used to connect the shim to agnet independently the socket type (VSOCK or serial) Signed-off-by: Jose Carlos Venegas Munoz --- virtcontainers/kata_agent.go | 1 - virtcontainers/kata_agent_test.go | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 64e6aa4d5..131db8833 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -59,7 +59,6 @@ var ( // KataAgentConfig is a structure storing information needed // to reach the Kata Containers agent. type KataAgentConfig struct { - GRPCSocket string LongLiveConn bool } diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 6a0bb09dd..5ba39d000 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -671,13 +671,11 @@ func TestAgentPathAPI(t *testing.T) { assert.Nil(err) assert.Equal(k1, k2) - c.GRPCSocket = "unixsocket" err = k1.generateVMSocket(id, c) assert.Nil(err) _, ok := k1.vmSocket.(Socket) assert.True(ok) - c.GRPCSocket = "vsock:100:200" err = k2.generateVMSocket(id, c) assert.Nil(err) _, ok = k2.vmSocket.(kataVSOCK) @@ -692,7 +690,7 @@ func TestAgentConfigure(t *testing.T) { k := &kataAgent{} h := &mockHypervisor{} - c := KataAgentConfig{GRPCSocket: "vsock:100:200"} + c := KataAgentConfig{} id := "foobar" invalidAgent := HyperConfig{} @@ -702,7 +700,6 @@ func TestAgentConfigure(t *testing.T) { err = k.configure(h, id, dir, true, c) assert.Nil(err) - c.GRPCSocket = "foobarfoobar" err = k.configure(h, id, dir, true, c) assert.Nil(err) From 4680e58e08d0b667fa1f006216bb14040b30eae4 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 23 Jul 2018 18:02:38 -0500 Subject: [PATCH 05/10] cli: add configuration option to enable/disable vsocks Add `use_vsock` option to enable or disable the use of vsocks for communication between host and guest. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Julio Montes --- cli/config.go | 24 ++++++++++++ cli/config/configuration.toml.in | 6 +++ cli/config_test.go | 67 +++++++++++++++++++++++++++++++- cli/kata-env.go | 7 +++- virtcontainers/hypervisor.go | 3 ++ 5 files changed, 104 insertions(+), 3 deletions(-) diff --git a/cli/config.go b/cli/config.go index a800f19f7..c859f2cbd 100644 --- a/cli/config.go +++ b/cli/config.go @@ -15,6 +15,7 @@ import ( "github.com/BurntSushi/toml" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) @@ -91,6 +92,7 @@ type hypervisor struct { Debug bool `toml:"enable_debug"` DisableNestingChecks bool `toml:"disable_nesting_checks"` EnableIOThreads bool `toml:"enable_iothreads"` + UseVSock bool `toml:"use_vsock"` } type proxy struct { @@ -267,6 +269,10 @@ func (h hypervisor) msize9p() uint32 { return h.Msize9p } +func (h hypervisor) useVSock() bool { + return h.UseVSock +} + func (p proxy) path() string { if p.Path == "" { return defaultProxyPath @@ -333,6 +339,16 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{}, err } + useVSock := false + if h.useVSock() { + if utils.SupportsVsocks() { + kataLog.Info("vsock supported") + useVSock = true + } else { + kataLog.Warn("No vsock support, falling back to legacy serial port") + } + } + return vc.HypervisorConfig{ HypervisorPath: hypervisor, KernelPath: kernel, @@ -355,6 +371,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { BlockDeviceDriver: blockDriver, EnableIOThreads: h.EnableIOThreads, Msize9p: h.msize9p(), + UseVSock: useVSock, }, nil } @@ -544,6 +561,13 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat return "", config, err } + // use no proxy if HypervisorConfig.UseVSock is true + if config.HypervisorConfig.UseVSock { + kataLog.Info("VSOCK supported, configure to not use proxy") + config.ProxyType = vc.NoProxyType + config.ProxyConfig = vc.ProxyConfig{} + } + return resolved, config, nil } diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index b7517284e..f98ae536d 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -134,6 +134,12 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # used for 9p packet payload. #msize_9p = @DEFMSIZE9P@ +# If true and vsocks are supported, use vsocks to communicate directly +# with the agent and no proxy is started, otherwise use unix +# sockets and start a proxy to communicate with the agent. +# Default false +#use_vsock = true + [factory] # VM templating support. Once enabled, new VMs are created from template # using vm cloning. They will share the same initial kernel, initramfs and diff --git a/cli/config_test.go b/cli/config_test.go index 9b79d5bab..5a29377d2 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -20,6 +20,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/stretchr/testify/assert" ) @@ -552,6 +553,52 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Fatalf("Got %+v\n expecting %+v", config, expectedConfig) } + // minimal config with vsock enabled + runtimeMinimalConfig = ` + # Runtime configuration file + [hypervisor.qemu] + use_vsock = true + + [proxy.kata] + path = "` + proxyPath + `" + + [shim.kata] + path = "` + shimPath + `" + + [agent.kata] +` + orgVHostVSockDevicePath := utils.VHostVSockDevicePath + orgVSockDevicePath := utils.VSockDevicePath + defer func() { + utils.VHostVSockDevicePath = orgVHostVSockDevicePath + utils.VSockDevicePath = orgVSockDevicePath + }() + utils.VHostVSockDevicePath = "/dev/null" + utils.VSockDevicePath = "/dev/null" + + configPath = path.Join(dir, "runtime.toml") + err = createConfig(configPath, runtimeMinimalConfig) + if err != nil { + t.Fatal(err) + } + + _, config, err = loadConfiguration(configPath, false) + if err != nil { + t.Fatal(err) + } + + if config.ProxyType != vc.NoProxyType { + t.Fatalf("Proxy type must be NoProxy, got %+v", config.ProxyType) + } + + if !reflect.DeepEqual(config.ProxyConfig, vc.ProxyConfig{}) { + t.Fatalf("Got %+v\n expecting %+v", config.ProxyConfig, vc.ProxyConfig{}) + } + + if config.HypervisorConfig.UseVSock != true { + t.Fatalf("use_vsock must be true, got %v", config.HypervisorConfig.UseVSock) + } + if err := os.Remove(configPath); err != nil { t.Fatal(err) } @@ -570,6 +617,14 @@ func TestNewQemuHypervisorConfig(t *testing.T) { machineType := "machineType" disableBlock := true enableIOThreads := true + orgVSockDevicePath := utils.VSockDevicePath + orgVHostVSockDevicePath := utils.VHostVSockDevicePath + defer func() { + utils.VSockDevicePath = orgVSockDevicePath + utils.VHostVSockDevicePath = orgVHostVSockDevicePath + }() + utils.VSockDevicePath = "/dev/abc/xyz" + utils.VHostVSockDevicePath = "/dev/abc/xyz" hypervisor := hypervisor{ Path: hypervisorPath, @@ -578,6 +633,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) { MachineType: machineType, DisableBlockDeviceUse: disableBlock, EnableIOThreads: enableIOThreads, + UseVSock: true, } files := []string{hypervisorPath, kernelPath, imagePath} @@ -597,12 +653,21 @@ func TestNewQemuHypervisorConfig(t *testing.T) { } } - // all paths exist now + // falling back to legacy serial port config, err := newQemuHypervisorConfig(hypervisor) if err != nil { t.Fatal(err) } + utils.VSockDevicePath = "/dev/null" + utils.VHostVSockDevicePath = "/dev/null" + + // all paths exist now + config, err = newQemuHypervisorConfig(hypervisor) + if err != nil { + t.Fatal(err) + } + if config.HypervisorPath != hypervisor.Path { t.Errorf("Expected hypervisor path %v, got %v", hypervisor.Path, config.HypervisorPath) } diff --git a/cli/kata-env.go b/cli/kata-env.go index 150b51ca7..6f9a864cb 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -11,19 +11,20 @@ import ( "os" "strings" + runtim "runtime" + "github.com/BurntSushi/toml" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/urfave/cli" - runtim "runtime" ) // Semantic version for the output of the command. // // XXX: Increment for every change to the output format // (meaning any change to the EnvInfo type). -const formatVersion = "1.0.12" +const formatVersion = "1.0.13" // MetaInfo stores information on the format of the output itself type MetaInfo struct { @@ -80,6 +81,7 @@ type HypervisorInfo struct { BlockDeviceDriver string Msize9p uint32 Debug bool + UseVSock bool } // ProxyInfo stores proxy details @@ -276,6 +278,7 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo { Path: hypervisorPath, BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, Msize9p: config.HypervisorConfig.Msize9p, + UseVSock: config.HypervisorConfig.UseVSock, } } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 1e8a04e12..334658471 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -217,6 +217,9 @@ type HypervisorConfig struct { // Msize9p is used as the msize for 9p shares Msize9p uint32 + // UseVSock use a vsock for agent communication + UseVSock bool + // BootToBeTemplate used to indicate if the VM is created to be a template VM BootToBeTemplate bool From 3adc8626e8cd8acc851361d9fc26ae17ed0864f5 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 23 Jul 2018 11:43:52 -0500 Subject: [PATCH 06/10] virtcontainers: log type of proxy started In order to see what proxy was started or not, we should log its type and the URL Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Julio Montes --- virtcontainers/kata_agent.go | 1 + virtcontainers/kata_proxy.go | 1 + virtcontainers/no_proxy.go | 1 + 3 files changed, 3 insertions(+) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 131db8833..45ff34c64 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -1200,6 +1200,7 @@ func (k *kataAgent) connect() error { return nil } + k.Logger().WithField("url", k.state.URL).Info("New client") client, err := kataclient.NewAgentClient(k.state.URL, k.proxyBuiltIn) if err != nil { return err diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go index a1d3d3aaa..fe4b01e9c 100644 --- a/virtcontainers/kata_proxy.go +++ b/virtcontainers/kata_proxy.go @@ -24,6 +24,7 @@ func (p *kataProxy) consoleWatched() bool { // start is kataProxy start implementation for proxy interface. func (p *kataProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { + sandbox.Logger().Info("Starting regular Kata proxy rather than built-in") if sandbox.agent == nil { return -1, "", fmt.Errorf("No agent") } diff --git a/virtcontainers/no_proxy.go b/virtcontainers/no_proxy.go index a3e9d0b78..33664a56b 100644 --- a/virtcontainers/no_proxy.go +++ b/virtcontainers/no_proxy.go @@ -24,6 +24,7 @@ type noProxy struct { // start is noProxy start implementation for proxy interface. func (p *noProxy) start(sandbox *Sandbox, params proxyParams) (int, string, error) { + sandbox.Logger().Info("No proxy started because of no-proxy implementation") if params.agentURL == "" { return -1, "", fmt.Errorf("AgentURL cannot be empty") } From 1515bd07a14ed8d8d00f2192652effec648aefee Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 25 Jul 2018 07:58:23 -0500 Subject: [PATCH 07/10] virtcontainers: update KataAgentConfig to support vsocks add extra field in KataAgentConfig structure to specify if the kata agent have to use a vsock instead of serial port. Signed-off-by: Julio Montes --- cli/config.go | 5 +++-- virtcontainers/agent_test.go | 2 +- virtcontainers/kata_agent.go | 16 ++++++++-------- virtcontainers/kata_agent_test.go | 1 + 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cli/config.go b/cli/config.go index c859f2cbd..3a26210fd 100644 --- a/cli/config.go +++ b/cli/config.go @@ -428,8 +428,9 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run case kataAgentTableType: config.AgentType = kataAgentTableType - config.AgentConfig = vc.KataAgentConfig{} - + config.AgentConfig = vc.KataAgentConfig{ + UseVSock: config.HypervisorConfig.UseVSock, + } } } diff --git a/virtcontainers/agent_test.go b/virtcontainers/agent_test.go index a1fa120db..2c75a6f20 100644 --- a/virtcontainers/agent_test.go +++ b/virtcontainers/agent_test.go @@ -128,7 +128,7 @@ func TestNewAgentConfigFromHyperstartAgentType(t *testing.T) { } func TestNewAgentConfigFromKataAgentType(t *testing.T) { - agentConfig := KataAgentConfig{} + agentConfig := KataAgentConfig{UseVSock: true} sandboxConfig := SandboxConfig{ AgentType: KataContainersAgent, diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 45ff34c64..5519315f8 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -60,6 +60,7 @@ var ( // to reach the Kata Containers agent. type KataAgentConfig struct { LongLiveConn bool + UseVSock bool } type kataVSOCK struct { @@ -128,8 +129,13 @@ func (k *kataAgent) getSharePath(id string) string { } func (k *kataAgent) generateVMSocket(id string, c KataAgentConfig) error { - cid, port, err := parseVSOCKAddr(c.GRPCSocket) - if err != nil { + if c.UseVSock { + // We want to go through VSOCK. The VM VSOCK endpoint will be our gRPC. + k.Logger().Debug("agent: Using vsock VM socket endpoint") + // We dont know yet the context ID - set empty vsock configuration + k.vmSocket = kataVSOCK{} + } else { + k.Logger().Debug("agent: Using unix socket form VM socket endpoint") // We need to generate a host UNIX socket path for the emulated serial port. kataSock, err := utils.BuildSocketPath(k.getVMPath(id), defaultKataSocketName) if err != nil { @@ -142,12 +148,6 @@ func (k *kataAgent) generateVMSocket(id string, c KataAgentConfig) error { HostPath: kataSock, Name: defaultKataChannel, } - } else { - // We want to go through VSOCK. The VM VSOCK endpoint will be our gRPC. - k.vmSocket = kataVSOCK{ - contextID: cid, - port: port, - } } return nil diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 5ba39d000..b221df02c 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -676,6 +676,7 @@ func TestAgentPathAPI(t *testing.T) { _, ok := k1.vmSocket.(Socket) assert.True(ok) + c.UseVSock = true err = k2.generateVMSocket(id, c) assert.Nil(err) _, ok = k2.vmSocket.(kataVSOCK) From 052769196da1ac4ed5f102b0832b0548b4e1ed0f Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 25 Jul 2018 09:04:29 -0500 Subject: [PATCH 08/10] virtcontainers: implement function to cold plug vsocks `appendVSockPCI` function can be used to cold plug vocks, vhost file descriptor holds the context ID and it's inherit by QEMU process, ID must be unique and disable-modern prevents qemu from relying on fast MMIO. Signed-off-by: Julio Montes --- virtcontainers/qemu.go | 15 +++++++++++++++ virtcontainers/qemu_arch_base.go | 17 +++++++++++++++++ virtcontainers/qemu_test.go | 22 ++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 1d84fa88d..143a5922c 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -62,6 +62,10 @@ type qemu struct { state QemuState arch qemuArch + + // fds is a list of file descriptors inherited by QEMU process + // they'll be closed once QEMU process is running + fds []*os.File } const ( @@ -497,6 +501,14 @@ func (q *qemu) startSandbox() error { q.Logger().WithField("default-kernel-parameters", formatted).Debug() } + defer func() { + for _, fd := range q.fds { + if err := fd.Close(); err != nil { + q.Logger().WithError(err).Error("After launching Qemu") + } + } + }() + strErr, err := govmmQemu.LaunchQemu(q.qemuConfig, newQMPLogger()) if err != nil { return fmt.Errorf("%s", strErr) @@ -956,6 +968,9 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { q.qemuConfig.Devices = q.arch.append9PVolume(q.qemuConfig.Devices, v) case Socket: q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v) + case kataVSOCK: + q.fds = append(q.fds, v.vhostFd) + q.qemuConfig.Devices = q.arch.appendVSockPCI(q.qemuConfig.Devices, v) case Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) case deviceDrivers.Drive: diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index e0188c4d3..730023e7f 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -68,6 +68,9 @@ type qemuArch interface { // appendSocket appends a socket to devices appendSocket(devices []govmmQemu.Device, socket Socket) []govmmQemu.Device + // appendVSockPCI appends a vsock PCI to devices + appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device + // appendNetwork appends a endpoint device to devices appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device @@ -389,6 +392,20 @@ func (q *qemuArchBase) appendSocket(devices []govmmQemu.Device, socket Socket) [ return devices } +func (q *qemuArchBase) appendVSockPCI(devices []govmmQemu.Device, vsock kataVSOCK) []govmmQemu.Device { + devices = append(devices, + govmmQemu.VSOCKDevice{ + ID: fmt.Sprintf("vsock-%d", vsock.contextID), + ContextID: vsock.contextID, + VHostFD: vsock.vhostFd, + DisableModern: q.nestedRun, + }, + ) + + return devices + +} + func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType { switch model { case NetXConnectBridgedModel: diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 55b443ea7..2d6d82155 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -246,6 +246,28 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) { testQemuAddDevice(t, socket, serialPortDev, expectedOut) } +func TestQemuAddDeviceKataVSOCK(t *testing.T) { + contextID := uint32(3) + port := uint32(1024) + vHostFD := os.NewFile(1, "vsock") + + expectedOut := []govmmQemu.Device{ + govmmQemu.VSOCKDevice{ + ID: fmt.Sprintf("vsock-%d", contextID), + ContextID: contextID, + VHostFD: vHostFD, + }, + } + + vsock := kataVSOCK{ + contextID: contextID, + port: port, + vhostFd: vHostFD, + } + + testQemuAddDevice(t, vsock, vSockPCIDev, expectedOut) +} + func TestQemuGetSandboxConsole(t *testing.T) { q := &qemu{} sandboxID := "testSandboxID" From 3c15bc50d0fd8a92f75256937d4dab4a05cc995c Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 25 Jul 2018 09:59:49 -0500 Subject: [PATCH 09/10] virtcontainers: remove parseVSOCKAddr function parseVSOCKAddr function is no more needed since now agent config contains a field to identify if vsocks should be used or not. Signed-off-by: Julio Montes --- virtcontainers/kata_agent.go | 21 --------------------- virtcontainers/kata_agent_test.go | 30 ------------------------------ 2 files changed, 51 deletions(-) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 5519315f8..069354bae 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -99,27 +99,6 @@ func (k *kataAgent) Logger() *logrus.Entry { return virtLog.WithField("subsystem", "kata_agent") } -func parseVSOCKAddr(sock string) (uint32, uint32, error) { - sp := strings.Split(sock, ":") - if len(sp) != 3 { - return 0, 0, fmt.Errorf("Invalid vsock address: %s", sock) - } - if sp[0] != vsockSocketScheme { - return 0, 0, fmt.Errorf("Invalid vsock URL scheme: %s", sp[0]) - } - - cid, err := strconv.ParseUint(sp[1], 10, 32) - if err != nil { - return 0, 0, fmt.Errorf("Invalid vsock cid: %s", sp[1]) - } - port, err := strconv.ParseUint(sp[2], 10, 32) - if err != nil { - return 0, 0, fmt.Errorf("Invalid vsock port: %s", sp[2]) - } - - return uint32(cid), uint32(port), nil -} - func (k *kataAgent) getVMPath(id string) string { return filepath.Join(RunVMStoragePath, id) } diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index b221df02c..1b8b0d686 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -708,36 +708,6 @@ func TestAgentConfigure(t *testing.T) { assert.Nil(err) } -func TestParseVSOCKAddr(t *testing.T) { - assert := assert.New(t) - - sock := "randomfoobar" - _, _, err := parseVSOCKAddr(sock) - assert.Error(err) - - sock = "vsock://1:2" - _, _, err = parseVSOCKAddr(sock) - assert.Error(err) - - sock = "unix:1:2" - _, _, err = parseVSOCKAddr(sock) - assert.Error(err) - - sock = "vsock:foo:2" - _, _, err = parseVSOCKAddr(sock) - assert.Error(err) - - sock = "vsock:1:bar" - _, _, err = parseVSOCKAddr(sock) - assert.Error(err) - - sock = "vsock:1:2" - cid, port, err := parseVSOCKAddr(sock) - assert.Nil(err) - assert.Equal(cid, uint32(1)) - assert.Equal(port, uint32(2)) -} - func TestCmdToKataProcess(t *testing.T) { assert := assert.New(t) From 33643797adeb88676d410642a8036561f7db6436 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 25 Jul 2018 09:45:24 -0500 Subject: [PATCH 10/10] virtcontainers: Use vsock if host support it When the hypervisor option `use_vsock` is true the runtime will check for vsock support. If vsock is supported, not proxy will be used and the shims will connect to the VM using VSOCKS. This flag is true by default, so will use VSOCK when possible and no proxy will be started. fixes #383 Signed-off-by: Jose Carlos Venegas Munoz jose.carlos.venegas.munoz@intel.com Signed-off-by: Julio Montes --- cli/kata-env.go | 4 ++++ virtcontainers/hypervisor.go | 3 +++ virtcontainers/kata_agent.go | 29 +++++++++++++++++++++-------- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/cli/kata-env.go b/cli/kata-env.go index 6f9a864cb..0cfeb70e3 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -214,6 +214,10 @@ func getHostInfo() (HostInfo, error) { } func getProxyInfo(config oci.RuntimeConfig) (ProxyInfo, error) { + if config.ProxyType == vc.NoProxyType { + return ProxyInfo{Type: string(config.ProxyType)}, nil + } + version, err := getCommandVersion(defaultProxyPath) if err != nil { version = unknown diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 334658471..991b7dfd3 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -68,6 +68,9 @@ const ( // SerialPortDev is the serial port device type. serialPortDev + // vSockPCIDev is the vhost vsock PCI device type. + vSockPCIDev + // VFIODevice is VFIO device type vfioDev diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 069354bae..65c56a128 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -47,13 +47,16 @@ var ( kataGuestSandboxDir = "/run/kata-containers/sandbox/" type9pFs = "9p" vsockSocketScheme = "vsock" - kata9pDevType = "9p" - kataBlkDevType = "blk" - kataSCSIDevType = "scsi" - sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L", "nodev"} - shmDir = "shm" - kataEphemeralDevType = "ephemeral" - ephemeralPath = filepath.Join(kataGuestSandboxDir, kataEphemeralDevType) + // port numbers below 1024 are called privileged ports. Only a process with + // CAP_NET_BIND_SERVICE capability may bind to these port numbers. + vSockPort = 1024 + kata9pDevType = "9p" + kataBlkDevType = "blk" + kataSCSIDevType = "scsi" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L", "nodev"} + shmDir = "shm" + kataEphemeralDevType = "ephemeral" + ephemeralPath = filepath.Join(kataGuestSandboxDir, kataEphemeralDevType) ) // KataAgentConfig is a structure storing information needed @@ -66,6 +69,7 @@ type KataAgentConfig struct { type kataVSOCK struct { contextID uint32 port uint32 + vhostFd *os.File } func (s *kataVSOCK) String() string { @@ -203,7 +207,16 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, return err } case kataVSOCK: - // TODO Add an hypervisor vsock + var err error + s.vhostFd, s.contextID, err = utils.FindContextID() + if err != nil { + return err + } + s.port = uint32(vSockPort) + if err := h.addDevice(s, vSockPCIDev); err != nil { + return err + } + k.vmSocket = s default: return fmt.Errorf("Invalid config type") }