From 62f64bb0091f5606c54267ffcfe0d29dd731a2cb Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 3 Oct 2019 10:53:01 -0700 Subject: [PATCH 01/23] version: Update containerd version to 1.3.0 These include features like privileged containers without host devices and support for per runtime annotations. Depends-on: github.com/kata-containers/tests#2029 Fixes #2099 Signed-off-by: Archana Shinde --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 8e675d16a..b57248a09 100644 --- a/versions.yaml +++ b/versions.yaml @@ -209,7 +209,7 @@ externals: Containerd Plugin for Kubernetes Container Runtime Interface. url: "github.com/containerd/cri" tarball_url: "https://storage.googleapis.com/cri-containerd-release" - version: "1.2.7" + version: "1.3.0" docker: description: "Moby project container manager" From 0e70b38d06f5bc3b43b2dbbabad1fb2a274dafa3 Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Tue, 5 Nov 2019 13:57:02 -0600 Subject: [PATCH 02/23] versions: Update kubernetes and cri-o to 1.16 Update k8s supported version from 1.15.3 to 1.16.2 and cri-o from 1.15.0 to 1.16.0 Fixes: #2166. Signed-off-by: Salvador Fuentes --- versions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yaml b/versions.yaml index 2e2015394..ed42ca990 100644 --- a/versions.yaml +++ b/versions.yaml @@ -199,7 +199,7 @@ externals: description: | OCI-based Kubernetes Container Runtime Interface implementation url: "https://github.com/cri-o/cri-o" - version: "v1.15.0" + version: "v1.16.0" meta: openshift: "6273bea4c9ed788aeb3d051ebf2d030060c05b6c" crictl: 1.0.0-beta.2 @@ -240,7 +240,7 @@ externals: uscan-url: >- https://github.com/kubernetes/kubernetes/tags .*/v?([\d\.]+)\.tar\.gz - version: "1.15.3-00" + version: "1.16.2-00" openshift: description: | From 5b3128255809e0e0a1f138c20377d77361fa6ef6 Mon Sep 17 00:00:00 2001 From: Yves Chan Date: Thu, 17 Oct 2019 23:53:08 +0800 Subject: [PATCH 03/23] vc/qemu: add mutex to qmp monitor channel in qmpSetup() Solve possible race condition in qmpSetup() and qmpShutdown() Fixes: #2139 Signed-off-by: Yves Chan --- virtcontainers/qemu.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 948feba80..95b0b0fa0 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "syscall" "time" "unsafe" @@ -48,6 +49,7 @@ const romFile = "" const defaultDisableModern = false type qmpChannel struct { + sync.Mutex ctx context.Context path string qmp *govmmQemu.QMP @@ -921,6 +923,9 @@ func (q *qemu) togglePauseSandbox(pause bool) error { } func (q *qemu) qmpSetup() error { + q.qmpMonitorCh.Lock() + defer q.qmpMonitorCh.Unlock() + if q.qmpMonitorCh.qmp != nil { return nil } @@ -949,6 +954,9 @@ func (q *qemu) qmpSetup() error { } func (q *qemu) qmpShutdown() { + q.qmpMonitorCh.Lock() + defer q.qmpMonitorCh.Unlock() + if q.qmpMonitorCh.qmp != nil { q.qmpMonitorCh.qmp.Shutdown() // wait on disconnected channel to be sure that the qmp channel has From 69ab09273dc3c1d7cf32c116e45ab93db7ecaa8e Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Wed, 6 Nov 2019 10:16:38 -0600 Subject: [PATCH 04/23] versions: Add cri-tools and conmon to the versions.yaml cri-tools version was managed in the tests repository, but as we define here cri-o, containerd and kubernetes versions, it make sense to have the cri-tools version defined in this repo. conmon has now to be installed/built separately. So add it to the list. Depends-on: github.com/kata-containers/tests#2057 Signed-off-by: Salvador Fuentes --- versions.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/versions.yaml b/versions.yaml index ed42ca990..28f6d7946 100644 --- a/versions.yaml +++ b/versions.yaml @@ -195,6 +195,11 @@ externals: url: "https://github.com/containernetworking/plugins" commit: "485be65581341430f9106a194a98f0f2412245fb" + conmon: + description: "An OCI container runtime monitor" + url: "https://github.com/containers/conmon" + version: "v2.0.1" + crio: description: | OCI-based Kubernetes Container Runtime Interface implementation @@ -211,6 +216,11 @@ externals: tarball_url: "https://storage.googleapis.com/cri-containerd-release" version: "1.2.7" + critools: + description: "CLI tool for Container Runtime Interface (CRI)" + url: "https://github.com/kubernetes-sigs/cri-tools" + version: "1.16.1" + docker: description: "Moby project container manager" notes: "Docker Swarm requires an older version of Docker." From 3f1a39c442b7786b028b05c81c381acee4edb1a9 Mon Sep 17 00:00:00 2001 From: Li Yuxuan Date: Fri, 8 Nov 2019 20:33:42 +0800 Subject: [PATCH 05/23] rootless: Fix rangeUID parsing `rangeUID` should be parsed as ids[2] rather than ids[1] Fixes: #2173 Signed-off-by: Li Yuxuan --- pkg/rootless/rootless.go | 2 +- pkg/rootless/rootless_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/rootless/rootless.go b/pkg/rootless/rootless.go index 9b0d22d52..495fdf19a 100644 --- a/pkg/rootless/rootless.go +++ b/pkg/rootless/rootless.go @@ -90,7 +90,7 @@ func setRootless() error { if err != nil { return parseError } - rangeUID, err := strconv.ParseUint(ids[1], 10, 0) + rangeUID, err := strconv.ParseUint(ids[2], 10, 0) if err != nil || rangeUID == 0 { return parseError } diff --git a/pkg/rootless/rootless_test.go b/pkg/rootless/rootless_test.go index addfd0e28..1a5289477 100644 --- a/pkg/rootless/rootless_test.go +++ b/pkg/rootless/rootless_test.go @@ -119,6 +119,7 @@ func TestIsRootless(t *testing.T) { uidMap: []uidMapping{ {0, 0, 0}, {1, 0, 0}, + {0, 1, 0}, {1, 1000, 0}, {1000, 1000, 0}, }, From 0bd41b9dbe691ed70fa8769089a4a8795a5d54c9 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 7 Nov 2019 16:45:47 -0800 Subject: [PATCH 06/23] FIPS: Add support for starting VM in FIPS mode. FIPS are a set of security standards for encryption algorithms in user and kernel space among others. Have Kata support this by starting the VM for a container in FIPS mode on detecting that the host is running in FIPS mode. Depends-on: github.com/kata-containers/packaging#788 Fixes #2170 Signed-off-by: Archana Shinde --- pkg/katautils/create.go | 36 +++++++++++++++++++++++++++++++ pkg/katautils/create_test.go | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go index facf27324..424d42ff3 100644 --- a/pkg/katautils/create.go +++ b/pkg/katautils/create.go @@ -9,6 +9,9 @@ package katautils import ( "context" "fmt" + "io/ioutil" + "strconv" + "strings" vc "github.com/kata-containers/runtime/virtcontainers" vf "github.com/kata-containers/runtime/virtcontainers/factory" @@ -115,6 +118,10 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo sandboxConfig.Stateful = true } + if err := checkForFIPS(&sandboxConfig); err != nil { + return nil, vc.Process{}, err + } + if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 { if rootFs.Source != "" { realPath, err := ResolvePath(rootFs.Source) @@ -175,6 +182,35 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo return sandbox, containers[0].Process(), nil } +var procFIPS = "/proc/sys/crypto/fips_enabled" + +func checkForFIPS(sandboxConfig *vc.SandboxConfig) error { + content, err := ioutil.ReadFile(procFIPS) + if err != nil { + // In case file cannot be found or read, simply return + return nil + } + + enabled, err := strconv.Atoi(strings.Trim(string(content), "\n\t ")) + if err != nil { + // Unexpected format, ignore and simply return early + return nil + } + + if enabled == 1 { + param := vc.Param{ + Key: "fips", + Value: "1", + } + + if err := sandboxConfig.HypervisorConfig.AddKernelParam(param); err != nil { + return fmt.Errorf("Error enabling fips mode : %v", err) + } + } + + return nil +} + // CreateContainer create a container func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) { var c vc.VCContainer diff --git a/pkg/katautils/create_test.go b/pkg/katautils/create_test.go index 7c95003c8..3e5eb370d 100644 --- a/pkg/katautils/create_test.go +++ b/pkg/katautils/create_test.go @@ -334,6 +334,48 @@ func TestCreateSandboxFail(t *testing.T) { assert.True(vcmock.IsMockError(err)) } +func TestCheckForFips(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(path) + + val := procFIPS + procFIPS = filepath.Join(path, "fips-enabled") + defer func() { + procFIPS = val + }() + + err = ioutil.WriteFile(procFIPS, []byte("1"), 0644) + assert.NoError(err) + + hconfig := vc.HypervisorConfig{ + KernelParams: []vc.Param{ + {Key: "init", Value: "/sys/init"}, + }, + } + config := vc.SandboxConfig{ + HypervisorConfig: hconfig, + } + assert.NoError(checkForFIPS(&config)) + + params := config.HypervisorConfig.KernelParams + assert.Equal(len(params), 2) + assert.Equal(params[1].Key, "fips") + assert.Equal(params[1].Value, "1") + + config.HypervisorConfig = hconfig + err = ioutil.WriteFile(procFIPS, []byte("unexpected contents"), 0644) + assert.NoError(err) + assert.NoError(checkForFIPS(&config)) + assert.Equal(config.HypervisorConfig, hconfig) + + assert.NoError(os.Remove(procFIPS)) + assert.NoError(checkForFIPS(&config)) + assert.Equal(config.HypervisorConfig, hconfig) +} + func TestCreateContainerContainerConfigFail(t *testing.T) { assert := assert.New(t) From f6ffb791e713c0c7d7617caeb6e72cf676a861c1 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 8 Nov 2019 18:30:11 -0800 Subject: [PATCH 07/23] rootless: Fix cgroup creation logic for rootless We do not want to create cgroups in case of rootless. Fix the logic to implement this. Fixes #2177 Signed-off-by: Archana Shinde --- virtcontainers/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtcontainers/container.go b/virtcontainers/container.go index aff4e8569..701d5c678 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -916,7 +916,7 @@ func (c *Container) create() (err error) { } c.process = *process - if !c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless() { + if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly { if err = c.cgroupsCreate(); err != nil { return } From dffc988d9210ffc74b2dd8666dc53e9e88e2799c Mon Sep 17 00:00:00 2001 From: Manohar Castelino Date: Tue, 5 Feb 2019 11:19:08 -0800 Subject: [PATCH 08/23] virtcontainers: Eliminate legacy networking models Prior to the addition of tcMirroring support kata-runtime had compatibility issues with some CNI plugins some of which were addressed by the bridged model. With the addition of tc mode there are no gaps in networking that can be filled by the bridged mode or enlightened mode (which was never implemented). Eliminate both of these options to simplify the setup. Fixes: #1213 Signed-off-by: Manohar Castelino --- cli/config/configuration-fc.toml.in | 8 +- cli/config/configuration-qemu.toml.in | 8 +- virtcontainers/documentation/api/1.0/api.md | 12 -- virtcontainers/network.go | 198 -------------------- virtcontainers/network_test.go | 38 +--- virtcontainers/qemu_arch_base.go | 7 - 6 files changed, 7 insertions(+), 264 deletions(-) diff --git a/cli/config/configuration-fc.toml.in b/cli/config/configuration-fc.toml.in index 520642d91..446db958b 100644 --- a/cli/config/configuration-fc.toml.in +++ b/cli/config/configuration-fc.toml.in @@ -289,12 +289,6 @@ path = "@NETMONPATH@" # the container network interface # Options: # -# - bridged (Deprecated) -# Uses a linux bridge to interconnect the container interface to -# the VM. Works for most cases except macvlan and ipvlan. -# ***NOTE: This feature has been deprecated with plans to remove this -# feature in the future. Please use other network models listed below. -# # - macvtap # Used when the Container network interface can be bridged using # macvtap. @@ -323,7 +317,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. # `disable_new_netns` conflicts with `enable_netmon` -# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. # If you are using docker, `disable_new_netns` only works with `docker run --net=none` diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 3fbbf3791..404eb4754 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -379,12 +379,6 @@ path = "@NETMONPATH@" # the container network interface # Options: # -# - bridged (Deprecated) -# Uses a linux bridge to interconnect the container interface to -# the VM. Works for most cases except macvlan and ipvlan. -# ***NOTE: This feature has been deprecated with plans to remove this -# feature in the future. Please use other network models listed below. -# # - macvtap # Used when the Container network interface can be bridged using # macvtap. @@ -413,7 +407,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. # `disable_new_netns` conflicts with `enable_netmon` -# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. # If you are using docker, `disable_new_netns` only works with `docker run --net=none` diff --git a/virtcontainers/documentation/api/1.0/api.md b/virtcontainers/documentation/api/1.0/api.md index 4753f5c9d..f5d3a7347 100644 --- a/virtcontainers/documentation/api/1.0/api.md +++ b/virtcontainers/documentation/api/1.0/api.md @@ -278,22 +278,10 @@ const ( // NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel NetXConnectDefaultModel NetInterworkingModel = iota - // NetXConnectBridgedModel uses a linux bridge to interconnect - // the container interface to the VM. This is the - // safe default that works for most cases except - // macvlan and ipvlan - NetXConnectBridgedModel - // NetXConnectMacVtapModel can be used when the Container network // interface can be bridged using macvtap NetXConnectMacVtapModel - // NetXConnectEnlightenedModel can be used when the Network plugins - // are enlightened to create VM native interfaces - // when requested by the runtime - // This will be used for vethtap, macvtap, ipvtap - NetXConnectEnlightenedModel - // NetXConnectInvalidModel is the last item to check valid values by IsValid() NetXConnectInvalidModel ) diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 83766a859..55bb77331 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -38,22 +38,10 @@ const ( // NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel NetXConnectDefaultModel NetInterworkingModel = iota - // NetXConnectBridgedModel uses a linux bridge to interconnect - // the container interface to the VM. This is the - // safe default that works for most cases except - // macvlan and ipvlan - NetXConnectBridgedModel - // NetXConnectMacVtapModel can be used when the Container network // interface can be bridged using macvtap NetXConnectMacVtapModel - // NetXConnectEnlightenedModel can be used when the Network plugins - // are enlightened to create VM native interfaces - // when requested by the runtime - // This will be used for vethtap, macvtap, ipvtap - NetXConnectEnlightenedModel - // NetXConnectTCFilterModel redirects traffic from the network interface // provided by the network plugin to a tap interface. // This works for ipvlan and macvlan as well. @@ -74,12 +62,8 @@ func (n NetInterworkingModel) IsValid() bool { const ( defaultNetModelStr = "default" - bridgedNetModelStr = "bridged" - macvtapNetModelStr = "macvtap" - enlightenedNetModelStr = "enlightened" - tcFilterNetModelStr = "tcfilter" noneNetModelStr = "none" @@ -91,15 +75,9 @@ func (n *NetInterworkingModel) SetModel(modelName string) error { case defaultNetModelStr: *n = DefaultNetInterworkingModel return nil - case bridgedNetModelStr: - *n = NetXConnectBridgedModel - return nil case macvtapNetModelStr: *n = NetXConnectMacVtapModel return nil - case enlightenedNetModelStr: - *n = NetXConnectEnlightenedModel - return nil case tcFilterNetModelStr: *n = NetXConnectTCFilterModel return nil @@ -325,11 +303,6 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin var fds []*os.File switch expectedLink.Type() { - case (&netlink.Bridge{}).Type(): - newLink = &netlink.Bridge{ - LinkAttrs: netlink.LinkAttrs{Name: name}, - MulticastSnooping: expectedLink.(*netlink.Bridge).MulticastSnooping, - } case (&netlink.Tuntap{}).Type(): flags := netlink.TUNTAP_VNET_HDR if queues > 0 { @@ -400,10 +373,6 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. } switch expectedLink.Type() { - case (&netlink.Bridge{}).Type(): - if l, ok := link.(*netlink.Bridge); ok { - return l, nil - } case (&netlink.Tuntap{}).Type(): if l, ok := link.(*netlink.Tuntap); ok { return l, nil @@ -448,14 +417,10 @@ func xConnectVMNetwork(endpoint Endpoint, h hypervisor) error { } switch netPair.NetInterworkingModel { - case NetXConnectBridgedModel: - return bridgeNetworkPair(endpoint, queues, disableVhostNet) case NetXConnectMacVtapModel: return tapNetworkPair(endpoint, queues, disableVhostNet) case NetXConnectTCFilterModel: return setupTCFiltering(endpoint, queues, disableVhostNet) - case NetXConnectEnlightenedModel: - return fmt.Errorf("Unsupported networking model") default: return fmt.Errorf("Invalid internetworking model") } @@ -470,14 +435,10 @@ func xDisconnectVMNetwork(endpoint Endpoint) error { } switch netPair.NetInterworkingModel { - case NetXConnectBridgedModel: - return unBridgeNetworkPair(endpoint) case NetXConnectMacVtapModel: return untapNetworkPair(endpoint) case NetXConnectTCFilterModel: return removeTCFiltering(endpoint) - case NetXConnectEnlightenedModel: - return fmt.Errorf("Unsupported networking model") default: return fmt.Errorf("Invalid internetworking model") } @@ -656,100 +617,6 @@ func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { return nil } -func bridgeNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) - if err != nil { - return fmt.Errorf("Could not create TAP interface: %s", err) - } - netPair.VMFds = fds - - if !disableVhostNet { - vhostFds, err := createVhostFds(queues) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) - } - netPair.VhostFds = vhostFds - } - - var attrs *netlink.LinkAttrs - var link netlink.Link - - link, err = getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - attrs = link.Attrs() - - // Save the veth MAC address to the TAP so that it can later be used - // to build the hypervisor command line. This MAC address has to be - // the one inside the VM in order to avoid any firewall issues. The - // bridge created by the network plugin on the host actually expects - // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - - if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) - } - - hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) - if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - mcastSnoop := false - bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, queues) - if err != nil { - return fmt.Errorf("Could not create bridge: %s", err) - } - - if err := netHandle.LinkSetMaster(tapLink, bridgeLink.(*netlink.Bridge)); err != nil { - return fmt.Errorf("Could not attach TAP %s to the bridge %s: %s", - netPair.TAPIface.Name, netPair.Name, err) - } - - if err := netHandle.LinkSetUp(tapLink); err != nil { - return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { - return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", - netPair.VirtIface.Name, netPair.Name, err) - } - - // Clear the IP addresses from the veth interface to prevent ARP conflict - netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) - } - - if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { - return fmt.Errorf("Unable to clear veth IP addresses: %s", err) - } - - if err := netHandle.LinkSetUp(link); err != nil { - return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetUp(bridgeLink); err != nil { - return fmt.Errorf("Could not enable bridge %s: %s", netPair.Name, err) - } - - return nil -} - func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { @@ -963,71 +830,6 @@ func untapNetworkPair(endpoint Endpoint) error { return err } -func unBridgeNetworkPair(endpoint Endpoint) error { - netHandle, err := netlink.NewHandle() - if err != nil { - return err - } - defer netHandle.Delete() - - netPair := endpoint.NetworkPair() - - tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) - if err != nil { - return fmt.Errorf("Could not get TAP interface: %s", err) - } - - bridgeLink, err := getLinkByName(netHandle, netPair.Name, &netlink.Bridge{}) - if err != nil { - return fmt.Errorf("Could not get bridge interface: %s", err) - } - - if err := netHandle.LinkSetDown(bridgeLink); err != nil { - return fmt.Errorf("Could not disable bridge %s: %s", netPair.Name, err) - } - - if err := netHandle.LinkSetDown(tapLink); err != nil { - return fmt.Errorf("Could not disable TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkSetNoMaster(tapLink); err != nil { - return fmt.Errorf("Could not detach TAP %s: %s", netPair.TAPIface.Name, err) - } - - if err := netHandle.LinkDel(bridgeLink); err != nil { - return fmt.Errorf("Could not remove bridge %s: %s", netPair.Name, err) - } - - if err := netHandle.LinkDel(tapLink); err != nil { - return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) - } - - link, err := getLinkForEndpoint(endpoint, netHandle) - if err != nil { - return err - } - - hardAddr, err := net.ParseMAC(netPair.TAPIface.HardAddr) - if err != nil { - return err - } - if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { - return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", - netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetDown(link); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) - } - - if err := netHandle.LinkSetNoMaster(link); err != nil { - return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) - } - - // Restore the IPs that were cleared - return setIPs(link, netPair.VirtIface.Addrs) -} - func removeTCFiltering(endpoint Endpoint) error { netHandle, err := netlink.NewHandle() if err != nil { diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index 83cec2007..c484b0866 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -110,10 +110,8 @@ func TestNetInterworkingModelIsValid(t *testing.T) { }{ {"Invalid Model", NetXConnectInvalidModel, false}, {"Default Model", NetXConnectDefaultModel, true}, - {"Bridged Model", NetXConnectBridgedModel, true}, {"TC Filter Model", NetXConnectTCFilterModel, true}, {"Macvtap Model", NetXConnectMacVtapModel, true}, - {"Enlightened Model", NetXConnectEnlightenedModel, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -133,9 +131,7 @@ func TestNetInterworkingModelSetModel(t *testing.T) { }{ {"Invalid Model", "Invalid", true}, {"default Model", defaultNetModelStr, false}, - {"bridged Model", bridgedNetModelStr, false}, {"macvtap Model", macvtapNetModelStr, false}, - {"enlightened Model", enlightenedNetModelStr, false}, {"tcfilter Model", tcFilterNetModelStr, false}, {"none Model", noneNetModelStr, false}, } @@ -167,30 +163,6 @@ func TestGenerateRandomPrivateMacAdd(t *testing.T) { assert.NotEqual(addr1, addr2) } -func TestCreateGetBridgeLink(t *testing.T) { - if tc.NotValid(ktu.NeedRoot()) { - t.Skip(testDisabledAsNonRoot) - } - - assert := assert.New(t) - - netHandle, err := netlink.NewHandle() - defer netHandle.Delete() - - assert.NoError(err) - - brName := "testbr0" - brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) - assert.NoError(err) - assert.NotNil(brLink) - - brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) - assert.NoError(err) - - err = netHandle.LinkDel(brLink) - assert.NoError(err) -} - func TestCreateGetTunTapLink(t *testing.T) { if tc.NotValid(ktu.NeedRoot()) { t.Skip(testDisabledAsNonRoot) @@ -228,11 +200,11 @@ func TestCreateMacVtap(t *testing.T) { assert.NoError(err) - brName := "testbr0" - brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + tapName := "testtap0" + tapLink, _, err := createLink(netHandle, tapName, &netlink.Tuntap{}, 1) assert.NoError(err) - attrs := brLink.Attrs() + attrs := tapLink.Attrs() mcLink := &netlink.Macvtap{ Macvlan: netlink.Macvlan{ @@ -253,10 +225,10 @@ func TestCreateMacVtap(t *testing.T) { err = netHandle.LinkDel(macvtapLink) assert.NoError(err) - brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + tapLink, err = getLinkByName(netHandle, tapName, &netlink.Tuntap{}) assert.NoError(err) - err = netHandle.LinkDel(brLink) + err = netHandle.LinkDel(tapLink) assert.NoError(err) } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 9d6d68cf3..41e7018c6 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -467,15 +467,8 @@ func (q *qemuArchBase) appendVSock(devices []govmmQemu.Device, vsock types.VSock func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType { switch model { - case NetXConnectBridgedModel: - return govmmQemu.MACVTAP //TODO: We should rename MACVTAP to .NET_FD case NetXConnectMacVtapModel: return govmmQemu.MACVTAP - //case ModelEnlightened: - // Here the Network plugin will create a VM native interface - // which could be MacVtap, IpVtap, SRIOV, veth-tap, vhost-user - // In these cases we will determine the interface type here - // and pass in the native interface through default: //TAP should work for most other cases return govmmQemu.TAP From 27433d917882b716b900a31b73482d434074afc7 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 2 Aug 2019 14:55:40 -0700 Subject: [PATCH 09/23] config: Get rid of bridged model docs in the configuration Since we have dropped support for bridged model, remove it from the configuration as well. Signed-off-by: Archana Shinde --- cli/config/configuration-nemu.toml.in | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cli/config/configuration-nemu.toml.in b/cli/config/configuration-nemu.toml.in index 09cc5a8f1..c92bb12f5 100644 --- a/cli/config/configuration-nemu.toml.in +++ b/cli/config/configuration-nemu.toml.in @@ -371,12 +371,6 @@ path = "@NETMONPATH@" # the container network interface # Options: # -# - bridged (Deprecated) -# Uses a linux bridge to interconnect the container interface to -# the VM. Works for most cases except macvlan and ipvlan. -# ***NOTE: This feature has been deprecated with plans to remove this -# feature in the future. Please use other network models listed below. -# # - macvtap # Used when the Container network interface can be bridged using # macvtap. From 744ccd4ed206762aaea142f7a10333a5954c4d21 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 2 Aug 2019 14:57:16 -0700 Subject: [PATCH 10/23] network: Set the default config to tcfilter If the configuration for networking is missing, tcfilter will be chosen. Signed-off-by: Archana Shinde --- pkg/katautils/config-settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index 02217f9d9..568eff7a7 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -28,7 +28,7 @@ const defaultMemSize uint32 = 2048 // MiB const defaultMemSlots uint32 = 10 const defaultMemOffset uint32 = 0 // MiB const defaultBridgesCount uint32 = 1 -const defaultInterNetworkingModel = "macvtap" +const defaultInterNetworkingModel = "tcfilter" const defaultDisableBlockDeviceUse bool = false const defaultBlockDeviceDriver = "virtio-scsi" const defaultBlockDeviceCacheSet bool = false From 0def9b01ded248457eda508d55d881e637bf69ea Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 11 Nov 2019 22:18:05 +0000 Subject: [PATCH 11/23] virtcontainers/firecracker: update API update firecracker API to 0.19.0 Signed-off-by: Julio Montes --- virtcontainers/pkg/firecracker/README | 4 +- .../client/models/instance_info.go | 10 +- .../create_sync_action_responses.go | 11 +- .../operations/describe_instance_responses.go | 10 +- .../get_machine_configuration_responses.go | 10 +- .../client/operations/get_mmds_responses.go | 15 +- .../client/operations/operations_client.go | 149 ++++------------ .../patch_guest_drive_by_id_responses.go | 11 +- ...guest_network_interface_by_id_responses.go | 11 +- .../patch_machine_configuration_responses.go | 11 +- .../client/operations/patch_mmds_responses.go | 11 +- .../put_guest_boot_source_responses.go | 11 +- .../put_guest_drive_by_id_responses.go | 11 +- ...guest_network_interface_by_id_responses.go | 11 +- .../put_guest_vsock_by_id_parameters.go | 160 ------------------ .../put_guest_vsock_by_id_responses.go | 145 ---------------- .../operations/put_guest_vsock_parameters.go | 139 +++++++++++++++ .../operations/put_guest_vsock_responses.go | 140 +++++++++++++++ .../client/operations/put_logger_responses.go | 11 +- .../put_machine_configuration_responses.go | 11 +- .../client/operations/put_mmds_responses.go | 11 +- ...ker-experimental.yaml => firecracker.yaml} | 20 +-- 22 files changed, 367 insertions(+), 556 deletions(-) delete mode 100644 virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_parameters.go delete mode 100644 virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_responses.go create mode 100644 virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_parameters.go create mode 100644 virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_responses.go rename virtcontainers/pkg/firecracker/{firecracker-experimental.yaml => firecracker.yaml} (97%) diff --git a/virtcontainers/pkg/firecracker/README b/virtcontainers/pkg/firecracker/README index bb042cdd4..d88abc1f0 100644 --- a/virtcontainers/pkg/firecracker/README +++ b/virtcontainers/pkg/firecracker/README @@ -5,8 +5,8 @@ The code is generated via go-swagger https://github.com/go-swagger/go-swagger#licensing ``` -swagger generate model -f ./firecracker-experimental.yaml --model-package=client/models --client-package=client -swagger generate client -f ./firecracker-experimental.yaml --model-package=client/models --client-package=client +swagger generate model -f ./firecracker.yaml --model-package=client/models --client-package=client +swagger generate client -f ./firecracker.yaml --model-package=client/models --client-package=client ``` ``` diff --git a/virtcontainers/pkg/firecracker/client/models/instance_info.go b/virtcontainers/pkg/firecracker/client/models/instance_info.go index 972eff8d4..f5995cc12 100644 --- a/virtcontainers/pkg/firecracker/client/models/instance_info.go +++ b/virtcontainers/pkg/firecracker/client/models/instance_info.go @@ -25,7 +25,7 @@ type InstanceInfo struct { // The current detailed state of the Firecracker instance. This value is read-only for the control-plane. // Required: true - // Enum: [Uninitialized Starting Running Halting Halted] + // Enum: [Uninitialized Starting Running] State *string `json:"state"` // MicroVM hypervisor build version. @@ -68,7 +68,7 @@ var instanceInfoTypeStatePropEnum []interface{} func init() { var res []string - if err := json.Unmarshal([]byte(`["Uninitialized","Starting","Running","Halting","Halted"]`), &res); err != nil { + if err := json.Unmarshal([]byte(`["Uninitialized","Starting","Running"]`), &res); err != nil { panic(err) } for _, v := range res { @@ -86,12 +86,6 @@ const ( // InstanceInfoStateRunning captures enum value "Running" InstanceInfoStateRunning string = "Running" - - // InstanceInfoStateHalting captures enum value "Halting" - InstanceInfoStateHalting string = "Halting" - - // InstanceInfoStateHalted captures enum value "Halted" - InstanceInfoStateHalted string = "Halted" ) // prop value enum diff --git a/virtcontainers/pkg/firecracker/client/operations/create_sync_action_responses.go b/virtcontainers/pkg/firecracker/client/operations/create_sync_action_responses.go index bb6743955..58cdb44d8 100644 --- a/virtcontainers/pkg/firecracker/client/operations/create_sync_action_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/create_sync_action_responses.go @@ -24,18 +24,21 @@ type CreateSyncActionReader struct { // ReadResponse reads a server response into the received o. func (o *CreateSyncActionReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewCreateSyncActionNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewCreateSyncActionBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewCreateSyncActionDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *CreateSyncActionBadRequest) Error() string { return fmt.Sprintf("[PUT /actions][%d] createSyncActionBadRequest %+v", 400, o.Payload) } -func (o *CreateSyncActionBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *CreateSyncActionBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *CreateSyncActionDefault) Error() string { return fmt.Sprintf("[PUT /actions][%d] createSyncAction default %+v", o._statusCode, o.Payload) } -func (o *CreateSyncActionDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *CreateSyncActionDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/describe_instance_responses.go b/virtcontainers/pkg/firecracker/client/operations/describe_instance_responses.go index 3b21f1d32..3c63f9dd4 100644 --- a/virtcontainers/pkg/firecracker/client/operations/describe_instance_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/describe_instance_responses.go @@ -24,12 +24,14 @@ type DescribeInstanceReader struct { // ReadResponse reads a server response into the received o. func (o *DescribeInstanceReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 200: result := NewDescribeInstanceOK() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + default: result := NewDescribeInstanceDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -59,10 +61,6 @@ func (o *DescribeInstanceOK) Error() string { return fmt.Sprintf("[GET /][%d] describeInstanceOK %+v", 200, o.Payload) } -func (o *DescribeInstanceOK) GetPayload() *models.InstanceInfo { - return o.Payload -} - func (o *DescribeInstanceOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.InstanceInfo) @@ -101,10 +99,6 @@ func (o *DescribeInstanceDefault) Error() string { return fmt.Sprintf("[GET /][%d] describeInstance default %+v", o._statusCode, o.Payload) } -func (o *DescribeInstanceDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *DescribeInstanceDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/get_machine_configuration_responses.go b/virtcontainers/pkg/firecracker/client/operations/get_machine_configuration_responses.go index f73171d9c..e9bff0f89 100644 --- a/virtcontainers/pkg/firecracker/client/operations/get_machine_configuration_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/get_machine_configuration_responses.go @@ -24,12 +24,14 @@ type GetMachineConfigurationReader struct { // ReadResponse reads a server response into the received o. func (o *GetMachineConfigurationReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 200: result := NewGetMachineConfigurationOK() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + default: result := NewGetMachineConfigurationDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -59,10 +61,6 @@ func (o *GetMachineConfigurationOK) Error() string { return fmt.Sprintf("[GET /machine-config][%d] getMachineConfigurationOK %+v", 200, o.Payload) } -func (o *GetMachineConfigurationOK) GetPayload() *models.MachineConfiguration { - return o.Payload -} - func (o *GetMachineConfigurationOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.MachineConfiguration) @@ -101,10 +99,6 @@ func (o *GetMachineConfigurationDefault) Error() string { return fmt.Sprintf("[GET /machine-config][%d] getMachineConfiguration default %+v", o._statusCode, o.Payload) } -func (o *GetMachineConfigurationDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *GetMachineConfigurationDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/get_mmds_responses.go b/virtcontainers/pkg/firecracker/client/operations/get_mmds_responses.go index 4390227d4..4119bf8ef 100644 --- a/virtcontainers/pkg/firecracker/client/operations/get_mmds_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/get_mmds_responses.go @@ -24,18 +24,21 @@ type GetMmdsReader struct { // ReadResponse reads a server response into the received o. func (o *GetMmdsReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 200: result := NewGetMmdsOK() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewGetMmdsBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewGetMmdsDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -65,10 +68,6 @@ func (o *GetMmdsOK) Error() string { return fmt.Sprintf("[GET /mmds][%d] getMmdsOK %+v", 200, o.Payload) } -func (o *GetMmdsOK) GetPayload() interface{} { - return o.Payload -} - func (o *GetMmdsOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { // response payload @@ -96,10 +95,6 @@ func (o *GetMmdsBadRequest) Error() string { return fmt.Sprintf("[GET /mmds][%d] getMmdsBadRequest %+v", 400, o.Payload) } -func (o *GetMmdsBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *GetMmdsBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -138,10 +133,6 @@ func (o *GetMmdsDefault) Error() string { return fmt.Sprintf("[GET /mmds][%d] GetMmds default %+v", o._statusCode, o.Payload) } -func (o *GetMmdsDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *GetMmdsDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/operations_client.go b/virtcontainers/pkg/firecracker/client/operations/operations_client.go index 960596baa..1be8b37cf 100644 --- a/virtcontainers/pkg/firecracker/client/operations/operations_client.go +++ b/virtcontainers/pkg/firecracker/client/operations/operations_client.go @@ -48,13 +48,8 @@ func (a *Client) GetMmds(params *GetMmdsParams) (*GetMmdsOK, error) { if err != nil { return nil, err } - success, ok := result.(*GetMmdsOK) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*GetMmdsDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*GetMmdsOK), nil + } /* @@ -81,13 +76,8 @@ func (a *Client) PatchMmds(params *PatchMmdsParams) (*PatchMmdsNoContent, error) if err != nil { return nil, err } - success, ok := result.(*PatchMmdsNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PatchMmdsDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PatchMmdsNoContent), nil + } /* @@ -114,13 +104,8 @@ func (a *Client) PutMmds(params *PutMmdsParams) (*PutMmdsNoContent, error) { if err != nil { return nil, err } - success, ok := result.(*PutMmdsNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutMmdsDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutMmdsNoContent), nil + } /* @@ -147,13 +132,8 @@ func (a *Client) CreateSyncAction(params *CreateSyncActionParams) (*CreateSyncAc if err != nil { return nil, err } - success, ok := result.(*CreateSyncActionNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*CreateSyncActionDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*CreateSyncActionNoContent), nil + } /* @@ -180,13 +160,8 @@ func (a *Client) DescribeInstance(params *DescribeInstanceParams) (*DescribeInst if err != nil { return nil, err } - success, ok := result.(*DescribeInstanceOK) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*DescribeInstanceDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*DescribeInstanceOK), nil + } /* @@ -215,13 +190,8 @@ func (a *Client) GetMachineConfiguration(params *GetMachineConfigurationParams) if err != nil { return nil, err } - success, ok := result.(*GetMachineConfigurationOK) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*GetMachineConfigurationDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*GetMachineConfigurationOK), nil + } /* @@ -250,13 +220,8 @@ func (a *Client) PatchGuestDriveByID(params *PatchGuestDriveByIDParams) (*PatchG if err != nil { return nil, err } - success, ok := result.(*PatchGuestDriveByIDNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PatchGuestDriveByIDDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PatchGuestDriveByIDNoContent), nil + } /* @@ -285,13 +250,8 @@ func (a *Client) PatchGuestNetworkInterfaceByID(params *PatchGuestNetworkInterfa if err != nil { return nil, err } - success, ok := result.(*PatchGuestNetworkInterfaceByIDNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PatchGuestNetworkInterfaceByIDDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PatchGuestNetworkInterfaceByIDNoContent), nil + } /* @@ -320,13 +280,8 @@ func (a *Client) PatchMachineConfiguration(params *PatchMachineConfigurationPara if err != nil { return nil, err } - success, ok := result.(*PatchMachineConfigurationNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PatchMachineConfigurationDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PatchMachineConfigurationNoContent), nil + } /* @@ -355,13 +310,8 @@ func (a *Client) PutGuestBootSource(params *PutGuestBootSourceParams) (*PutGuest if err != nil { return nil, err } - success, ok := result.(*PutGuestBootSourceNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutGuestBootSourceDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutGuestBootSourceNoContent), nil + } /* @@ -390,13 +340,8 @@ func (a *Client) PutGuestDriveByID(params *PutGuestDriveByIDParams) (*PutGuestDr if err != nil { return nil, err } - success, ok := result.(*PutGuestDriveByIDNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutGuestDriveByIDDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutGuestDriveByIDNoContent), nil + } /* @@ -425,48 +370,38 @@ func (a *Client) PutGuestNetworkInterfaceByID(params *PutGuestNetworkInterfaceBy if err != nil { return nil, err } - success, ok := result.(*PutGuestNetworkInterfaceByIDNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutGuestNetworkInterfaceByIDDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutGuestNetworkInterfaceByIDNoContent), nil + } /* -PutGuestVsockByID creates new vsock with ID specified by the id parameter +PutGuestVsock creates updates a vsock device -If the vsock device with the specified ID already exists, its body will be updated based on the new input. May fail if update is not possible. +The first call creates the device with the configuration specified in body. Subsequent calls will update the device configuration. May fail if update is not possible. */ -func (a *Client) PutGuestVsockByID(params *PutGuestVsockByIDParams) (*PutGuestVsockByIDNoContent, error) { +func (a *Client) PutGuestVsock(params *PutGuestVsockParams) (*PutGuestVsockNoContent, error) { // TODO: Validate the params before sending if params == nil { - params = NewPutGuestVsockByIDParams() + params = NewPutGuestVsockParams() } result, err := a.transport.Submit(&runtime.ClientOperation{ - ID: "putGuestVsockByID", + ID: "putGuestVsock", Method: "PUT", - PathPattern: "/vsocks/{id}", + PathPattern: "/vsock", ProducesMediaTypes: []string{"application/json"}, ConsumesMediaTypes: []string{"application/json"}, Schemes: []string{"http"}, Params: params, - Reader: &PutGuestVsockByIDReader{formats: a.formats}, + Reader: &PutGuestVsockReader{formats: a.formats}, Context: params.Context, Client: params.HTTPClient, }) if err != nil { return nil, err } - success, ok := result.(*PutGuestVsockByIDNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutGuestVsockByIDDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutGuestVsockNoContent), nil + } /* @@ -493,13 +428,8 @@ func (a *Client) PutLogger(params *PutLoggerParams) (*PutLoggerNoContent, error) if err != nil { return nil, err } - success, ok := result.(*PutLoggerNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutLoggerDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutLoggerNoContent), nil + } /* @@ -528,13 +458,8 @@ func (a *Client) PutMachineConfiguration(params *PutMachineConfigurationParams) if err != nil { return nil, err } - success, ok := result.(*PutMachineConfigurationNoContent) - if ok { - return success, nil - } - // unexpected success response - unexpectedSuccess := result.(*PutMachineConfigurationDefault) - return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) + return result.(*PutMachineConfigurationNoContent), nil + } // SetTransport changes the transport on the client diff --git a/virtcontainers/pkg/firecracker/client/operations/patch_guest_drive_by_id_responses.go b/virtcontainers/pkg/firecracker/client/operations/patch_guest_drive_by_id_responses.go index ba80a3e21..672473f83 100644 --- a/virtcontainers/pkg/firecracker/client/operations/patch_guest_drive_by_id_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/patch_guest_drive_by_id_responses.go @@ -24,18 +24,21 @@ type PatchGuestDriveByIDReader struct { // ReadResponse reads a server response into the received o. func (o *PatchGuestDriveByIDReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPatchGuestDriveByIDNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPatchGuestDriveByIDBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPatchGuestDriveByIDDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PatchGuestDriveByIDBadRequest) Error() string { return fmt.Sprintf("[PATCH /drives/{drive_id}][%d] patchGuestDriveByIdBadRequest %+v", 400, o.Payload) } -func (o *PatchGuestDriveByIDBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchGuestDriveByIDBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PatchGuestDriveByIDDefault) Error() string { return fmt.Sprintf("[PATCH /drives/{drive_id}][%d] patchGuestDriveByID default %+v", o._statusCode, o.Payload) } -func (o *PatchGuestDriveByIDDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchGuestDriveByIDDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/patch_guest_network_interface_by_id_responses.go b/virtcontainers/pkg/firecracker/client/operations/patch_guest_network_interface_by_id_responses.go index 652130d75..cb9ed11b7 100644 --- a/virtcontainers/pkg/firecracker/client/operations/patch_guest_network_interface_by_id_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/patch_guest_network_interface_by_id_responses.go @@ -24,18 +24,21 @@ type PatchGuestNetworkInterfaceByIDReader struct { // ReadResponse reads a server response into the received o. func (o *PatchGuestNetworkInterfaceByIDReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPatchGuestNetworkInterfaceByIDNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPatchGuestNetworkInterfaceByIDBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPatchGuestNetworkInterfaceByIDDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PatchGuestNetworkInterfaceByIDBadRequest) Error() string { return fmt.Sprintf("[PATCH /network-interfaces/{iface_id}][%d] patchGuestNetworkInterfaceByIdBadRequest %+v", 400, o.Payload) } -func (o *PatchGuestNetworkInterfaceByIDBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchGuestNetworkInterfaceByIDBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PatchGuestNetworkInterfaceByIDDefault) Error() string { return fmt.Sprintf("[PATCH /network-interfaces/{iface_id}][%d] patchGuestNetworkInterfaceByID default %+v", o._statusCode, o.Payload) } -func (o *PatchGuestNetworkInterfaceByIDDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchGuestNetworkInterfaceByIDDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/patch_machine_configuration_responses.go b/virtcontainers/pkg/firecracker/client/operations/patch_machine_configuration_responses.go index f828e63c1..b796d056a 100644 --- a/virtcontainers/pkg/firecracker/client/operations/patch_machine_configuration_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/patch_machine_configuration_responses.go @@ -24,18 +24,21 @@ type PatchMachineConfigurationReader struct { // ReadResponse reads a server response into the received o. func (o *PatchMachineConfigurationReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPatchMachineConfigurationNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPatchMachineConfigurationBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPatchMachineConfigurationDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PatchMachineConfigurationBadRequest) Error() string { return fmt.Sprintf("[PATCH /machine-config][%d] patchMachineConfigurationBadRequest %+v", 400, o.Payload) } -func (o *PatchMachineConfigurationBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchMachineConfigurationBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PatchMachineConfigurationDefault) Error() string { return fmt.Sprintf("[PATCH /machine-config][%d] patchMachineConfiguration default %+v", o._statusCode, o.Payload) } -func (o *PatchMachineConfigurationDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchMachineConfigurationDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/patch_mmds_responses.go b/virtcontainers/pkg/firecracker/client/operations/patch_mmds_responses.go index fe0a8e993..113c5abaa 100644 --- a/virtcontainers/pkg/firecracker/client/operations/patch_mmds_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/patch_mmds_responses.go @@ -24,18 +24,21 @@ type PatchMmdsReader struct { // ReadResponse reads a server response into the received o. func (o *PatchMmdsReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPatchMmdsNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPatchMmdsBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPatchMmdsDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PatchMmdsBadRequest) Error() string { return fmt.Sprintf("[PATCH /mmds][%d] patchMmdsBadRequest %+v", 400, o.Payload) } -func (o *PatchMmdsBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchMmdsBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PatchMmdsDefault) Error() string { return fmt.Sprintf("[PATCH /mmds][%d] PatchMmds default %+v", o._statusCode, o.Payload) } -func (o *PatchMmdsDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PatchMmdsDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_boot_source_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_boot_source_responses.go index 5c7f60478..3d39bebe8 100644 --- a/virtcontainers/pkg/firecracker/client/operations/put_guest_boot_source_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/put_guest_boot_source_responses.go @@ -24,18 +24,21 @@ type PutGuestBootSourceReader struct { // ReadResponse reads a server response into the received o. func (o *PutGuestBootSourceReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPutGuestBootSourceNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPutGuestBootSourceBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPutGuestBootSourceDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PutGuestBootSourceBadRequest) Error() string { return fmt.Sprintf("[PUT /boot-source][%d] putGuestBootSourceBadRequest %+v", 400, o.Payload) } -func (o *PutGuestBootSourceBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PutGuestBootSourceBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PutGuestBootSourceDefault) Error() string { return fmt.Sprintf("[PUT /boot-source][%d] putGuestBootSource default %+v", o._statusCode, o.Payload) } -func (o *PutGuestBootSourceDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PutGuestBootSourceDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_drive_by_id_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_drive_by_id_responses.go index a6909f865..6dc8d6419 100644 --- a/virtcontainers/pkg/firecracker/client/operations/put_guest_drive_by_id_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/put_guest_drive_by_id_responses.go @@ -24,18 +24,21 @@ type PutGuestDriveByIDReader struct { // ReadResponse reads a server response into the received o. func (o *PutGuestDriveByIDReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPutGuestDriveByIDNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPutGuestDriveByIDBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPutGuestDriveByIDDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PutGuestDriveByIDBadRequest) Error() string { return fmt.Sprintf("[PUT /drives/{drive_id}][%d] putGuestDriveByIdBadRequest %+v", 400, o.Payload) } -func (o *PutGuestDriveByIDBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PutGuestDriveByIDBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PutGuestDriveByIDDefault) Error() string { return fmt.Sprintf("[PUT /drives/{drive_id}][%d] putGuestDriveByID default %+v", o._statusCode, o.Payload) } -func (o *PutGuestDriveByIDDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PutGuestDriveByIDDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_network_interface_by_id_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_network_interface_by_id_responses.go index ed4fbf781..ebfbe2d60 100644 --- a/virtcontainers/pkg/firecracker/client/operations/put_guest_network_interface_by_id_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/put_guest_network_interface_by_id_responses.go @@ -24,18 +24,21 @@ type PutGuestNetworkInterfaceByIDReader struct { // ReadResponse reads a server response into the received o. func (o *PutGuestNetworkInterfaceByIDReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPutGuestNetworkInterfaceByIDNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPutGuestNetworkInterfaceByIDBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPutGuestNetworkInterfaceByIDDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PutGuestNetworkInterfaceByIDBadRequest) Error() string { return fmt.Sprintf("[PUT /network-interfaces/{iface_id}][%d] putGuestNetworkInterfaceByIdBadRequest %+v", 400, o.Payload) } -func (o *PutGuestNetworkInterfaceByIDBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PutGuestNetworkInterfaceByIDBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PutGuestNetworkInterfaceByIDDefault) Error() string { return fmt.Sprintf("[PUT /network-interfaces/{iface_id}][%d] putGuestNetworkInterfaceByID default %+v", o._statusCode, o.Payload) } -func (o *PutGuestNetworkInterfaceByIDDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PutGuestNetworkInterfaceByIDDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_parameters.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_parameters.go deleted file mode 100644 index 29a85e270..000000000 --- a/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_parameters.go +++ /dev/null @@ -1,160 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - - strfmt "github.com/go-openapi/strfmt" - - models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" -) - -// NewPutGuestVsockByIDParams creates a new PutGuestVsockByIDParams object -// with the default values initialized. -func NewPutGuestVsockByIDParams() *PutGuestVsockByIDParams { - var () - return &PutGuestVsockByIDParams{ - - timeout: cr.DefaultTimeout, - } -} - -// NewPutGuestVsockByIDParamsWithTimeout creates a new PutGuestVsockByIDParams object -// with the default values initialized, and the ability to set a timeout on a request -func NewPutGuestVsockByIDParamsWithTimeout(timeout time.Duration) *PutGuestVsockByIDParams { - var () - return &PutGuestVsockByIDParams{ - - timeout: timeout, - } -} - -// NewPutGuestVsockByIDParamsWithContext creates a new PutGuestVsockByIDParams object -// with the default values initialized, and the ability to set a context for a request -func NewPutGuestVsockByIDParamsWithContext(ctx context.Context) *PutGuestVsockByIDParams { - var () - return &PutGuestVsockByIDParams{ - - Context: ctx, - } -} - -// NewPutGuestVsockByIDParamsWithHTTPClient creates a new PutGuestVsockByIDParams object -// with the default values initialized, and the ability to set a custom HTTPClient for a request -func NewPutGuestVsockByIDParamsWithHTTPClient(client *http.Client) *PutGuestVsockByIDParams { - var () - return &PutGuestVsockByIDParams{ - HTTPClient: client, - } -} - -/*PutGuestVsockByIDParams contains all the parameters to send to the API endpoint -for the put guest vsock by ID operation typically these are written to a http.Request -*/ -type PutGuestVsockByIDParams struct { - - /*Body - Guest vsock properties - - */ - Body *models.Vsock - /*ID - The id of the vsock device - - */ - ID string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithTimeout adds the timeout to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) WithTimeout(timeout time.Duration) *PutGuestVsockByIDParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) WithContext(ctx context.Context) *PutGuestVsockByIDParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) WithHTTPClient(client *http.Client) *PutGuestVsockByIDParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) WithBody(body *models.Vsock) *PutGuestVsockByIDParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) SetBody(body *models.Vsock) { - o.Body = body -} - -// WithID adds the id to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) WithID(id string) *PutGuestVsockByIDParams { - o.SetID(id) - return o -} - -// SetID adds the id to the put guest vsock by ID params -func (o *PutGuestVsockByIDParams) SetID(id string) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *PutGuestVsockByIDParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param id - if err := r.SetPathParam("id", o.ID); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_responses.go deleted file mode 100644 index 74e56f91b..000000000 --- a/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_by_id_responses.go +++ /dev/null @@ -1,145 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - - strfmt "github.com/go-openapi/strfmt" - - models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" -) - -// PutGuestVsockByIDReader is a Reader for the PutGuestVsockByID structure. -type PutGuestVsockByIDReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *PutGuestVsockByIDReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewPutGuestVsockByIDNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewPutGuestVsockByIDBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - result := NewPutGuestVsockByIDDefault(response.Code()) - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - if response.Code()/100 == 2 { - return result, nil - } - return nil, result - } -} - -// NewPutGuestVsockByIDNoContent creates a PutGuestVsockByIDNoContent with default headers values -func NewPutGuestVsockByIDNoContent() *PutGuestVsockByIDNoContent { - return &PutGuestVsockByIDNoContent{} -} - -/*PutGuestVsockByIDNoContent handles this case with default header values. - -Vsock created/updated -*/ -type PutGuestVsockByIDNoContent struct { -} - -func (o *PutGuestVsockByIDNoContent) Error() string { - return fmt.Sprintf("[PUT /vsocks/{id}][%d] putGuestVsockByIdNoContent ", 204) -} - -func (o *PutGuestVsockByIDNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewPutGuestVsockByIDBadRequest creates a PutGuestVsockByIDBadRequest with default headers values -func NewPutGuestVsockByIDBadRequest() *PutGuestVsockByIDBadRequest { - return &PutGuestVsockByIDBadRequest{} -} - -/*PutGuestVsockByIDBadRequest handles this case with default header values. - -Vsock cannot be created due to bad input -*/ -type PutGuestVsockByIDBadRequest struct { - Payload *models.Error -} - -func (o *PutGuestVsockByIDBadRequest) Error() string { - return fmt.Sprintf("[PUT /vsocks/{id}][%d] putGuestVsockByIdBadRequest %+v", 400, o.Payload) -} - -func (o *PutGuestVsockByIDBadRequest) GetPayload() *models.Error { - return o.Payload -} - -func (o *PutGuestVsockByIDBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Error) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewPutGuestVsockByIDDefault creates a PutGuestVsockByIDDefault with default headers values -func NewPutGuestVsockByIDDefault(code int) *PutGuestVsockByIDDefault { - return &PutGuestVsockByIDDefault{ - _statusCode: code, - } -} - -/*PutGuestVsockByIDDefault handles this case with default header values. - -Internal server error -*/ -type PutGuestVsockByIDDefault struct { - _statusCode int - - Payload *models.Error -} - -// Code gets the status code for the put guest vsock by ID default response -func (o *PutGuestVsockByIDDefault) Code() int { - return o._statusCode -} - -func (o *PutGuestVsockByIDDefault) Error() string { - return fmt.Sprintf("[PUT /vsocks/{id}][%d] putGuestVsockByID default %+v", o._statusCode, o.Payload) -} - -func (o *PutGuestVsockByIDDefault) GetPayload() *models.Error { - return o.Payload -} - -func (o *PutGuestVsockByIDDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Error) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_parameters.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_parameters.go new file mode 100644 index 000000000..f57ddeaf4 --- /dev/null +++ b/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_parameters.go @@ -0,0 +1,139 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + + strfmt "github.com/go-openapi/strfmt" + + models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" +) + +// NewPutGuestVsockParams creates a new PutGuestVsockParams object +// with the default values initialized. +func NewPutGuestVsockParams() *PutGuestVsockParams { + var () + return &PutGuestVsockParams{ + + timeout: cr.DefaultTimeout, + } +} + +// NewPutGuestVsockParamsWithTimeout creates a new PutGuestVsockParams object +// with the default values initialized, and the ability to set a timeout on a request +func NewPutGuestVsockParamsWithTimeout(timeout time.Duration) *PutGuestVsockParams { + var () + return &PutGuestVsockParams{ + + timeout: timeout, + } +} + +// NewPutGuestVsockParamsWithContext creates a new PutGuestVsockParams object +// with the default values initialized, and the ability to set a context for a request +func NewPutGuestVsockParamsWithContext(ctx context.Context) *PutGuestVsockParams { + var () + return &PutGuestVsockParams{ + + Context: ctx, + } +} + +// NewPutGuestVsockParamsWithHTTPClient creates a new PutGuestVsockParams object +// with the default values initialized, and the ability to set a custom HTTPClient for a request +func NewPutGuestVsockParamsWithHTTPClient(client *http.Client) *PutGuestVsockParams { + var () + return &PutGuestVsockParams{ + HTTPClient: client, + } +} + +/*PutGuestVsockParams contains all the parameters to send to the API endpoint +for the put guest vsock operation typically these are written to a http.Request +*/ +type PutGuestVsockParams struct { + + /*Body + Guest vsock properties + + */ + Body *models.Vsock + + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithTimeout adds the timeout to the put guest vsock params +func (o *PutGuestVsockParams) WithTimeout(timeout time.Duration) *PutGuestVsockParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the put guest vsock params +func (o *PutGuestVsockParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the put guest vsock params +func (o *PutGuestVsockParams) WithContext(ctx context.Context) *PutGuestVsockParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the put guest vsock params +func (o *PutGuestVsockParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the put guest vsock params +func (o *PutGuestVsockParams) WithHTTPClient(client *http.Client) *PutGuestVsockParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the put guest vsock params +func (o *PutGuestVsockParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WithBody adds the body to the put guest vsock params +func (o *PutGuestVsockParams) WithBody(body *models.Vsock) *PutGuestVsockParams { + o.SetBody(body) + return o +} + +// SetBody adds the body to the put guest vsock params +func (o *PutGuestVsockParams) SetBody(body *models.Vsock) { + o.Body = body +} + +// WriteToRequest writes these params to a swagger request +func (o *PutGuestVsockParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + + if o.Body != nil { + if err := r.SetBodyParam(o.Body); err != nil { + return err + } + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_responses.go new file mode 100644 index 000000000..d6b78ac8b --- /dev/null +++ b/virtcontainers/pkg/firecracker/client/operations/put_guest_vsock_responses.go @@ -0,0 +1,140 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "fmt" + "io" + + "github.com/go-openapi/runtime" + + strfmt "github.com/go-openapi/strfmt" + + models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" +) + +// PutGuestVsockReader is a Reader for the PutGuestVsock structure. +type PutGuestVsockReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *PutGuestVsockReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + + case 204: + result := NewPutGuestVsockNoContent() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + + case 400: + result := NewPutGuestVsockBadRequest() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return nil, result + + default: + result := NewPutGuestVsockDefault(response.Code()) + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + if response.Code()/100 == 2 { + return result, nil + } + return nil, result + } +} + +// NewPutGuestVsockNoContent creates a PutGuestVsockNoContent with default headers values +func NewPutGuestVsockNoContent() *PutGuestVsockNoContent { + return &PutGuestVsockNoContent{} +} + +/*PutGuestVsockNoContent handles this case with default header values. + +Vsock created/updated +*/ +type PutGuestVsockNoContent struct { +} + +func (o *PutGuestVsockNoContent) Error() string { + return fmt.Sprintf("[PUT /vsock][%d] putGuestVsockNoContent ", 204) +} + +func (o *PutGuestVsockNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + return nil +} + +// NewPutGuestVsockBadRequest creates a PutGuestVsockBadRequest with default headers values +func NewPutGuestVsockBadRequest() *PutGuestVsockBadRequest { + return &PutGuestVsockBadRequest{} +} + +/*PutGuestVsockBadRequest handles this case with default header values. + +Vsock cannot be created due to bad input +*/ +type PutGuestVsockBadRequest struct { + Payload *models.Error +} + +func (o *PutGuestVsockBadRequest) Error() string { + return fmt.Sprintf("[PUT /vsock][%d] putGuestVsockBadRequest %+v", 400, o.Payload) +} + +func (o *PutGuestVsockBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.Error) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} + +// NewPutGuestVsockDefault creates a PutGuestVsockDefault with default headers values +func NewPutGuestVsockDefault(code int) *PutGuestVsockDefault { + return &PutGuestVsockDefault{ + _statusCode: code, + } +} + +/*PutGuestVsockDefault handles this case with default header values. + +Internal server error +*/ +type PutGuestVsockDefault struct { + _statusCode int + + Payload *models.Error +} + +// Code gets the status code for the put guest vsock default response +func (o *PutGuestVsockDefault) Code() int { + return o._statusCode +} + +func (o *PutGuestVsockDefault) Error() string { + return fmt.Sprintf("[PUT /vsock][%d] putGuestVsock default %+v", o._statusCode, o.Payload) +} + +func (o *PutGuestVsockDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.Error) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} diff --git a/virtcontainers/pkg/firecracker/client/operations/put_logger_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_logger_responses.go index b9bfebe9f..c98676878 100644 --- a/virtcontainers/pkg/firecracker/client/operations/put_logger_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/put_logger_responses.go @@ -24,18 +24,21 @@ type PutLoggerReader struct { // ReadResponse reads a server response into the received o. func (o *PutLoggerReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPutLoggerNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPutLoggerBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPutLoggerDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PutLoggerBadRequest) Error() string { return fmt.Sprintf("[PUT /logger][%d] putLoggerBadRequest %+v", 400, o.Payload) } -func (o *PutLoggerBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PutLoggerBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PutLoggerDefault) Error() string { return fmt.Sprintf("[PUT /logger][%d] putLogger default %+v", o._statusCode, o.Payload) } -func (o *PutLoggerDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PutLoggerDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/put_machine_configuration_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_machine_configuration_responses.go index f36aa154f..6127fea6e 100644 --- a/virtcontainers/pkg/firecracker/client/operations/put_machine_configuration_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/put_machine_configuration_responses.go @@ -24,18 +24,21 @@ type PutMachineConfigurationReader struct { // ReadResponse reads a server response into the received o. func (o *PutMachineConfigurationReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPutMachineConfigurationNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPutMachineConfigurationBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPutMachineConfigurationDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PutMachineConfigurationBadRequest) Error() string { return fmt.Sprintf("[PUT /machine-config][%d] putMachineConfigurationBadRequest %+v", 400, o.Payload) } -func (o *PutMachineConfigurationBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PutMachineConfigurationBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PutMachineConfigurationDefault) Error() string { return fmt.Sprintf("[PUT /machine-config][%d] putMachineConfiguration default %+v", o._statusCode, o.Payload) } -func (o *PutMachineConfigurationDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PutMachineConfigurationDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/client/operations/put_mmds_responses.go b/virtcontainers/pkg/firecracker/client/operations/put_mmds_responses.go index 1f8116d71..7b1e47b35 100644 --- a/virtcontainers/pkg/firecracker/client/operations/put_mmds_responses.go +++ b/virtcontainers/pkg/firecracker/client/operations/put_mmds_responses.go @@ -24,18 +24,21 @@ type PutMmdsReader struct { // ReadResponse reads a server response into the received o. func (o *PutMmdsReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { switch response.Code() { + case 204: result := NewPutMmdsNoContent() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return result, nil + case 400: result := NewPutMmdsBadRequest() if err := result.readResponse(response, consumer, o.formats); err != nil { return nil, err } return nil, result + default: result := NewPutMmdsDefault(response.Code()) if err := result.readResponse(response, consumer, o.formats); err != nil { @@ -86,10 +89,6 @@ func (o *PutMmdsBadRequest) Error() string { return fmt.Sprintf("[PUT /mmds][%d] putMmdsBadRequest %+v", 400, o.Payload) } -func (o *PutMmdsBadRequest) GetPayload() *models.Error { - return o.Payload -} - func (o *PutMmdsBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) @@ -128,10 +127,6 @@ func (o *PutMmdsDefault) Error() string { return fmt.Sprintf("[PUT /mmds][%d] PutMmds default %+v", o._statusCode, o.Payload) } -func (o *PutMmdsDefault) GetPayload() *models.Error { - return o.Payload -} - func (o *PutMmdsDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { o.Payload = new(models.Error) diff --git a/virtcontainers/pkg/firecracker/firecracker-experimental.yaml b/virtcontainers/pkg/firecracker/firecracker.yaml similarity index 97% rename from virtcontainers/pkg/firecracker/firecracker-experimental.yaml rename to virtcontainers/pkg/firecracker/firecracker.yaml index e54415e00..5f7fcbdc8 100644 --- a/virtcontainers/pkg/firecracker/firecracker-experimental.yaml +++ b/virtcontainers/pkg/firecracker/firecracker.yaml @@ -5,7 +5,7 @@ info: The API is accessible through HTTP calls on specific URLs carrying JSON modeled data. The transport medium is a Unix Domain Socket. - version: 0.18.0 + version: 0.19.0 termsOfService: "" contact: email: "compute-capsule@amazon.com" @@ -353,19 +353,15 @@ paths: schema: $ref: "#/definitions/Error" - /vsocks/{id}: + /vsock: put: - summary: Creates new vsock with ID specified by the id parameter. + summary: Creates/updates a vsock device. description: - If the vsock device with the specified ID already exists, its body will - be updated based on the new input. May fail if update is not possible. - operationId: putGuestVsockByID + The first call creates the device with the configuration specified + in body. Subsequent calls will update the device configuration. + May fail if update is not possible. + operationId: putGuestVsock parameters: - - name: id - in: path - description: The id of the vsock device - required: true - type: string - name: body in: body description: Guest vsock properties @@ -481,8 +477,6 @@ definitions: - Uninitialized - Starting - Running - - Halting - - Halted vmm_version: description: MicroVM hypervisor build version. type: string From 77b0dfb05f8a05a2c1083002134a742d45de81dd Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 11 Nov 2019 22:19:57 +0000 Subject: [PATCH 12/23] virtcontainers: use new firecracker API Support new firecracker API 0.19.0: * remove vsock ID from http request fixes #2183 Signed-off-by: Julio Montes --- virtcontainers/fc.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index c92e63e6c..fb2e32aca 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -299,7 +299,7 @@ func (fc *firecracker) vmRunning() bool { return false case models.InstanceInfoStateRunning: return true - case models.InstanceInfoStateUninitialized, models.InstanceInfoStateHalting, models.InstanceInfoStateHalted: + case models.InstanceInfoStateUninitialized: return false default: return false @@ -784,7 +784,7 @@ func (fc *firecracker) fcAddVsock(hvs types.HybridVSock) error { udsPath = filepath.Join("/", defaultHybridVSocketName) } - vsockParams := ops.NewPutGuestVsockByIDParams() + vsockParams := ops.NewPutGuestVsockParams() vsockID := "root" ctxID := defaultGuestVSockCID vsock := &models.Vsock{ @@ -792,10 +792,9 @@ func (fc *firecracker) fcAddVsock(hvs types.HybridVSock) error { UdsPath: &udsPath, VsockID: &vsockID, } - vsockParams.SetID(vsockID) vsockParams.SetBody(vsock) - _, err := fc.client().Operations.PutGuestVsockByID(vsockParams) + _, err := fc.client().Operations.PutGuestVsock(vsockParams) if err != nil { return err } From 78ca966e8d615798f8193c5ed38de985cd0cbfc4 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 11 Nov 2019 22:21:43 +0000 Subject: [PATCH 13/23] virtcontainers: bump firecracker minimum supported version firecracker 0.19.0 API is not backward compatible, hence we need to bump the firecracker minimum supported version to 0.19.0 Signed-off-by: Julio Montes --- virtcontainers/fc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index fb2e32aca..bebad7c3f 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -69,7 +69,7 @@ const ( ) // Specify the minimum version of firecracker supported -var fcMinSupportedVersion = semver.MustParse("0.18.0") +var fcMinSupportedVersion = semver.MustParse("0.19.0") var fcKernelParams = append(commonVirtioblkKernelRootParams, []Param{ // The boot source is the first partition of the first block device added From 615421081a660f481bb2580b958fe827471783b1 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 11 Nov 2019 22:28:01 +0000 Subject: [PATCH 14/23] versions: bump firecracker version update to firecracker 0.19.0 Signed-off-by: Julio Montes --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 1196c5be4..a61a6a88e 100644 --- a/versions.yaml +++ b/versions.yaml @@ -76,7 +76,7 @@ assets: uscan-url: >- https://github.com/firecracker-microvm/firecracker/tags .*/v?(\d\S+)\.tar\.gz - version: "v0.18.0" + version: "v0.19.0" nemu: description: "Reduced-emulation VMM that uses KVM" From 519eff723666f3ae419cf5a845ef593a2e085be9 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 13 Nov 2019 14:10:18 -0800 Subject: [PATCH 15/23] fc: config: vhost-net not supported vhost-net backend is not supported by the Firecracker VMM. It doesn't make sense to have this in the configuration, and we should explicitly disable it. Fixes: #2192 Signed-off-by: Eric Ernst --- cli/config/configuration-fc.toml.in | 3 --- pkg/katautils/config.go | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/config/configuration-fc.toml.in b/cli/config/configuration-fc.toml.in index 520642d91..b272ba9e5 100644 --- a/cli/config/configuration-fc.toml.in +++ b/cli/config/configuration-fc.toml.in @@ -160,9 +160,6 @@ use_vsock = true # Default false #hotplug_vfio_on_root_bus = true -# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. -# Default false -#disable_vhost_net = true # # Default entropy source. # The path to a host source of entropy (including a real hardware RNG) diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 51d2ceba4..386f9df5a 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -533,6 +533,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableNestingChecks: h.DisableNestingChecks, BlockDeviceDriver: blockDriver, EnableIOThreads: h.EnableIOThreads, + DisableVhostNet: true, // vhost-net backend is not supported in Firecracker UseVSock: true, GuestHookPath: h.guestHookPath(), }, nil From 0c482b25576ac7cd05c2cb5edbde697baae2cea5 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 13 Nov 2019 14:48:59 -0800 Subject: [PATCH 16/23] qemu-configs: update disable_vhost_net description Updated to better clarify this configuration option. Currently in QEMU pre 1.9 release of Kata, vhost-net is used for the virtio-net backend. This results in efficient network I/O performance, but does rely on the backend running in ring0 (host kernel). Update comment to clarify this trade-off for end-users. Fixes: #2198 Signed-off-by: Eric Ernst --- cli/config/configuration-qemu-virtiofs.toml.in | 5 +++-- cli/config/configuration-qemu.toml.in | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/config/configuration-qemu-virtiofs.toml.in b/cli/config/configuration-qemu-virtiofs.toml.in index da04b4eaa..6a13cebae 100644 --- a/cli/config/configuration-qemu-virtiofs.toml.in +++ b/cli/config/configuration-qemu-virtiofs.toml.in @@ -212,9 +212,10 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #hotplug_vfio_on_root_bus = true -# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. -# Default false +# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off +# security (vhost-net runs ring0) for network I/O performance. #disable_vhost_net = true + # # Default entropy source. # The path to a host source of entropy (including a real hardware RNG) diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 3fbbf3791..d38f3a162 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -213,9 +213,10 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #hotplug_vfio_on_root_bus = true -# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. -# Default false +# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off +# security (vhost-net runs ring0) for network I/O performance. #disable_vhost_net = true + # # Default entropy source. # The path to a host source of entropy (including a real hardware RNG) From 13a00a2cf21d680fd57940f8884867d1242afd47 Mon Sep 17 00:00:00 2001 From: lifupan Date: Fri, 15 Nov 2019 11:34:26 +0800 Subject: [PATCH 17/23] virtcontainers: add a stateful to FC struct Add a 'sateful' variable to FC to indicate when it is called with cli or shimv2. Signed-off-by: lifupan --- virtcontainers/acrn.go | 2 +- virtcontainers/acrn_test.go | 2 +- virtcontainers/fc.go | 8 +++++--- virtcontainers/hypervisor.go | 2 +- virtcontainers/mock_hypervisor.go | 2 +- virtcontainers/mock_hypervisor_test.go | 4 ++-- virtcontainers/qemu.go | 2 +- virtcontainers/qemu_test.go | 10 +++++----- virtcontainers/sandbox.go | 4 ++-- virtcontainers/vm.go | 2 +- 10 files changed, 20 insertions(+), 18 deletions(-) diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go index 5c69b8ae3..a734a7ab9 100644 --- a/virtcontainers/acrn.go +++ b/virtcontainers/acrn.go @@ -348,7 +348,7 @@ func (a *Acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) { } // createSandbox is the Hypervisor sandbox creation. -func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { +func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error { // Save the tracing context a.ctx = ctx diff --git a/virtcontainers/acrn_test.go b/virtcontainers/acrn_test.go index f335833a8..12a0ea5fd 100644 --- a/virtcontainers/acrn_test.go +++ b/virtcontainers/acrn_test.go @@ -230,7 +230,7 @@ func TestAcrnCreateSandbox(t *testing.T) { //set PID to 1 to ignore hypercall to get UUID and set a random UUID a.state.PID = 1 a.state.UUID = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" - err = a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil) + err = a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false) assert.NoError(err) assert.Exactly(acrnConfig, a.config) } diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index bebad7c3f..66b01bafe 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -141,8 +141,9 @@ type firecracker struct { config HypervisorConfig pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready - state firecrackerState - jailed bool //Set to true if jailer is enabled + state firecrackerState + jailed bool //Set to true if jailer is enabled + stateful bool //Set to true if running with shimv2 } type firecrackerDevice struct { @@ -211,7 +212,7 @@ func (fc *firecracker) bindMount(ctx context.Context, source, destination string // For firecracker this call only sets the internal structure up. // The sandbox will be created and started through startSandbox(). -func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error { fc.ctx = ctx span, _ := fc.trace("createSandbox") @@ -223,6 +224,7 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N fc.store = vcStore fc.state.set(notReady) fc.config = *hypervisorConfig + fc.stateful = stateful // When running with jailer all resources need to be under // a specific location and that location needs to have diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index cdb2767a6..69bdd5ba4 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -716,7 +716,7 @@ func generateVMSocket(id string, useVsock bool) (interface{}, error) { // hypervisor is the virtcontainers hypervisor interface. // The default hypervisor implementation is Qemu. type hypervisor interface { - createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error + createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error startSandbox(timeout int) error stopSandbox() error pauseSandbox() error diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 30962e87a..30bd38cb4 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -27,7 +27,7 @@ func (m *mockHypervisor) hypervisorConfig() HypervisorConfig { return HypervisorConfig{} } -func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { +func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error { err := hypervisorConfig.valid() if err != nil { return err diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go index 799f50275..10c6a90cd 100644 --- a/virtcontainers/mock_hypervisor_test.go +++ b/virtcontainers/mock_hypervisor_test.go @@ -31,7 +31,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { ctx := context.Background() // wrong config - err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil) + err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false) assert.Error(err) sandbox.config.HypervisorConfig = HypervisorConfig{ @@ -40,7 +40,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - err = m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil) + err = m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false) assert.NoError(err) } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 95b0b0fa0..b23168fe5 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -463,7 +463,7 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo } // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. -func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error { // Save the tracing context q.ctx = ctx diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 1f0d988b7..a503bb4f7 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -99,7 +99,7 @@ func TestQemuCreateSandbox(t *testing.T) { parentDir := store.SandboxConfigurationRootPath(sandbox.id) assert.NoError(os.MkdirAll(parentDir, store.DirMode)) - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) assert.NoError(os.RemoveAll(parentDir)) assert.Exactly(qemuConfig, q.config) @@ -131,7 +131,7 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { parentDir := store.SandboxConfigurationRootPath(sandbox.id) assert.NoError(os.RemoveAll(parentDir)) - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) } @@ -429,7 +429,7 @@ func TestQemuFileBackedMem(t *testing.T) { q := &qemu{} sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) assert.Equal(q.qemuConfig.Knobs.FileBackedMem, true) @@ -445,7 +445,7 @@ func TestQemuFileBackedMem(t *testing.T) { sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS sandbox.config.HypervisorConfig.MemoryPath = fallbackFileBackedMemDir - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) expectErr := errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work") assert.Equal(expectErr.Error(), err.Error()) @@ -456,7 +456,7 @@ func TestQemuFileBackedMem(t *testing.T) { q = &qemu{} sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc" - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false) assert.Equal(q.qemuConfig.Knobs.MemShared, false) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 9771f0d24..d8d548ef3 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -569,7 +569,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor s.Restore() // new store doesn't require hypervisor to be stored immediately - if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil); err != nil { + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil, s.stateful); err != nil { return nil, err } } else { @@ -591,7 +591,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor s.state = state } - if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil { + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store, s.stateful); err != nil { return nil, err } } diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index 77f85450c..4b77c22de 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -172,7 +172,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { } }() - if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore); err != nil { + if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore, false); err != nil { return nil, err } From a2b6afcd9a95e3dca851a24733ccb627cece83af Mon Sep 17 00:00:00 2001 From: lifupan Date: Thu, 14 Nov 2019 17:54:40 +0800 Subject: [PATCH 18/23] FC: log out the firecracker's console when debug enabled For shimv2 case, when hypervisor's debug option set, log out the firecracker's console output which contains the kernel boot logs; thus it would be easy for system panic debugging. When agent debug was enabled by passing "agent.log=debug" to kernel parameter, it will also log out the agent logs from the console output. Fixes: #2201 Signed-off-by: lifupan --- Gopkg.lock | 2 +- Gopkg.toml | 4 +++ virtcontainers/fc.go | 70 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 80dcc8cb3..21ad8808c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -89,7 +89,6 @@ revision = "c4b9ac5c7601384c965b9646fc515884e091ebb9" [[projects]] - branch = "master" digest = "1:da4daad2ec1737eec4ebeeed7afedb631711f96bbac0c361a17a4d0369d00c6d" name = "github.com/containerd/console" packages = ["."] @@ -707,6 +706,7 @@ "github.com/BurntSushi/toml", "github.com/blang/semver", "github.com/containerd/cgroups", + "github.com/containerd/console", "github.com/containerd/containerd/api/events", "github.com/containerd/containerd/api/types", "github.com/containerd/containerd/api/types/task", diff --git a/Gopkg.toml b/Gopkg.toml index 076fc520b..d242baa2a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -78,6 +78,10 @@ branch = "master" name = "github.com/hashicorp/yamux" +[[constraint]] + revision = "0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f" + name = "github.com/containerd/console" + [prune] non-go = true go-tests = true diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 66b01bafe..e6fdb0a30 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -6,8 +6,10 @@ package virtcontainers import ( + "bufio" "context" "fmt" + "io" "net" "net/http" "os" @@ -31,6 +33,7 @@ import ( "github.com/sirupsen/logrus" "github.com/blang/semver" + "github.com/containerd/console" "github.com/kata-containers/runtime/virtcontainers/device/config" fcmodels "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" "github.com/kata-containers/runtime/virtcontainers/store" @@ -77,16 +80,12 @@ var fcKernelParams = append(commonVirtioblkKernelRootParams, []Param{ {"reboot", "k"}, {"panic", "1"}, {"iommu", "off"}, - {"8250.nr_uarts", "0"}, {"net.ifnames", "0"}, {"random.trust_cpu", "on"}, // Firecracker doesn't support ACPI // Fix kernel error "ACPI BIOS Error (bug)" {"acpi", "off"}, - - // Tell agent where to send the logs - {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, }...) func (s vmmState) String() string { @@ -388,13 +387,17 @@ func (fc *firecracker) fcInit(timeout int) error { var args []string var cmd *exec.Cmd + if !fc.config.Debug && fc.stateful { + args = append(args, "--daemonize") + } + //https://github.com/firecracker-microvm/firecracker/blob/master/docs/jailer.md#jailer-usage //--seccomp-level specifies whether seccomp filters should be installed and how restrictive they should be. Possible values are: //0 : disabled. //1 : basic filtering. This prohibits syscalls not whitelisted by Firecracker. //2 (default): advanced filtering. This adds further checks on some of the parameters of the allowed syscalls. if fc.jailed { - args = []string{ + args = append(args, "--id", fc.id, "--node", "0", //FIXME: Comprehend NUMA topology or explicit ignore "--seccomp-level", "2", @@ -402,8 +405,7 @@ func (fc *firecracker) fcInit(timeout int) error { "--uid", "0", //https://github.com/kata-containers/runtime/issues/1869 "--gid", "0", "--chroot-base-dir", fc.chrootBaseDir, - "--daemonize", - } + ) if fc.netNSPath != "" { args = append(args, "--netns", fc.netNSPath) } @@ -414,6 +416,16 @@ func (fc *firecracker) fcInit(timeout int) error { } + if fc.config.Debug && fc.stateful { + stdin, err := fc.watchConsole() + if err != nil { + return err + } + + cmd.Stderr = stdin + cmd.Stdout = stdin + } + fc.Logger().WithField("hypervisor args", args).Debug() fc.Logger().WithField("hypervisor cmd", cmd).Debug() if err := cmd.Start(); err != nil { @@ -662,6 +674,16 @@ func (fc *firecracker) startSandbox(timeout int) error { return err } + if fc.config.Debug && fc.stateful { + fcKernelParams = append(fcKernelParams, Param{"console", "ttyS0"}) + } else { + fcKernelParams = append(fcKernelParams, []Param{ + {"8250.nr_uarts", "0"}, + // Tell agent where to send the logs + {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, + }...) + } + kernelParams := append(fc.config.KernelParams, fcKernelParams...) strParams := SerializeParams(kernelParams, "=") formattedParams := strings.Join(strParams, " ") @@ -1099,3 +1121,37 @@ func (fc *firecracker) generateSocket(id string, useVsock bool) (interface{}, er Port: uint32(vSockPort), }, nil } + +func (fc *firecracker) watchConsole() (*os.File, error) { + master, slave, err := console.NewPty() + if err != nil { + fc.Logger().WithField("Error create pseudo tty", err).Debug() + return nil, err + } + + stdio, err := os.OpenFile(slave, syscall.O_RDWR, 0700) + if err != nil { + fc.Logger().WithError(err).Debugf("open pseudo tty %s", slave) + return nil, err + } + + go func() { + scanner := bufio.NewScanner(master) + for scanner.Scan() { + fc.Logger().WithFields(logrus.Fields{ + "sandbox": fc.id, + "vmconsole": scanner.Text(), + }).Infof("reading guest console") + } + + if err := scanner.Err(); err != nil { + if err == io.EOF { + fc.Logger().Info("console watcher quits") + } else { + fc.Logger().WithError(err).Error("Failed to read guest console") + } + } + }() + + return stdio, nil +} From 70297c2184d69d3b8f761b33e4a210c8ad7f38e5 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 13 Nov 2019 14:31:39 -0800 Subject: [PATCH 19/23] nemu: remove nemu support NEMU deprecation was announced in 1.8 of Kata. Removing from tree. Thanks for all the fish! Fixes: #2195 Signed-off-by: Eric Ernst --- Makefile | 57 +--- arch/amd64-options.mk | 3 - cli/config/configuration-nemu.toml.in | 422 -------------------------- versions.yaml | 22 -- virtcontainers/types/sandbox.go | 1 - 5 files changed, 1 insertion(+), 504 deletions(-) delete mode 100644 cli/config/configuration-nemu.toml.in diff --git a/Makefile b/Makefile index 9a27fbbca..5baf7d5e1 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,6 @@ endif # Prefix where depedencies are installed PREFIXDEPS := $(PREFIX) BINDIR := $(EXEC_PREFIX)/bin -NEMUBINDIR := $(PREFIXDEPS)/bin QEMUBINDIR := $(PREFIXDEPS)/bin FCBINDIR := $(PREFIXDEPS)/bin ACRNBINDIR := $(PREFIXDEPS)/bin @@ -117,7 +116,6 @@ CONFIG_FILE = configuration.toml HYPERVISOR_ACRN = acrn HYPERVISOR_FC = firecracker JAILER_FC = jailer -HYPERVISOR_NEMU = nemu HYPERVISOR_QEMU = qemu HYPERVISOR_QEMU_VIRTIOFS = qemu-virtiofs @@ -125,14 +123,12 @@ HYPERVISOR_QEMU_VIRTIOFS = qemu-virtiofs DEFAULT_HYPERVISOR = $(HYPERVISOR_QEMU) # List of hypervisors this build system can generate configuration for. -HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) $(HYPERVISOR_NEMU) +HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) -NEMUPATH := $(NEMUBINDIR)/$(NEMUCMD) - FCPATH = $(FCBINDIR)/$(FCCMD) FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD) @@ -172,7 +168,6 @@ DEFENTROPYSOURCE := /dev/urandom DEFDISABLEBLOCK := false DEFSHAREDFS := virtio-9p -DEFSHAREDFS_NEMU := virtio-9p DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd # Default DAX mapping cache size in MiB @@ -263,33 +258,6 @@ ifneq (,$(QEMUVIRTIOFSCMD)) KERNELVIRTIOFSPATH = $(KERNELDIR)/$(KERNELNAMEVIRTIOFS) endif -ifneq (,$(NEMUCMD)) - KNOWN_HYPERVISORS += $(HYPERVISOR_NEMU) - - CONFIG_FILE_NEMU = configuration-nemu.toml - CONFIG_NEMU = $(CLI_DIR)/config/$(CONFIG_FILE_NEMU) - CONFIG_NEMU_IN = $(CONFIG_NEMU).in - - CONFIG_PATH_NEMU = $(abspath $(CONFDIR)/$(CONFIG_FILE_NEMU)) - CONFIG_PATHS += $(CONFIG_PATH_NEMU) - - SYSCONFIG_NEMU = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_NEMU)) - SYSCONFIG_PATHS += $(SYSCONFIG_NEMU) - - CONFIGS += $(CONFIG_NEMU) - - # nemu-specific options (all should be suffixed by "_NEMU") - # currently, huge pages are required for virtiofsd support - DEFENABLEHUGEPAGES_NEMU := false - # nemu uses virt machine type - DEFMACHINETYPE_NEMU := virt - DEFBLOCKSTORAGEDRIVER_NEMU := virtio-scsi - DEFNETWORKMODEL_NEMU := tcfilter - KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) - KERNELPATH = $(KERNELDIR)/$(KERNELNAME) - FIRMWAREPATH_NEMU := $(SHAREDIR)/kata-nemu/OVMF.fd -endif - ifneq (,$(FCCMD)) KNOWN_HYPERVISORS += $(HYPERVISOR_FC) @@ -360,10 +328,6 @@ ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_FC)) DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_FC) endif -ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_NEMU)) - DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_NEMU) -endif - ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_ACRN)) DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_ACRN) endif @@ -393,8 +357,6 @@ USER_VARS += ACRNCTLPATH USER_VARS += FCCMD USER_VARS += FCPATH USER_VARS += FCJAILERPATH -USER_VARS += NEMUCMD -USER_VARS += NEMUPATH USER_VARS += SYSCONFIG USER_VARS += IMAGENAME USER_VARS += IMAGEPATH @@ -406,9 +368,7 @@ USER_VARS += KERNELTYPE USER_VARS += KERNELTYPE_FC USER_VARS += KERNELTYPE_ACRN USER_VARS += FIRMWAREPATH -USER_VARS += FIRMWAREPATH_NEMU USER_VARS += MACHINEACCELERATORS -USER_VARS += DEFMACHINETYPE_NEMU USER_VARS += KERNELPARAMS USER_VARS += LIBEXECDIR USER_VARS += LOCALSTATEDIR @@ -439,7 +399,6 @@ USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL_ACRN USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFNETWORKMODEL_QEMU -USER_VARS += DEFNETWORKMODEL_NEMU USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFAULTEXPFEATURES USER_VARS += DEFDISABLEBLOCK @@ -447,9 +406,7 @@ USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN USER_VARS += DEFBLOCKSTORAGEDRIVER_FC USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS -USER_VARS += DEFBLOCKSTORAGEDRIVER_NEMU USER_VARS += DEFSHAREDFS -USER_VARS += DEFSHAREDFS_NEMU USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS USER_VARS += DEFVIRTIOFSDAEMON USER_VARS += DEFVIRTIOFSCACHESIZE @@ -458,7 +415,6 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS USER_VARS += DEFENABLEIOTHREADS USER_VARS += DEFENABLEMEMPREALLOC USER_VARS += DEFENABLEHUGEPAGES -USER_VARS += DEFENABLEHUGEPAGES_NEMU USER_VARS += DEFENABLESWAP USER_VARS += DEFENABLEDEBUG USER_VARS += DEFDISABLENESTINGCHECKS @@ -563,12 +519,10 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@CONFIG_ACRN_IN@|$(CONFIG_ACRN_IN)|g" \ -e "s|@CONFIG_QEMU_IN@|$(CONFIG_QEMU_IN)|g" \ -e "s|@CONFIG_QEMU_VIRTIOFS_IN@|$(CONFIG_QEMU_VIRTIOFS_IN)|g" \ - -e "s|@CONFIG_NEMU_IN@|$(CONFIG_NEMU_IN)|g" \ -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ -e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \ - -e "s|@NEMUPATH@|$(NEMUPATH)|g" \ -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ @@ -579,9 +533,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@KERNELVIRTIOFSPATH@|$(KERNELVIRTIOFSPATH)|g" \ -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ -e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \ - -e "s|@FIRMWAREPATH_NEMU@|$(FIRMWAREPATH_NEMU)|g" \ -e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \ - -e "s|@DEFMACHINETYPE_NEMU@|$(DEFMACHINETYPE_NEMU)|g" \ -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ @@ -607,7 +559,6 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFNETWORKMODEL_ACRN@|$(DEFNETWORKMODEL_ACRN)|g" \ -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ - -e "s|@DEFNETWORKMODEL_NEMU@|$(DEFNETWORKMODEL_NEMU)|g" \ -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ -e "s|@DEFAULTEXPFEATURES@|$(DEFAULTEXPFEATURES)|g" \ -e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \ @@ -615,9 +566,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFBLOCKSTORAGEDRIVER_FC@|$(DEFBLOCKSTORAGEDRIVER_FC)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_QEMU@|$(DEFBLOCKSTORAGEDRIVER_QEMU)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS@|$(DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS)|g" \ - -e "s|@DEFBLOCKSTORAGEDRIVER_NEMU@|$(DEFBLOCKSTORAGEDRIVER_NEMU)|g" \ -e "s|@DEFSHAREDFS@|$(DEFSHAREDFS)|g" \ - -e "s|@DEFSHAREDFS_NEMU@|$(DEFSHAREDFS_NEMU)|g" \ -e "s|@DEFSHAREDFS_QEMU_VIRTIOFS@|$(DEFSHAREDFS_QEMU_VIRTIOFS)|g" \ -e "s|@DEFVIRTIOFSDAEMON@|$(DEFVIRTIOFSDAEMON)|g" \ -e "s|@DEFVIRTIOFSCACHESIZE@|$(DEFVIRTIOFSCACHESIZE)|g" \ @@ -626,7 +575,6 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \ -e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \ -e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \ - -e "s|@DEFENABLEHUGEPAGES_NEMU@|$(DEFENABLEHUGEPAGES_NEMU)|g" \ -e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \ -e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \ -e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \ @@ -781,9 +729,6 @@ endif ifneq (,$(findstring $(HYPERVISOR_QEMU_VIRTIOFS),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_QEMU_VIRTIOFS) hypervisor path (QEMUVIRTIOFSPATH) : %s\n" $(abspath $(QEMUVIRTIOFSPATH)) endif -ifneq (,$(findstring $(HYPERVISOR_NEMU),$(KNOWN_HYPERVISORS))) - @printf "\t$(HYPERVISOR_NEMU) hypervisor path (NEMUPATH) : %s\n" $(abspath $(NEMUPATH)) -endif ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH)) endif diff --git a/arch/amd64-options.mk b/arch/amd64-options.mk index d4b076ba0..797372f60 100644 --- a/arch/amd64-options.mk +++ b/arch/amd64-options.mk @@ -19,9 +19,6 @@ FCCMD := firecracker # Firecracker's jailer binary name FCJAILERCMD := jailer -# NEMU binary name -NEMUCMD := nemu-system-x86_64 - #ACRN binary name ACRNCMD := acrn-dm ACRNCTLCMD := acrnctl diff --git a/cli/config/configuration-nemu.toml.in b/cli/config/configuration-nemu.toml.in deleted file mode 100644 index c92bb12f5..000000000 --- a/cli/config/configuration-nemu.toml.in +++ /dev/null @@ -1,422 +0,0 @@ -# Copyright (c) 2019 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -# XXX: WARNING: this file is auto-generated. -# XXX: -# XXX: Source file: "@CONFIG_NEMU_IN@" -# XXX: Project: -# XXX: Name: @PROJECT_NAME@ -# XXX: Type: @PROJECT_TYPE@ - -# nemu utilizes the 'qemu' hypervisor template type, since it closely matches qemu -[hypervisor.qemu] -path = "@NEMUPATH@" -kernel = "@KERNELPATH@" -image = "@IMAGEPATH@" -machine_type = "@DEFMACHINETYPE_NEMU@" - -# Optional space-separated list of options to pass to the guest kernel. -# For example, use `kernel_params = "vsyscall=emulate"` if you are having -# trouble running pre-2.15 glibc. -# -# WARNING: - any parameter specified here will take priority over the default -# parameter value of the same name used to start the virtual machine. -# Do not set values here unless you understand the impact of doing so as you -# may stop the virtual machine from booting. -# To see the list of default parameters, enable hypervisor debug, create a -# container and look for 'default-kernel-parameters' log entries. -kernel_params = "@KERNELPARAMS@" - -# Path to the firmware. -# If you want that qemu uses the default firmware leave this option empty -firmware = "@FIRMWAREPATH_NEMU@" - -# Machine accelerators -# comma-separated list of machine accelerators to pass to the hypervisor. -# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` -machine_accelerators="" - -# Default number of vCPUs per SB/VM: -# unspecified or 0 --> will be set to @DEFVCPUS@ -# < 0 --> will be set to the actual number of physical cores -# > 0 <= number of physical cores --> will be set to the specified number -# > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 - -# Default maximum number of vCPUs per SB/VM: -# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number -# of vCPUs supported by KVM if that number is exceeded -# > 0 <= number of physical cores --> will be set to the specified number -# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number -# of vCPUs supported by KVM if that number is exceeded -# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when -# the actual number of physical cores is greater than it. -# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU -# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs -# can be added to a SB/VM, but the memory footprint will be big. Another example, with -# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of -# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, -# unless you know what are you doing. -default_maxvcpus = @DEFMAXVCPUS@ - -# Bridges can be used to hot plug devices. -# Limitations: -# * Currently only pci bridges are supported -# * Until 30 devices per bridge can be hot plugged. -# * Until 5 PCI bridges can be cold plugged per VM. -# This limitation could be a bug in the kernel -# Default number of bridges per SB/VM: -# unspecified or 0 --> will be set to @DEFBRIDGES@ -# > 1 <= 5 --> will be set to the specified number -# > 5 --> will be set to 5 -default_bridges = @DEFBRIDGES@ - -# Default memory size in MiB for SB/VM. -# If unspecified then it will be set @DEFMEMSZ@ MiB. -default_memory = @DEFMEMSZ@ -# -# Default memory slots per SB/VM. -# If unspecified then it will be set @DEFMEMSLOTS@. -# This is will determine the times that memory will be hotadded to sandbox/VM. -#memory_slots = @DEFMEMSLOTS@ - -# The size in MiB will be plused to max memory of hypervisor. -# It is the memory address space for the NVDIMM devie. -# If set block storage driver (block_device_driver) to "nvdimm", -# should set memory_offset to the size of block device. -# Default 0 -#memory_offset = 0 - -# Disable block device from being used for a container's rootfs. -# In case of a storage driver like devicemapper where a container's -# root file system is backed by a block device, the block device is passed -# directly to the hypervisor for performance reasons. -# This flag prevents the block device from being passed to the hypervisor, -# 9pfs is used instead to pass the rootfs. -disable_block_device_use = @DEFDISABLEBLOCK@ - -# Shared file system type: -# - virtio-9p (default) -# - virtio-fs -shared_fs = "@DEFSHAREDFS_NEMU@" - -# Path to vhost-user-fs daemon. -virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" - -# Default size of DAX cache in MiB -virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ - -# Cache mode: -# -# - none -# Metadata, data, and pathname lookup are not cached in guest. They are -# always fetched from host and any changes are immediately pushed to host. -# -# - auto -# Metadata and pathname lookup cache expires after a configured amount of -# time (default is 1 second). Data is cached while the file is open (close -# to open consistency). -# -# - always -# Metadata, data, and pathname lookup are cached in guest and never expire. -virtio_fs_cache = "@DEFVIRTIOFSCACHE@" - -# Extra args for virtiofsd daemon -# -# Format example: -# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] -# -# see `virtiofsd -h` for possible options. -virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ - -# Block storage driver to be used for the hypervisor in case the container -# rootfs is backed by a block device. This is virtio-scsi, virtio-blk -# or nvdimm. -block_device_driver = "@DEFBLOCKSTORAGEDRIVER_NEMU@" - -# Specifies cache-related options will be set to block devices or not. -# Default false -#block_device_cache_set = true - -# Specifies cache-related options for block devices. -# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. -# Default false -#block_device_cache_direct = true - -# Specifies cache-related options for block devices. -# Denotes whether flush requests for the device are ignored. -# Default false -#block_device_cache_noflush = true - -# Enable iothreads (data-plane) to be used. This causes IO to be -# handled in a separate IO thread. This is currently only implemented -# for SCSI. -# -enable_iothreads = @DEFENABLEIOTHREADS@ - -# Enable pre allocation of VM RAM, default false -# Enabling this will result in lower container density -# as all of the memory will be allocated and locked -# This is useful when you want to reserve all the memory -# upfront or in the cases where you want memory latencies -# to be very predictable -# Default false -#enable_mem_prealloc = true - -# Enable huge pages for VM RAM, default false -# Enabling this will result in the VM memory -# being allocated using huge pages. -# This is useful when you want to use vhost-user network -# stacks within the container. This will automatically -# result in memory pre allocation -enable_hugepages = @DEFENABLEHUGEPAGES_NEMU@ - -# Enable swap of vm memory. Default false. -# The behaviour is undefined if mem_prealloc is also set to true -#enable_swap = true - -# This option changes the default hypervisor and kernel parameters -# to enable debug output where available. This extra output is added -# to the proxy logs, but only when proxy debug is also enabled. -# -# Default false -#enable_debug = true - -# Disable the customizations done in the runtime when it detects -# that it is running on top a VMM. This will result in the runtime -# behaving as it would when running on bare metal. -# -#disable_nesting_checks = true - -# This is the msize used for 9p shares. It is the number of bytes -# used for 9p packet payload. -#msize_9p = @DEFMSIZE9P@ - -# If true and vsocks are supported, use vsocks to communicate directly -# with the agent (no proxy is started). -# Default true -#use_vsock = true - -# VFIO devices are hotplugged on a bridge by default. -# Enable hotplugging on root bus. This may be required for devices with -# a large PCI bar, as this is a current limitation with hotplugging on -# a bridge. This value is valid for "pc" machine type. -# Default false -#hotplug_vfio_on_root_bus = true - -# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. -# Default false -#disable_vhost_net = true -# -# Default entropy source. -# The path to a host source of entropy (including a real hardware RNG) -# /dev/urandom and /dev/random are two main options. -# Be aware that /dev/random is a blocking source of entropy. If the host -# runs out of entropy, the VMs boot time will increase leading to get startup -# timeouts. -# The source of entropy /dev/urandom is non-blocking and provides a -# generally acceptable source of entropy. It should work well for pretty much -# all practical purposes. -#entropy_source= "@DEFENTROPYSOURCE@" - -# Path to OCI hook binaries in the *guest rootfs*. -# This does not affect host-side hooks which must instead be added to -# the OCI spec passed to the runtime. -# -# You can create a rootfs with hooks by customizing the osbuilder scripts: -# https://github.com/kata-containers/osbuilder -# -# Hooks must be stored in a subdirectory of guest_hook_path according to their -# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}". -# The agent will scan these directories for executable files and add them, in -# lexicographical order, to the lifecycle of the guest container. -# Hooks are executed in the runtime namespace of the guest. See the official documentation: -# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks -# Warnings will be logged if any error is encountered will scanning for hooks, -# but it will not abort container execution. -#guest_hook_path = "/usr/share/oci/hooks" - -[factory] -# VM templating support. Once enabled, new VMs are created from template -# using vm cloning. They will share the same initial kernel, initramfs and -# agent memory by mapping it readonly. It helps speeding up new container -# creation and saves a lot of memory if there are many kata containers running -# on the same host. -# -# When disabled, new VMs are created from scratch. -# -# Note: Requires "initrd=" to be set ("image=" is not supported). -# -# Default false -#enable_template = true - -# Specifies the path of template. -# -# Default "/run/vc/vm/template" -#template_path = "/run/vc/vm/template" - -# The number of caches of VMCache: -# unspecified or == 0 --> VMCache is disabled -# > 0 --> will be set to the specified number -# -# VMCache is a function that creates VMs as caches before using it. -# It helps speed up new container creation. -# The function consists of a server and some clients communicating -# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. -# The VMCache server will create some VMs and cache them by factory cache. -# It will convert the VM to gRPC format and transport it when gets -# requestion from clients. -# Factory grpccache is the VMCache client. It will request gRPC format -# VM and convert it back to a VM. If VMCache function is enabled, -# kata-runtime will request VM from factory grpccache when it creates -# a new sandbox. -# -# Default 0 -#vm_cache_number = 0 - -# Specify the address of the Unix socket that is used by VMCache. -# -# Default /var/run/kata-containers/cache.sock -#vm_cache_endpoint = "/var/run/kata-containers/cache.sock" - -[proxy.@PROJECT_TYPE@] -path = "@PROXYPATH@" - -# If enabled, proxy messages will be sent to the system log -# (default: disabled) -#enable_debug = true - -[shim.@PROJECT_TYPE@] -path = "@SHIMPATH@" - -# If enabled, shim messages will be sent to the system log -# (default: disabled) -#enable_debug = true - -# If enabled, the shim will create opentracing.io traces and spans. -# (See https://www.jaegertracing.io/docs/getting-started). -# -# Note: By default, the shim runs in a separate network namespace. Therefore, -# to allow it to send trace details to the Jaeger agent running on the host, -# it is necessary to set 'disable_new_netns=true' so that it runs in the host -# network namespace. -# -# (default: disabled) -#enable_tracing = true - -[agent.@PROJECT_TYPE@] -# If enabled, make the agent display debug-level messages. -# (default: disabled) -#enable_debug = true - -# Enable agent tracing. -# -# If enabled, the default trace mode is "dynamic" and the -# default trace type is "isolated". The trace mode and type are set -# explicity with the `trace_type=` and `trace_mode=` options. -# -# Notes: -# -# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly -# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` -# will NOT activate agent tracing. -# -# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for -# full details. -# -# (default: disabled) -#enable_tracing = true -# -#trace_mode = "dynamic" -#trace_type = "isolated" - -# Comma separated list of kernel modules and their parameters. -# These modules will be loaded in the guest kernel using modprobe(8). -# The following example can be used to load two kernel modules with parameters -# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] -# The first word is considered as the module name and the rest as its parameters. -# Container will not be started when: -# * A kernel module is specified and the modprobe command is not installed in the guest -# or it fails loading the module. -# * The module is not available in the guest or it doesn't met the guest kernel -# requirements, like architecture and version. -# -kernel_modules=[] - -[netmon] -# If enabled, the network monitoring process gets started when the -# sandbox is created. This allows for the detection of some additional -# network being added to the existing network namespace, after the -# sandbox has been created. -# (default: disabled) -#enable_netmon = true - -# Specify the path to the netmon binary. -path = "@NETMONPATH@" - -# If enabled, netmon messages will be sent to the system log -# (default: disabled) -#enable_debug = true - -[runtime] -# If enabled, the runtime will log additional debug messages to the -# system log -# (default: disabled) -#enable_debug = true -# -# Internetworking model -# Determines how the VM should be connected to the -# the container network interface -# Options: -# -# - macvtap -# Used when the Container network interface can be bridged using -# macvtap. -# -# - none -# Used when customize network. Only creates a tap device. No veth pair. -# -# - tcfilter -# Uses tc filter rules to redirect traffic from the network interface -# provided by plugin to a tap interface connected to the VM. -# -internetworking_model="@DEFNETWORKMODEL_NEMU@" - -# disable guest seccomp -# Determines whether container seccomp profiles are passed to the virtual -# machine and applied by the kata agent. If set to true, seccomp is not applied -# within the guest -# (default: true) -disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ - -# If enabled, the runtime will create opentracing.io traces and spans. -# (See https://www.jaegertracing.io/docs/getting-started). -# (default: disabled) -#enable_tracing = true - -# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. -# This option may have some potential impacts to your host. It should only be used when you know what you're doing. -# `disable_new_netns` conflicts with `enable_netmon` -# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only -# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge -# (like OVS) directly. -# If you are using docker, `disable_new_netns` only works with `docker run --net=none` -# (default: false) -#disable_new_netns = true - -# if enable, the runtime use the parent cgroup of a container PodSandbox. This -# should be enabled for users where the caller setup the parent cgroup of the -# containers running in a sandbox so all the resouces of the kata container run -# in the same cgroup and performance isolation its more accurate. -sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ - -# Enabled experimental feature list, format: ["a", "b"]. -# Experimental features are features not stable enough for production, -# They may break compatibility, and are prepared for a big version bump. -# Supported experimental features: -# 1. "newstore": new persist storage driver which breaks backward compatibility, -# expected to move out of experimental in 2.0.0. -# (default: []) -experimental=@DEFAULTEXPFEATURES@ diff --git a/versions.yaml b/versions.yaml index a61a6a88e..428ee2bc6 100644 --- a/versions.yaml +++ b/versions.yaml @@ -78,28 +78,6 @@ assets: .*/v?(\d\S+)\.tar\.gz version: "v0.19.0" - nemu: - description: "Reduced-emulation VMM that uses KVM" - url: "https://github.com/intel/nemu" - uscan-url: >- - https://github.com/intel/nemu/tags - .*/release-?(\d\S+)\.tar\.gz - version: "release-2019-05-21" - - nemu-ovmf: - description: "OVMF firmware used by nemu VMM" - url: "https://github.com/intel/ovmf-virt" - uscan-url: >- - https://github.com/intel/ovmf-virt/tags - .*/?(\d\S+)\.tar\.gz - version: "0.6" - - qemu-lite: - description: "lightweight VMM that uses KVM" - url: "https://github.com/kata-containers/qemu" - branch: "qemu-lite-2.11.0" - commit: "87517afd726526e6e32a3e0be07eca34b8cc6962" - qemu: description: "VMM that uses KVM" url: "https://github.com/qemu/qemu" diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go index 4e3a62378..59570cc1e 100644 --- a/virtcontainers/types/sandbox.go +++ b/virtcontainers/types/sandbox.go @@ -170,7 +170,6 @@ func (v *Volumes) String() string { // VSock defines a virtio-socket to communicate between // the host and any process inside the VM. // This kind of socket is not supported in all hypervisors. -// QEMU and NEMU support it. type VSock struct { ContextID uint64 Port uint32 From 01713d59cb634b07720a11a12907399584dba5e1 Mon Sep 17 00:00:00 2001 From: Johan Kuijpers Date: Tue, 5 Nov 2019 15:52:59 +0100 Subject: [PATCH 20/23] runtime: added cloud hypervisor driver Initial release of cloud hypervisor driver for kata-runtime Fixes: #2046 Signed-off-by: Johan Kuijpers --- Makefile | 45 +- arch/amd64-options.mk | 3 + cli/config/configuration-clh.toml.in | 212 ++++ cli/kata-check_amd64.go | 4 + pkg/katautils/config.go | 90 ++ pkg/katautils/config_test.go | 60 ++ virtcontainers/clh.go | 1294 +++++++++++++++++++++++++ virtcontainers/hypervisor.go | 10 + virtcontainers/virtcontainers_test.go | 1 + 9 files changed, 1718 insertions(+), 1 deletion(-) create mode 100644 cli/config/configuration-clh.toml.in create mode 100644 virtcontainers/clh.go diff --git a/Makefile b/Makefile index 5baf7d5e1..a06549f9a 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ endif PREFIXDEPS := $(PREFIX) BINDIR := $(EXEC_PREFIX)/bin QEMUBINDIR := $(PREFIXDEPS)/bin +CLHBINDIR := $(PREFIXDEPS)/bin FCBINDIR := $(PREFIXDEPS)/bin ACRNBINDIR := $(PREFIXDEPS)/bin VIRTIOFSDBINDIR := $(PREFIXDEPS)/bin @@ -117,18 +118,21 @@ HYPERVISOR_ACRN = acrn HYPERVISOR_FC = firecracker JAILER_FC = jailer HYPERVISOR_QEMU = qemu +HYPERVISOR_CLH = cloud-hypervisor HYPERVISOR_QEMU_VIRTIOFS = qemu-virtiofs # Determines which hypervisor is specified in $(CONFIG_FILE). DEFAULT_HYPERVISOR = $(HYPERVISOR_QEMU) # List of hypervisors this build system can generate configuration for. -HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) +HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) $(HYPERVISOR_CLH) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) +CLHPATH := $(CLHBINDIR)/$(CLHCMD) + FCPATH = $(FCBINDIR)/$(FCCMD) FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD) @@ -258,6 +262,30 @@ ifneq (,$(QEMUVIRTIOFSCMD)) KERNELVIRTIOFSPATH = $(KERNELDIR)/$(KERNELNAMEVIRTIOFS) endif +ifneq (,$(CLHCMD)) + KNOWN_HYPERVISORS += $(HYPERVISOR_CLH) + + CONFIG_FILE_CLH = configuration-clh.toml + CONFIG_CLH = $(CLI_DIR)/config/$(CONFIG_FILE_CLH) + CONFIG_CLH_IN = $(CONFIG_CLH).in + + CONFIG_PATH_CLH = $(abspath $(CONFDIR)/$(CONFIG_FILE_CLH)) + CONFIG_PATHS += $(CONFIG_PATH_CLH) + + SYSCONFIG_CLH = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_CLH)) + SYSCONFIG_PATHS += $(SYSCONFIG_CLH) + + CONFIGS += $(CONFIG_CLH) + + # CLH-specific options (all should be suffixed by "_CLH") + # currently, huge pages are required for virtiofsd support + DEFENABLEHUGEPAGES_CLH := true + DEFNETWORKMODEL_CLH := tcfilter + KERNELTYPE_CLH = uncompressed + KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH)) + KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH) +endif + ifneq (,$(FCCMD)) KNOWN_HYPERVISORS += $(HYPERVISOR_FC) @@ -332,6 +360,10 @@ ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_ACRN)) DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_ACRN) endif +ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_CLH)) + DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_CLH) +endif + CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR) @@ -367,8 +399,10 @@ USER_VARS += KERNELDIR USER_VARS += KERNELTYPE USER_VARS += KERNELTYPE_FC USER_VARS += KERNELTYPE_ACRN +USER_VARS += KERNELTYPE_CLH USER_VARS += FIRMWAREPATH USER_VARS += MACHINEACCELERATORS +USER_VARS += DEFMACHINETYPE_CLH USER_VARS += KERNELPARAMS USER_VARS += LIBEXECDIR USER_VARS += LOCALSTATEDIR @@ -519,21 +553,26 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@CONFIG_ACRN_IN@|$(CONFIG_ACRN_IN)|g" \ -e "s|@CONFIG_QEMU_IN@|$(CONFIG_QEMU_IN)|g" \ -e "s|@CONFIG_QEMU_VIRTIOFS_IN@|$(CONFIG_QEMU_VIRTIOFS_IN)|g" \ + -e "s|@CONFIG_CLH_IN@|$(CONFIG_CLH_IN)|g" \ -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ -e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \ -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ + -e "s|@CLHPATH@|$(CLHPATH)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ -e "s|@KERNELPATH_FC@|$(KERNELPATH_FC)|g" \ + -e "s|@KERNELPATH_CLH@|$(KERNELPATH_CLH)|g" \ -e "s|@KERNELPATH@|$(KERNELPATH)|g" \ -e "s|@KERNELVIRTIOFSPATH@|$(KERNELVIRTIOFSPATH)|g" \ -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ -e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \ -e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \ + -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ + -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ @@ -557,6 +596,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFMEMSLOTS@|$(DEFMEMSLOTS)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ -e "s|@DEFNETWORKMODEL_ACRN@|$(DEFNETWORKMODEL_ACRN)|g" \ + -e "s|@DEFNETWORKMODEL_CLH@|$(DEFNETWORKMODEL_CLH)|g" \ -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ @@ -729,6 +769,9 @@ endif ifneq (,$(findstring $(HYPERVISOR_QEMU_VIRTIOFS),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_QEMU_VIRTIOFS) hypervisor path (QEMUVIRTIOFSPATH) : %s\n" $(abspath $(QEMUVIRTIOFSPATH)) endif +ifneq (,$(findstring $(HYPERVISOR_CLH),$(KNOWN_HYPERVISORS))) + @printf "\t$(HYPERVISOR_CLH) hypervisor path (CLHPATH) : %s\n" $(abspath $(CLHPATH)) +endif ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH)) endif diff --git a/arch/amd64-options.mk b/arch/amd64-options.mk index 797372f60..7bfc5ff97 100644 --- a/arch/amd64-options.mk +++ b/arch/amd64-options.mk @@ -22,3 +22,6 @@ FCJAILERCMD := jailer #ACRN binary name ACRNCMD := acrn-dm ACRNCTLCMD := acrnctl + +# cloud-hypervisor binary name +CLHCMD := cloud-hypervisor diff --git a/cli/config/configuration-clh.toml.in b/cli/config/configuration-clh.toml.in new file mode 100644 index 000000000..a63b6c875 --- /dev/null +++ b/cli/config/configuration-clh.toml.in @@ -0,0 +1,212 @@ +# Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "@CONFIG_CLH_IN@" +# XXX: Project: +# XXX: Name: @PROJECT_NAME@ +# XXX: Type: @PROJECT_TYPE@ + +[hypervisor.clh] +path = "@CLHPATH@" +kernel = "@KERNELPATH_CLH@" +image = "@IMAGEPATH@" + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "@KERNELPARAMS@" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to @DEFVCPUS@ +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS@ + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set @DEFMEMSZ@ MiB. +default_memory = @DEFMEMSZ@ + +# Default memory slots per SB/VM. +# If unspecified then it will be set @DEFMEMSLOTS@. +# This is will determine the times that memory will be hotadded to sandbox/VM. +#memory_slots = @DEFMEMSLOTS@ + +# Path to vhost-user-fs daemon. +virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" + +# cloud-hypervisor prefers virtiofs caching (dax) for performance reasons +virtio_fs_cache = "always" + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. This extra output is added +# to the proxy logs, but only when proxy debug is also enabled. +# +# Default false +# enable_debug = true + +[proxy.@PROJECT_TYPE@] +path = "@PROXYPATH@" + +# If enabled, proxy messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +[shim.@PROJECT_TYPE@] +path = "@SHIMPATH@" + +# If enabled, shim messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +# If enabled, the shim will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# +# Note: By default, the shim runs in a separate network namespace. Therefore, +# to allow it to send trace details to the Jaeger agent running on the host, +# it is necessary to set 'disable_new_netns=true' so that it runs in the host +# network namespace. +# +# (default: disabled) +#enable_tracing = true + + +[agent.@PROJECT_TYPE@] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +#enable_debug = true + +# Enable agent tracing. +# +# If enabled, the default trace mode is "dynamic" and the +# default trace type is "isolated". The trace mode and type are set +# explicity with the `trace_type=` and `trace_mode=` options. +# +# Notes: +# +# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly +# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` +# will NOT activate agent tracing. +# +# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for +# full details. +# +# (default: disabled) +#enable_tracing = true +# +#trace_mode = "dynamic" +#trace_type = "isolated" + + +[netmon] +# If enabled, the network monitoring process gets started when the +# sandbox is created. This allows for the detection of some additional +# network being added to the existing network namespace, after the +# sandbox has been created. +# (default: disabled) +#enable_netmon = true + +# Specify the path to the netmon binary. +path = "@NETMONPATH@" + +# If enabled, netmon messages will be sent to the system log +# (default: disabled) +#enable_debug = true + + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +#enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - bridged (Deprecated) +# Uses a linux bridge to interconnect the container interface to +# the VM. Works for most cases except macvlan and ipvlan. +# ***NOTE: This feature has been deprecated with plans to remove this +# feature in the future. Please use other network models listed below. +# +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="@DEFNETWORKMODEL_CLH@" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `enable_netmon` +# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# If you are using docker, `disable_new_netns` only works with `docker run --net=none` +# (default: false) +#disable_new_netns = true + +# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. +# The container cgroups in the host are not created, just one single cgroup per sandbox. +# The sandbox cgroup is not constrained by the runtime +# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. +# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType +sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# They may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# 1. "newstore": new persist storage driver which breaks backward compatibility, +# expected to move out of experimental in 2.0.0. +# (default: []) +experimental=@DEFAULTEXPFEATURES@ diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go index 3962e09e8..1464b9090 100644 --- a/cli/kata-check_amd64.go +++ b/cli/kata-check_amd64.go @@ -107,6 +107,8 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { switch hypervisorType { case "firecracker": fallthrough + case "clh": + fallthrough case "qemu": archRequiredCPUFlags = map[string]string{ cpuFlagVMX: "Virtualization support", @@ -291,6 +293,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { switch hypervisorType { case "qemu": fallthrough + case "clh": + fallthrough case "firecracker": return kvmIsUsable() case "acrn": diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 386f9df5a..153f0fa98 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -49,6 +49,7 @@ var ( const ( // supported hypervisor component types firecrackerHypervisorTableType = "firecracker" + clhHypervisorTableType = "clh" qemuHypervisorTableType = "qemu" acrnHypervisorTableType = "acrn" @@ -701,6 +702,92 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + hypervisor, err := h.path() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err + } + + initrd, image, err := h.getInitrdAndImage() + if err != nil { + return vc.HypervisorConfig{}, err + } + + if initrd != "" { + return vc.HypervisorConfig{}, + errors.New("having an initrd defined in the configuration file is not supported") + } + + if image == "" { + return vc.HypervisorConfig{}, + errors.New("image must be defined in the configuration file") + } + + firmware, err := h.firmware() + if err != nil { + return vc.HypervisorConfig{}, err + } + + machineAccelerators := h.machineAccelerators() + kernelParams := h.kernelParams() + machineType := h.machineType() + + blockDriver, err := h.blockDeviceDriver() + if err != nil { + return vc.HypervisorConfig{}, err + } + + sharedFS := config.VirtioFS + + if h.VirtioFSDaemon == "" { + return vc.HypervisorConfig{}, + errors.New("virtio-fs daemon path is missing in configuration file") + } + + return vc.HypervisorConfig{ + HypervisorPath: hypervisor, + KernelPath: kernel, + InitrdPath: initrd, + ImagePath: image, + FirmwarePath: firmware, + MachineAccelerators: machineAccelerators, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + HypervisorMachineType: machineType, + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + MemOffset: h.defaultMemOffset(), + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), + DisableBlockDeviceUse: h.DisableBlockDeviceUse, + SharedFS: sharedFS, + VirtioFSDaemon: h.VirtioFSDaemon, + VirtioFSCacheSize: h.VirtioFSCacheSize, + VirtioFSCache: h.VirtioFSCache, + MemPrealloc: h.MemPrealloc, + HugePages: h.HugePages, + FileBackedMemRootDir: h.FileBackedMemRootDir, + Mlock: !h.Swap, + Debug: h.Debug, + DisableNestingChecks: h.DisableNestingChecks, + BlockDeviceDriver: blockDriver, + BlockDeviceCacheSet: h.BlockDeviceCacheSet, + BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, + BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, + EnableIOThreads: h.EnableIOThreads, + Msize9p: h.msize9p(), + HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, + DisableVhostNet: true, + UseVSock: true, + }, nil +} + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { if f.TemplatePath == "" { f.TemplatePath = defaultTemplatePath @@ -744,6 +831,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case acrnHypervisorTableType: config.HypervisorType = vc.AcrnHypervisor hConfig, err = newAcrnHypervisorConfig(hypervisor) + case clhHypervisorTableType: + config.HypervisorType = vc.ClhHypervisor + hConfig, err = newClhHypervisorConfig(hypervisor) } if err != nil { diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index 3ad78a6bc..edf41826c 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -885,6 +885,66 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) { assert.Error(err) } +func TestNewClhHypervisorConfig(t *testing.T) { + + assert := assert.New(t) + + tmpdir, err := ioutil.TempDir(testDir, "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + hypervisorPath := path.Join(tmpdir, "hypervisor") + kernelPath := path.Join(tmpdir, "kernel") + imagePath := path.Join(tmpdir, "image") + virtioFsDaemon := path.Join(tmpdir, "virtiofsd") + + for _, file := range []string{imagePath, hypervisorPath, kernelPath, virtioFsDaemon} { + err = createEmptyFile(file) + assert.NoError(err) + } + + hypervisor := hypervisor{ + Path: hypervisorPath, + Kernel: kernelPath, + Image: imagePath, + VirtioFSDaemon: virtioFsDaemon, + VirtioFSCache: "always", + } + config, err := newClhHypervisorConfig(hypervisor) + if err != nil { + t.Fatal(err) + } + + if config.HypervisorPath != hypervisor.Path { + t.Errorf("Expected hypervisor path %v, got %v", hypervisor.Path, config.HypervisorPath) + } + + if config.KernelPath != hypervisor.Kernel { + t.Errorf("Expected kernel path %v, got %v", hypervisor.Kernel, config.KernelPath) + } + + if config.ImagePath != hypervisor.Image { + t.Errorf("Expected image path %v, got %v", hypervisor.Image, config.ImagePath) + } + + if config.ImagePath != hypervisor.Image { + t.Errorf("Expected image path %v, got %v", hypervisor.Image, config.ImagePath) + } + + if config.UseVSock != true { + t.Errorf("Expected UseVSock %v, got %v", true, config.UseVSock) + } + + if config.DisableVhostNet != true { + t.Errorf("Expected DisableVhostNet %v, got %v", true, config.DisableVhostNet) + } + + if config.VirtioFSCache != "always" { + t.Errorf("Expected VirtioFSCache %v, got %v", true, config.VirtioFSCache) + } + +} + func TestNewShimConfig(t *testing.T) { dir, err := ioutil.TempDir(testDir, "shim-config-") if err != nil { diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go new file mode 100644 index 000000000..2e05418c0 --- /dev/null +++ b/virtcontainers/clh.go @@ -0,0 +1,1294 @@ +// Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + opentracing "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +// +// Constants and type definitions related to cloud hypervisor +// + +type clhState uint8 + +const ( + clhNotReady clhState = iota + clhReady +) + +const ( + clhTimeout = 10 + clhSocket = "clh.sock" + clhAPISocket = "clh-api.sock" + clhLogFile = "clh.log" + virtioFsSocket = "virtiofsd.sock" + clhSerial = "serial-tty.log" + supportedMajorVersion = 0 + supportedMinorVersion = 3 + defaultClhPath = "/usr/local/bin/cloud-hypervisor" + virtioFsCacheAlways = "always" +) + +type CloudHypervisorVersion struct { + Major int + Minor int + Revision int +} + +// +// Cloud hypervisor state +// +type CloudHypervisorState struct { + state clhState + PID int + VirtiofsdPID int +} + +func (s *CloudHypervisorState) reset() { + s.PID = 0 + s.VirtiofsdPID = 0 + s.state = clhNotReady +} + +type cloudHypervisor struct { + id string + state CloudHypervisorState + store *store.VCStore + config HypervisorConfig + ctx context.Context + socketPath string + version CloudHypervisorVersion + cliBuilder *DefaultCLIBuilder +} + +var clhKernelParams = []Param{ + + {"root", "/dev/vda1"}, + {"panic", "1"}, // upon kernel panic wait 1 second before reboot + {"no_timer_check", ""}, // do not check broken timer IRQ resources + {"noreplace-smp", ""}, // do not replace SMP instructions + {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, // tell the agent where to send the logs +} + +var clhDebugKernelParams = []Param{ + + {"console", "ttyS0,115200n8"}, // enable serial console + {"systemd.log_level", "debug"}, // enable systemd debug output + {"systemd.log_target", "console"}, // send loggng to the console + {"initcall_debug", "1"}, // print init call timing information to the console +} + +//########################################################### +// +// hypervisor interface implementation for cloud-hypervisor +// +//########################################################### + +func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { + clh.ctx = ctx + + span, _ := clh.trace("createSandbox") + defer span.Finish() + + err := hypervisorConfig.valid() + if err != nil { + return err + } + + clh.id = id + clh.store = vcStore + clh.config = *hypervisorConfig + clh.state.state = clhNotReady + + err = clh.getAvailableVersion() + if err != nil { + return err + } + + if clh.version.Major < supportedMajorVersion && clh.version.Minor < supportedMinorVersion { + errorMessage := fmt.Sprintf("Unsupported version: cloud-hypervisor %d.%d not supported by this driver version (%d.%d)", + clh.version.Major, + clh.version.Minor, + supportedMajorVersion, + supportedMinorVersion) + return errors.New(errorMessage) + } + + clh.cliBuilder = &DefaultCLIBuilder{} + + socketPath, err := clh.vsockSocketPath(id) + if err != nil { + clh.Logger().Info("Invalid socket path for cloud-hypervisor") + return nil + } + clh.socketPath = socketPath + + clh.Logger().WithField("function", "createSandbox").Info("creating Sandbox") + + // No need to return an error from there since there might be nothing + // to fetch if this is the first time the hypervisor is created. + if err := clh.store.Load(store.Hypervisor, &clh.state); err != nil { + clh.Logger().WithField("function", "createSandbox").WithError(err).Info("No info could be fetched") + } + + // Set initial memomory size of the cloud hypervisor + clh.cliBuilder.SetMemory(&CLIMemory{ + memorySize: clh.config.MemorySize, + backingFile: "/dev/shm", + }) + // Set initial amount of cpu's for the cloud hypervisor + clh.cliBuilder.SetCpus(&CLICpus{ + cpus: clh.config.NumVCPUs, + }) + + // Add the kernel path + kernelPath, err := clh.config.KernelAssetPath() + if err != nil { + return err + } + clh.cliBuilder.SetKernel(&CLIKernel{ + path: kernelPath, + }) + + // First take the default parameters defined by this driver + clh.cliBuilder.AddKernelParameters(clhKernelParams) + + // Followed by extra debug parameters if debug enabled in configuration file + if clh.config.Debug { + clh.cliBuilder.AddKernelParameters(clhDebugKernelParams) + } + + // Followed by extra debug parameters defined in the configuration file + clh.cliBuilder.AddKernelParameters(clh.config.KernelParams) + + // set random device generator to hypervisor + clh.cliBuilder.SetRng(&CLIRng{ + src: clh.config.EntropySource, + iommu: false, + }) + + // Add the hybrid vsock device to hypervisor + clh.cliBuilder.SetVsock(&CLIVsock{ + cid: 3, + socketPath: clh.socketPath, + iommu: false, + }) + + // set the initial root/boot disk of hypervisor + imagePath, err := clh.config.ImageAssetPath() + if err != nil { + return err + } + + if imagePath != "" { + clh.cliBuilder.SetDisk(&CLIDisk{ + path: imagePath, + iommu: false, + }) + } + + // set the virtio-fs to the hypervisor + vfsdSockPath, err := clh.virtioFsSocketPath(clh.id) + if err != nil { + return err + } + if clh.config.VirtioFSCache == virtioFsCacheAlways { + clh.cliBuilder.SetFs(&CLIFs{ + tag: "kataShared", + socketPath: vfsdSockPath, + queues: 1, + queueSize: 512, + dax: true, + }) + } else { + clh.cliBuilder.SetFs(&CLIFs{ + tag: "kataShared", + socketPath: vfsdSockPath, + queues: 1, + queueSize: 512, + }) + } + + // set the serial console to the cloud hypervisor + if clh.config.Debug { + serialPath, err := clh.serialPath(clh.id) + if err != nil { + return err + } + clh.cliBuilder.SetSerial(&CLISerialConsole{ + consoleType: cctFILE, + filePath: serialPath, + }) + logFilePath, err := clh.logFilePath(clh.id) + if err != nil { + return err + } + clh.cliBuilder.SetLogFile(&CLILogFile{ + path: logFilePath, + }) + } + + clh.cliBuilder.SetConsole(&CLIConsole{ + consoleType: cctOFF, + }) + + // Move the API endpoint socket location for the + // by default enabled api endpoint + apiSocketPath, err := clh.apiSocketPath(id) + if err != nil { + clh.Logger().Info("Invalid api socket path for cloud-hypervisor") + return nil + } + clh.cliBuilder.SetAPISocket(&CLIAPISocket{ + socketPath: apiSocketPath, + }) + + return nil +} + +func (clh *cloudHypervisor) startSandbox(timeout int) error { + span, _ := clh.trace("startSandbox") + defer span.Finish() + + clh.Logger().WithField("function", "startSandbox").Info("starting Sandbox") + + vmPath := filepath.Join(store.RunVMStoragePath(), clh.id) + err := os.MkdirAll(vmPath, store.DirMode) + if err != nil { + return err + } + + if clh.config.SharedFS == config.VirtioFS { + clh.Logger().WithField("function", "startSandbox").Info("Starting virtiofsd") + _, err = clh.setupVirtiofsd(timeout) + if err != nil { + return err + } + if err = clh.storeState(); err != nil { + return err + } + } else { + return errors.New("cloud-hypervisor only supports virtio based file sharing") + } + + var strErr string + strErr, pid, err := clh.LaunchClh() + if err != nil { + return fmt.Errorf("failed to launch cloud-hypervisor: %s, error messages from log: %s", err, strErr) + } + if err := clh.waitVMM(clhTimeout); err != nil { + clh.Logger().WithField("error", err).Warn("cloud-hypervisor init failed") + clh.shutdownVirtiofsd() + return err + } + + clh.state.PID = pid + clh.state.state = clhReady + clh.storeState() + + return nil +} + +func (clh *cloudHypervisor) getSandboxConsole(id string) (string, error) { + clh.Logger().WithField("function", "getSandboxConsole").WithField("id", id).Info("Get Sandbox Console") + return "", nil +} + +func (clh *cloudHypervisor) disconnect() { + clh.Logger().WithField("function", "disconnect").Info("Disconnecting Sandbox Console") +} + +func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) { + + clh.Logger().WithField("function", "getThreadIDs").Info("get thread ID's") + + var vcpuInfo vcpuThreadIDs + + vcpuInfo.vcpus = make(map[int]int) + + return vcpuInfo, nil +} + +func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + clh.Logger().WithField("function", "hotplugAddDevice").Warn("hotplug add device not supported") + return nil, nil +} + +func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + clh.Logger().WithField("function", "hotplugRemoveDevice").Warn("hotplug remove device not supported") + return nil, nil +} + +func (clh *cloudHypervisor) hypervisorConfig() HypervisorConfig { + return clh.config +} + +func (clh *cloudHypervisor) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + clh.Logger().WithField("function", "resizeMemory").Warn("not supported") + return 0, memoryDevice{}, nil +} + +func (clh *cloudHypervisor) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { + clh.Logger().WithField("function", "resizeVCPUs").Warn("not supported") + return 0, 0, nil +} + +func (clh *cloudHypervisor) cleanup() error { + clh.Logger().WithField("function", "cleanup").Info("cleanup") + return nil +} + +func (clh *cloudHypervisor) pauseSandbox() error { + clh.Logger().WithField("function", "pauseSandbox").Info("Pause Sandbox") + return nil +} + +func (clh *cloudHypervisor) saveSandbox() error { + clh.Logger().WithField("function", "saveSandboxC").Info("Save Sandbox") + return nil +} + +func (clh *cloudHypervisor) resumeSandbox() error { + clh.Logger().WithField("function", "resumeSandbox").Info("Resume Sandbox") + return nil +} + +// stopSandbox will stop the Sandbox's VM. +func (clh *cloudHypervisor) stopSandbox() (err error) { + span, _ := clh.trace("stopSandbox") + defer span.Finish() + clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox") + return clh.terminate() +} + +func (clh *cloudHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("cloudHypervisor is not supported by VM cache") +} + +func (clh *cloudHypervisor) toGrpc() ([]byte, error) { + return nil, errors.New("cloudHypervisor is not supported by VM cache") +} + +func (clh *cloudHypervisor) save() (s persistapi.HypervisorState) { + s.Pid = clh.state.PID + s.Type = string(ClhHypervisor) + return +} + +func (clh *cloudHypervisor) load(s persistapi.HypervisorState) { + clh.state.PID = s.Pid + clh.state.VirtiofsdPID = s.VirtiofsdPid +} + +func (clh *cloudHypervisor) check() error { + return nil +} + +func (clh *cloudHypervisor) getPids() []int { + + var pids []int + pids = append(pids, clh.state.PID) + + return pids +} + +//########################################################################### +// +// Local helper methods related to the hypervisor interface implementation +// +//########################################################################### + +func (clh *cloudHypervisor) addDevice(devInfo interface{}, devType deviceType) error { + span, _ := clh.trace("addDevice") + defer span.Finish() + + var err error + + switch v := devInfo.(type) { + case Endpoint: + clh.Logger().WithField("function", "addDevice").Infof("Adding Endpoint of type %v", v) + clh.cliBuilder.AddNet(CLINet{ + device: v.Name(), + mac: v.HardwareAddr(), + }) + + default: + clh.Logger().WithField("function", "addDevice").Warnf("Add device of type %v is not supported.", v) + } + + return err +} + +func (clh *cloudHypervisor) Logger() *log.Entry { + return virtLog.WithField("subsystem", "cloudHypervisor") +} + +func (clh *cloudHypervisor) capabilities() types.Capabilities { + span, _ := clh.trace("capabilities") + defer span.Finish() + + clh.Logger().WithField("function", "capabilities").Info("get Capabilities") + var caps types.Capabilities + return caps + +} + +func (clh *cloudHypervisor) trace(name string) (opentracing.Span, context.Context) { + + if clh.ctx == nil { + clh.Logger().WithField("type", "bug").Error("trace called before context set") + clh.ctx = context.Background() + } + + span, ctx := opentracing.StartSpanFromContext(clh.ctx, name) + + span.SetTag("subsystem", "cloudHypervisor") + span.SetTag("type", "clh") + + return span, ctx +} + +func (clh *cloudHypervisor) terminate() (err error) { + span, _ := clh.trace("terminate") + defer span.Finish() + + defer func() { + if err != nil { + clh.Logger().Info("Terminate Cloud Hypervisor failed") + } else { + clh.Logger().Info("Cloud Hypervisor stopped") + clh.reset() + clh.Logger().Debug("removing virtiofsd and vm sockets") + path, err := clh.virtioFsSocketPath(clh.id) + if err == nil { + rerr := os.Remove(path) + if rerr != nil { + clh.Logger().WithField("path", path).Warn("removing virtiofsd socket failed") + } + } + path, err = clh.vsockSocketPath(clh.id) + if err == nil { + rerr := os.Remove(path) + if rerr != nil { + clh.Logger().WithField("path", path).Warn("removing vm socket failed") + } + } + } + }() + + pid := clh.state.PID + if pid == 0 { + clh.Logger().WithField("PID", pid).Info("Skipping kill cloud hypervisor. invalid pid") + return nil + } + clh.Logger().WithField("PID", pid).Info("Stopping Cloud Hypervisor") + + // Check if VM process is running, in case it is not, let's + // return from here. + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + return nil + } + + // Send a SIGTERM to the VM process to try to stop it properly + if err = syscall.Kill(pid, syscall.SIGTERM); err != nil { + return err + } + + // Wait for the VM process to terminate + tInit := time.Now() + for { + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + return nil + } + + if time.Since(tInit).Seconds() >= fcStopSandboxTimeout { + clh.Logger().Warnf("VM still running after waiting %ds", fcStopSandboxTimeout) + break + } + + // Let's avoid to run a too busy loop + time.Sleep(time.Duration(50) * time.Millisecond) + } + + // Let's try with a hammer now, a SIGKILL should get rid of the + // VM process. + return syscall.Kill(pid, syscall.SIGKILL) +} + +func (clh *cloudHypervisor) reset() { + clh.state.reset() + clh.storeState() +} + +func (clh *cloudHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) { + if !useVsock { + return nil, fmt.Errorf("Can't generate socket path for cloud-hypervisor: vsocks is disabled") + } + + udsPath, err := clh.vsockSocketPath(id) + if err != nil { + clh.Logger().Info("Can't generate socket path for cloud-hypervisor") + return types.HybridVSock{}, err + } + clh.Logger().WithField("function", "generateSocket").Infof("Using hybrid vsock %s:%d", udsPath, vSockPort) + clh.socketPath = udsPath + return types.HybridVSock{ + UdsPath: udsPath, + Port: uint32(vSockPort), + }, nil +} + +func (clh *cloudHypervisor) setupVirtiofsd(timeout int) (remain int, err error) { + + sockPath, perr := clh.virtioFsSocketPath(clh.id) + if perr != nil { + return 0, perr + } + + theArgs, err := clh.virtiofsdArgs(sockPath) + if err != nil { + return 0, err + } + + clh.Logger().WithField("path", clh.config.VirtioFSDaemon).Info() + clh.Logger().WithField("args", strings.Join(theArgs, " ")).Info() + + cmd := exec.Command(clh.config.VirtioFSDaemon, theArgs...) + stderr, err := cmd.StderrPipe() + if err != nil { + return 0, err + } + + if err = cmd.Start(); err != nil { + return 0, err + } + defer func() { + if err != nil { + clh.state.VirtiofsdPID = 0 + cmd.Process.Kill() + } else { + clh.state.VirtiofsdPID = cmd.Process.Pid + + } + clh.storeState() + }() + + // Wait for socket to become available + sockReady := make(chan error, 1) + timeStart := time.Now() + go func() { + scanner := bufio.NewScanner(stderr) + var sent bool + for scanner.Scan() { + if clh.config.Debug { + clh.Logger().WithField("source", "virtiofsd").Debug(scanner.Text()) + } + if !sent && strings.Contains(scanner.Text(), "Waiting for vhost-user socket connection...") { + sockReady <- nil + sent = true + } + } + if !sent { + if err := scanner.Err(); err != nil { + sockReady <- err + } else { + sockReady <- fmt.Errorf("virtiofsd did not announce socket connection") + } + } + clh.Logger().Info("virtiofsd quits") + // Wait to release resources of virtiofsd process + cmd.Process.Wait() + + }() + + return clh.waitVirtiofsd(timeStart, timeout, sockReady, + fmt.Sprintf("virtiofsd (pid=%d) socket %s", cmd.Process.Pid, sockPath)) +} + +func (clh *cloudHypervisor) waitVirtiofsd(start time.Time, timeout int, ready chan error, errMsg string) (int, error) { + var err error + + timeoutDuration := time.Duration(timeout) * time.Second + select { + case err = <-ready: + case <-time.After(timeoutDuration): + err = fmt.Errorf("timed out waiting for %s", errMsg) + } + if err != nil { + return 0, err + } + + // Now reduce timeout by the elapsed time + elapsed := time.Since(start) + if elapsed < timeoutDuration { + timeout = timeout - int(elapsed.Seconds()) + } else { + timeout = 0 + } + return timeout, nil +} + +func (clh *cloudHypervisor) virtiofsdArgs(sockPath string) ([]string, error) { + + sourcePath := filepath.Join(kataHostSharedDir(), clh.id) + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + os.MkdirAll(sourcePath, os.ModePerm) + } + + args := []string{ + "-f", + "-o", "vhost_user_socket=" + sockPath, + "-o", "source=" + sourcePath, + "-o", "cache=" + clh.config.VirtioFSCache} + + if len(clh.config.VirtioFSExtraArgs) != 0 { + args = append(args, clh.config.VirtioFSExtraArgs...) + } + return args, nil +} + +func (clh *cloudHypervisor) shutdownVirtiofsd() (err error) { + + err = syscall.Kill(-clh.state.VirtiofsdPID, syscall.SIGKILL) + + if err != nil { + clh.state.VirtiofsdPID = 0 + clh.storeState() + } + return err + +} + +func (clh *cloudHypervisor) virtioFsSocketPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, virtioFsSocket) +} + +func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhSocket) +} + +func (clh *cloudHypervisor) serialPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhSerial) +} + +func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhAPISocket) +} + +func (clh *cloudHypervisor) logFilePath(id string) (string, error) { + return utils.BuildSocketPath(store.RunVMStoragePath(), id, clhLogFile) +} + +func (clh *cloudHypervisor) storeState() error { + if clh.store != nil { + if err := clh.store.Store(store.Hypervisor, clh.state); err != nil { + return err + } + } + return nil +} + +func (clh *cloudHypervisor) waitVMM(timeout int) error { + + var err error + timeoutDuration := time.Duration(timeout) * time.Second + + sockReady := make(chan error, 1) + go func() { + udsPath, err := clh.vsockSocketPath(clh.id) + if err != nil { + sockReady <- err + } + + for { + addr, err := net.ResolveUnixAddr("unix", udsPath) + if err != nil { + sockReady <- err + } + conn, err := net.DialUnix("unix", nil, addr) + + if err != nil { + time.Sleep(50 * time.Millisecond) + } else { + conn.Close() + sockReady <- nil + + break + } + } + }() + + select { + case err = <-sockReady: + case <-time.After(timeoutDuration): + err = fmt.Errorf("timed out waiting for cloud-hypervisor vsock") + } + + time.Sleep(1000 * time.Millisecond) + return err +} + +func (clh *cloudHypervisor) clhPath() (string, error) { + p, err := clh.config.HypervisorAssetPath() + if err != nil { + return "", err + } + + if p == "" { + p = defaultClhPath + } + + if _, err = os.Stat(p); os.IsNotExist(err) { + return "", fmt.Errorf("Cloud-Hypervisor path (%s) does not exist", p) + } + + return p, nil +} + +func (clh *cloudHypervisor) getAvailableVersion() error { + + clhPath, err := clh.clhPath() + if err != nil { + return err + } + + cmd := exec.Command(clhPath, "--version") + out, err := cmd.CombinedOutput() + if err != nil { + return err + } + + words := strings.Fields(string(out)) + if len(words) != 2 { + return errors.New("Failed to parse cloud-hypervisor version response. Illegal length") + } + versionSplit := strings.SplitN(words[1], ".", -1) + if len(versionSplit) != 3 { + return errors.New("Failed to parse cloud-hypervisor version field. Illegal length") + } + + major, err := strconv.ParseUint(versionSplit[0], 10, 64) + if err != nil { + return err + } + minor, err := strconv.ParseUint(versionSplit[1], 10, 64) + if err != nil { + return err + } + revision, err := strconv.ParseUint(versionSplit[2], 10, 64) + if err != nil { + return err + } + clh.version = CloudHypervisorVersion{ + Major: int(major), + Minor: int(minor), + Revision: int(revision), + } + return nil +} + +func (clh *cloudHypervisor) LaunchClh() (string, int, error) { + + errStr := "" + + clhPath, err := clh.clhPath() + if err != nil { + return "", -1, err + } + director := &CommandLineDirector{} + + cli, err := director.Build(clh.cliBuilder) + if err != nil { + return "", -1, err + } + + clh.Logger().WithField("path", clhPath).Info() + clh.Logger().WithField("args", strings.Join(cli.args, " ")).Info() + + cmd := exec.Command(clhPath, cli.args...) + cmd.Stderr = ioutil.Discard + + if clh.config.Debug { + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "RUST_BACKTRACE=FULL") + } + + if err := cmd.Start(); err != nil { + fmt.Println("Error starting cloudHypervisor", err) + if cmd.Process != nil { + cmd.Process.Kill() + } + return errStr, 0, err + } + + return errStr, cmd.Process.Pid, nil +} + +//########################################################################### +// +// Cloud-hypervisor CLI builder +// +//########################################################################### + +const ( + cctOFF string = "off" + cctFILE string = "file" +) + +const ( + cscApisocket string = "--api-socket" + cscCmdline string = "--cmdline" + cscConsole string = "--console" + cscCpus string = "--cpus" + cscDisk string = "--disk" + cscFs string = "--fs" + cscKernel string = "--kernel" + cscLogFile string = "--log-file" + cscMemory string = "--memory" + cscNet string = "--net" + cscRng string = "--rng" + cscSerial string = "--serial" + cscVsock string = "--vsock" +) + +type CommandLineBuilder interface { + AddKernelParameters(cmdline []Param) + SetConsole(console *CLIConsole) + SetCpus(cpus *CLICpus) + SetDisk(disk *CLIDisk) + SetFs(fs *CLIFs) + SetKernel(kernel *CLIKernel) + SetMemory(memory *CLIMemory) + AddNet(net CLINet) + SetRng(rng *CLIRng) + SetSerial(serial *CLISerialConsole) + SetVsock(vsock *CLIVsock) + SetAPISocket(apiSocket *CLIAPISocket) + SetLogFile(logFile *CLILogFile) + GetCommandLine() (*CommandLine, error) +} + +type CLIOption interface { + Build(cmdline *CommandLine) +} + +type CommandLine struct { + args []string +} + +//********************************** +// The (virtio) Console +//********************************** +type CLIConsole struct { + consoleType string + filePath string + iommu bool +} + +func (o *CLIConsole) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscConsole) + + consoleArg := "" + if o.consoleType == cctFILE { + consoleArg = o.consoleType + "=" + o.filePath + if o.iommu { + consoleArg += ",iommu=on" + } else { + consoleArg += ",iommu=off" + } + } else { + consoleArg = o.consoleType + } + + cmdline.args = append(cmdline.args, consoleArg) +} + +//********************************** +// The serial port +//********************************** +type CLISerialConsole struct { + consoleType string + filePath string +} + +func (o *CLISerialConsole) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscSerial) + if o.consoleType == cctFILE { + cmdline.args = append(cmdline.args, o.consoleType+"="+o.filePath) + } else { + cmdline.args = append(cmdline.args, o.consoleType) + } + +} + +//********************************** +// The API socket +//********************************** +type CLIAPISocket struct { + socketPath string +} + +func (o *CLIAPISocket) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscApisocket) + if o.socketPath != "" { + cmdline.args = append(cmdline.args, o.socketPath) + } +} + +//********************************** +// The amount of memory in Mb +//********************************** +type CLIMemory struct { + memorySize uint32 + backingFile string +} + +func (o *CLIMemory) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscMemory) + if o.backingFile == "" { + cmdline.args = append(cmdline.args, "size="+strconv.FormatUint(uint64(o.memorySize), 10)+"M") + } else { + cmdline.args = append(cmdline.args, "size="+strconv.FormatUint(uint64(o.memorySize), 10)+"M,file="+o.backingFile) + } + +} + +//********************************** +// The number of CPU's +//********************************** +type CLICpus struct { + cpus uint32 +} + +func (o *CLICpus) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscCpus) + cmdline.args = append(cmdline.args, strconv.FormatUint(uint64(o.cpus), 10)) + +} + +//********************************** +// The Path to the kernel image +//********************************** +type CLIKernel struct { + path string +} + +func (o *CLIKernel) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscKernel) + cmdline.args = append(cmdline.args, o.path) + +} + +//**************************************** +// The Path to the root (boot) disk image +//**************************************** +type CLIDisk struct { + path string + iommu bool +} + +func (o *CLIDisk) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscDisk) + if o.iommu { + cmdline.args = append(cmdline.args, "path="+o.path+",iommu=on") + } else { + cmdline.args = append(cmdline.args, "path="+o.path+",iommu=off") + } + +} + +//**************************************** +// The random device +//**************************************** +type CLIRng struct { + src string + iommu bool +} + +func (o *CLIRng) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscRng) + if o.iommu { + cmdline.args = append(cmdline.args, "src="+o.src+",iommu=on") + } else { + cmdline.args = append(cmdline.args, "src="+o.src+",iommu=off") + } + +} + +//**************************************** +// The VSock socket +//**************************************** +type CLIVsock struct { + socketPath string + cid uint32 + iommu bool +} + +func (o *CLIVsock) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscVsock) + if o.iommu { + cmdline.args = append(cmdline.args, "cid="+strconv.FormatUint(uint64(o.cid), 10)+",sock="+o.socketPath+",iommu=on") + } else { + cmdline.args = append(cmdline.args, "cid="+strconv.FormatUint(uint64(o.cid), 10)+",sock="+o.socketPath+",iommu=off") + } +} + +//**************************************** +// The shard (virtio) file system +//**************************************** +type CLIFs struct { + tag string + socketPath string + queues uint32 + queueSize uint32 + dax bool +} + +func (o *CLIFs) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscFs) + + fsarg := "tag=" + o.tag + ",sock=" + o.socketPath + if o.dax { + fsarg += ",dax=on" + } else { + fsarg += ",num_queues=" + strconv.FormatUint(uint64(o.queues), 10) + ",queue_size=" + strconv.FormatUint(uint64(o.queueSize), 10) + } + cmdline.args = append(cmdline.args, fsarg) +} + +//**************************************** +// The net (nic) +//**************************************** +type CLINet struct { + device string + mac string + iommu bool +} + +type CLINets struct { + networks []CLINet +} + +func (o *CLINets) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscNet) + + networks := "" + netIndex := 1 + for _, net := range o.networks { + tapName := "tap" + strconv.FormatUint(uint64(netIndex), 10) + netIndex++ + if net.iommu { + networks += "tap=" + tapName + ",mac=" + net.mac + ",iommu=on" + } else { + networks += "tap=" + tapName + ",mac=" + net.mac + } + } + cmdline.args = append(cmdline.args, networks) +} + +//**************************************** +// The log file +//**************************************** +type CLILogFile struct { + path string +} + +func (o *CLILogFile) Build(cmdline *CommandLine) { + + if o.path != "" { + cmdline.args = append(cmdline.args, cscLogFile) + cmdline.args = append(cmdline.args, o.path) + } +} + +//**************************************** +// The kernel command line +//**************************************** +type CLICmdline struct { + params []Param +} + +func (o *CLICmdline) Build(cmdline *CommandLine) { + + cmdline.args = append(cmdline.args, cscCmdline) + + var paramBuilder strings.Builder + for _, p := range o.params { + paramBuilder.WriteString(p.Key) + if len(p.Value) > 0 { + + paramBuilder.WriteString("=") + paramBuilder.WriteString(p.Value) + } + paramBuilder.WriteString(" ") + } + cmdline.args = append(cmdline.args, strings.TrimSpace(paramBuilder.String())) + +} + +//********************************** +// The Default Builder +//********************************** +type DefaultCLIBuilder struct { + console *CLIConsole + serial *CLISerialConsole + apiSocket *CLIAPISocket + cpus *CLICpus + memory *CLIMemory + kernel *CLIKernel + disk *CLIDisk + fs *CLIFs + rng *CLIRng + logFile *CLILogFile + vsock *CLIVsock + cmdline *CLICmdline + nets *CLINets +} + +func (d *DefaultCLIBuilder) AddKernelParameters(params []Param) { + + if d.cmdline == nil { + d.cmdline = &CLICmdline{} + } + d.cmdline.params = append(d.cmdline.params, params...) +} + +func (d *DefaultCLIBuilder) SetConsole(console *CLIConsole) { + d.console = console +} + +func (d *DefaultCLIBuilder) SetCpus(cpus *CLICpus) { + d.cpus = cpus +} + +func (d *DefaultCLIBuilder) SetDisk(disk *CLIDisk) { + d.disk = disk +} + +func (d *DefaultCLIBuilder) SetFs(fs *CLIFs) { + d.fs = fs +} + +func (d *DefaultCLIBuilder) SetKernel(kernel *CLIKernel) { + d.kernel = kernel +} + +func (d *DefaultCLIBuilder) SetMemory(memory *CLIMemory) { + d.memory = memory +} + +func (d *DefaultCLIBuilder) AddNet(net CLINet) { + if d.nets == nil { + d.nets = &CLINets{} + } + d.nets.networks = append(d.nets.networks, net) +} + +func (d *DefaultCLIBuilder) SetRng(rng *CLIRng) { + d.rng = rng +} + +func (d *DefaultCLIBuilder) SetSerial(serial *CLISerialConsole) { + d.serial = serial +} + +func (d *DefaultCLIBuilder) SetVsock(vsock *CLIVsock) { + d.vsock = vsock +} + +func (d *DefaultCLIBuilder) SetAPISocket(apiSocket *CLIAPISocket) { + d.apiSocket = apiSocket +} + +func (d *DefaultCLIBuilder) SetLogFile(logFile *CLILogFile) { + d.logFile = logFile +} + +func (d *DefaultCLIBuilder) GetCommandLine() (*CommandLine, error) { + + cmdLine := &CommandLine{} + + if d.serial != nil { + d.serial.Build(cmdLine) + } + + if d.console != nil { + d.console.Build(cmdLine) + } + + if d.logFile != nil { + d.logFile.Build(cmdLine) + } + + if d.cpus != nil { + d.cpus.Build(cmdLine) + } + if d.memory != nil { + d.memory.Build(cmdLine) + } + if d.disk != nil { + d.disk.Build(cmdLine) + } + if d.rng != nil { + d.rng.Build(cmdLine) + } + if d.vsock != nil { + d.vsock.Build(cmdLine) + } + if d.fs != nil { + d.fs.Build(cmdLine) + } + if d.kernel != nil { + d.kernel.Build(cmdLine) + } + if d.nets != nil { + d.nets.Build(cmdLine) + } + if d.cmdline != nil { + d.cmdline.Build(cmdLine) + } + + return cmdLine, nil +} + +type CommandLineDirector struct{} + +func (s *CommandLineDirector) Build(builder CommandLineBuilder) (*CommandLine, error) { + return builder.GetCommandLine() +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index cdb2767a6..82bf44b65 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -42,6 +42,9 @@ const ( // AcrnHypervisor is the ACRN hypervisor. AcrnHypervisor HypervisorType = "acrn" + // ClhHypervisor is the ICH hypervisor. + ClhHypervisor HypervisorType = "clh" + // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" ) @@ -152,6 +155,9 @@ func (hType *HypervisorType) Set(value string) error { case "acrn": *hType = AcrnHypervisor return nil + case "clh": + *hType = ClhHypervisor + return nil case "mock": *hType = MockHypervisor return nil @@ -169,6 +175,8 @@ func (hType *HypervisorType) String() string { return string(FirecrackerHypervisor) case AcrnHypervisor: return string(AcrnHypervisor) + case ClhHypervisor: + return string(ClhHypervisor) case MockHypervisor: return string(MockHypervisor) default: @@ -185,6 +193,8 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { return &firecracker{}, nil case AcrnHypervisor: return &Acrn{}, nil + case ClhHypervisor: + return &cloudHypervisor{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: diff --git a/virtcontainers/virtcontainers_test.go b/virtcontainers/virtcontainers_test.go index 7b0ccbda6..4514250ba 100644 --- a/virtcontainers/virtcontainers_test.go +++ b/virtcontainers/virtcontainers_test.go @@ -46,6 +46,7 @@ var testAcrnKernelPath = "" var testAcrnImagePath = "" var testAcrnPath = "" var testAcrnCtlPath = "" + var testHyperstartCtlSocket = "" var testHyperstartTtySocket = "" From 784066a49d29f9f455ad95bd2b9655e8868a6631 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Thu, 14 Nov 2019 20:56:46 +0000 Subject: [PATCH 21/23] Makefile: clh: Use virtiofs kernel use virtiofs kernel to allow boot kata. Signed-off-by: Jose Carlos Venegas Munoz --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a06549f9a..d48e7ab56 100644 --- a/Makefile +++ b/Makefile @@ -282,7 +282,7 @@ ifneq (,$(CLHCMD)) DEFENABLEHUGEPAGES_CLH := true DEFNETWORKMODEL_CLH := tcfilter KERNELTYPE_CLH = uncompressed - KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH)) + KERNEL_NAME_CLH = $(call MAKE_KERNEL_VIRTIOFS_NAME,$(KERNELTYPE_CLH)) KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH) endif From ee9a53ca4b9cdb6b523e4aa2ae4bc1ca5713abd6 Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Mon, 18 Nov 2019 07:06:32 -0800 Subject: [PATCH 22/23] vc: Remove extra call to Kill Fixes #2207 Signed-off-by: Ted Yu --- virtcontainers/clh.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go index 2e05418c0..3aa407725 100644 --- a/virtcontainers/clh.go +++ b/virtcontainers/clh.go @@ -508,14 +508,11 @@ func (clh *cloudHypervisor) terminate() (err error) { } clh.Logger().WithField("PID", pid).Info("Stopping Cloud Hypervisor") - // Check if VM process is running, in case it is not, let's - // return from here. - if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { - return nil - } - // Send a SIGTERM to the VM process to try to stop it properly if err = syscall.Kill(pid, syscall.SIGTERM); err != nil { + if err == syscall.ESRCH { + return nil + } return err } From 510f0a668733b6e5390ad15a3a8fe252c0a4a100 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Tue, 19 Nov 2019 01:02:04 -0800 Subject: [PATCH 23/23] clh: fix build PR #2202 changed createSandbox() interface but didn't get a chance to match with cloud hypervisor change. Fixes: #2213 Signed-off-by: Peng Tao --- virtcontainers/clh.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go index 2e05418c0..b47936d9b 100644 --- a/virtcontainers/clh.go +++ b/virtcontainers/clh.go @@ -109,7 +109,7 @@ var clhDebugKernelParams = []Param{ // //########################################################### -func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error { clh.ctx = ctx span, _ := clh.trace("createSandbox")