From 4d26ceee7954fa6c68138b60874556f9fd1c7db4 Mon Sep 17 00:00:00 2001 From: Vijay Dhanraj Date: Sat, 8 Jun 2019 11:49:54 -0700 Subject: [PATCH 1/6] Make: Add ACRN hypervisor and generate configuration file This patch covers the following aspects, 1. Add ACRN as a supported hypervisor for amd64 architecture. 2. Build and install configuration file for ACRN hypervisor. v1->v2: 1. Deleted autogenerated configuration-acrn.toml. 2. Trimmed configuration options not used by ACRN. Fixes: #1778 Signed-off-by: Vijay Dhanraj --- Makefile | 51 +++++- arch/amd64-options.mk | 4 + cli/config/configuration-acrn.toml.in | 235 ++++++++++++++++++++++++++ 3 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 cli/config/configuration-acrn.toml.in diff --git a/Makefile b/Makefile index 800ab8984..6caced7c3 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,7 @@ BINDIR := $(EXEC_PREFIX)/bin NEMUBINDIR := $(PREFIXDEPS)/bin QEMUBINDIR := $(PREFIXDEPS)/bin FCBINDIR := $(PREFIXDEPS)/bin +ACRNBINDIR := $(PREFIXDEPS)/bin VIRTIOFSDBINDIR := $(PREFIXDEPS)/bin SYSCONFDIR := /etc LOCALSTATEDIR := /var @@ -113,6 +114,7 @@ FIRMWAREPATH := # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml +HYPERVISOR_ACRN = acrn HYPERVISOR_FC = firecracker HYPERVISOR_NEMU = nemu HYPERVISOR_QEMU = qemu @@ -121,7 +123,7 @@ HYPERVISOR_QEMU = qemu DEFAULT_HYPERVISOR = $(HYPERVISOR_QEMU) # List of hypervisors this build system can generate configuration for. -HYPERVISORS := $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_NEMU) +HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_NEMU) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) @@ -129,6 +131,9 @@ NEMUPATH := $(NEMUBINDIR)/$(NEMUCMD) FCPATH = $(FCBINDIR)/$(FCCMD) +ACRNPATH := $(ACRNBINDIR)/$(ACRNCMD) +ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) + SHIMCMD := $(BIN_PREFIX)-shim SHIMPATH := $(PKGLIBEXECDIR)/$(SHIMCMD) @@ -198,7 +203,7 @@ VERSION := ${shell cat ./VERSION} # List of configuration files to build and install CONFIGS = -CONFIG_PATHS = +CONFIG_PATHS = SYSCONFIG_PATHS = # List of hypervisors known for the current architecture @@ -276,6 +281,28 @@ ifneq (,$(FCCMD)) KERNELPATH_FC = $(KERNELDIR)/$(KERNEL_NAME_FC) endif +ifneq (,$(ACRNCMD)) + KNOWN_HYPERVISORS += $(HYPERVISOR_ACRN) + + CONFIG_FILE_ACRN = configuration-acrn.toml + CONFIG_ACRN = $(CLI_DIR)/config/$(CONFIG_FILE_ACRN) + CONFIG_ACRN_IN = $(CONFIG_ACRN).in + + CONFIG_PATH_ACRN = $(abspath $(CONFDIR)/$(CONFIG_FILE_ACRN)) + CONFIG_PATHS += $(CONFIG_PATH_ACRN) + + SYSCONFIG_ACRN = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_ACRN)) + SYSCONFIG_PATHS += $(SYSCONFIG_ACRN) + + CONFIGS += $(CONFIG_ACRN) + + # acrn-specific options (all should be suffixed by "_ACRN") + DEFBLOCKSTORAGEDRIVER_ACRN := virtio-blk + DEFNETWORKMODEL_ACRN := bridged + KERNEL_NAME_ACRN = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) + KERNELPATH_ACRN = $(KERNELDIR)/$(KERNEL_NAME_ACRN) +endif + ifeq (,$(KNOWN_HYPERVISORS)) $(error "ERROR: No hypervisors known for architecture $(ARCH) (looked for: $(HYPERVISORS))") endif @@ -300,6 +327,10 @@ ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_NEMU)) DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_NEMU) endif +ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_ACRN)) + DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_ACRN) +endif + CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR) @@ -318,6 +349,10 @@ USER_VARS += BINDIR USER_VARS += CONFIG_PATH USER_VARS += DESTDIR USER_VARS += DEFAULT_HYPERVISOR +USER_VARS += ACRNCMD +USER_VARS += ACRNCTLCMD +USER_VARS += ACRNPATH +USER_VARS += ACRNCTLPATH USER_VARS += FCCMD USER_VARS += FCPATH USER_VARS += NEMUCMD @@ -331,6 +366,7 @@ USER_VARS += MACHINETYPE USER_VARS += KERNELDIR USER_VARS += KERNELTYPE USER_VARS += KERNELTYPE_FC +USER_VARS += KERNELTYPE_ACRN USER_VARS += FIRMWAREPATH USER_VARS += FIRMWAREPATH_NEMU USER_VARS += MACHINEACCELERATORS @@ -359,12 +395,14 @@ USER_VARS += DEFMAXVCPUS USER_VARS += DEFMEMSZ USER_VARS += DEFMEMSLOTS USER_VARS += DEFBRIDGES +USER_VARS += DEFNETWORKMODEL_ACRN USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFNETWORKMODEL_QEMU USER_VARS += DEFNETWORKMODEL_NEMU USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFAULTEXPFEATURES USER_VARS += DEFDISABLEBLOCK +USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN USER_VARS += DEFBLOCKSTORAGEDRIVER_FC USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU USER_VARS += DEFBLOCKSTORAGEDRIVER_NEMU @@ -472,14 +510,18 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit $(QUIET_GENERATE)$(SED) \ -e "s|@COMMIT@|$(shell cat .git-commit)|g" \ -e "s|@VERSION@|$(VERSION)|g" \ + -e "s|@CONFIG_ACRN_IN@|$(CONFIG_ACRN_IN)|g" \ -e "s|@CONFIG_QEMU_IN@|$(CONFIG_QEMU_IN)|g" \ -e "s|@CONFIG_NEMU_IN@|$(CONFIG_NEMU_IN)|g" \ -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ -e "s|@NEMUPATH@|$(NEMUPATH)|g" \ + -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ + -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ + -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ -e "s|@KERNELPATH_FC@|$(KERNELPATH_FC)|g" \ -e "s|@KERNELPATH@|$(KERNELPATH)|g" \ -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ @@ -507,12 +549,14 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \ -e "s|@DEFMEMSLOTS@|$(DEFMEMSLOTS)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ + -e "s|@DEFNETWORKMODEL_ACRN@|$(DEFNETWORKMODEL_ACRN)|g" \ -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ -e "s|@DEFNETWORKMODEL_NEMU@|$(DEFNETWORKMODEL_NEMU)|g" \ -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ -e "s|@DEFAULTEXPFEATURES@|$(DEFAULTEXPFEATURES)|g" \ -e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \ + -e "s|@DEFBLOCKSTORAGEDRIVER_ACRN@|$(DEFBLOCKSTORAGEDRIVER_ACRN)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_FC@|$(DEFBLOCKSTORAGEDRIVER_FC)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_QEMU@|$(DEFBLOCKSTORAGEDRIVER_QEMU)|g" \ -e "s|@DEFBLOCKSTORAGEDRIVER_NEMU@|$(DEFBLOCKSTORAGEDRIVER_NEMU)|g" \ @@ -680,6 +724,9 @@ ifneq (,$(findstring $(HYPERVISOR_NEMU),$(KNOWN_HYPERVISORS))) endif ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) @printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH)) +endif +ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS))) + @printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH)) endif @printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR)) @printf "\tproxy+shim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR)) diff --git a/arch/amd64-options.mk b/arch/amd64-options.mk index 78db6662c..a038b71e4 100644 --- a/arch/amd64-options.mk +++ b/arch/amd64-options.mk @@ -16,3 +16,7 @@ FCCMD := firecracker # NEMU binary name NEMUCMD := nemu-system-x86_64 + +#ACRN binary name +ACRNCMD := acrn-dm +ACRNCTLCMD := acrnctl diff --git a/cli/config/configuration-acrn.toml.in b/cli/config/configuration-acrn.toml.in new file mode 100644 index 000000000..cbb2cc174 --- /dev/null +++ b/cli/config/configuration-acrn.toml.in @@ -0,0 +1,235 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "@CONFIG_ACRN_IN@" +# XXX: Project: +# XXX: Name: @PROJECT_NAME@ +# XXX: Type: @PROJECT_TYPE@ + +[hypervisor.acrn] +path = "@ACRNPATH@" +ctlpath = "@ACRNCTLPATH@" +kernel = "@KERNELPATH_ACRN@" +image = "@IMAGEPATH@" + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "@KERNELPARAMS@" + +# Path to the firmware. +# If you want that acrn uses the default firmware leave this option empty +firmware = "@FIRMWAREPATH@" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to @DEFVCPUS@ +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS@ + +# Bridges can be used to hot plug devices. +# Limitations: +# * Currently only pci bridges are supported +# * Until 30 devices per bridge can be hot plugged. +# * Until 5 PCI bridges can be cold plugged per VM. +# This limitation could be a bug in the kernel +# Default number of bridges per SB/VM: +# unspecified or 0 --> will be set to @DEFBRIDGES@ +# > 1 <= 5 --> will be set to the specified number +# > 5 --> will be set to 5 +default_bridges = @DEFBRIDGES@ + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set @DEFMEMSZ@ MiB. +default_memory = @DEFMEMSZ@ + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. ACRN only supports virtio-blk. +block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@" + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. This extra output is added +# to the proxy logs, but only when proxy debug is also enabled. +# +# Default false +#enable_debug = true + +# Disable the customizations done in the runtime when it detects +# that it is running on top a VMM. This will result in the runtime +# behaving as it would when running on bare metal. +# +#disable_nesting_checks = true + +# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. +# Default false +#disable_vhost_net = true + +# Path to OCI hook binaries in the *guest rootfs*. +# This does not affect host-side hooks which must instead be added to +# the OCI spec passed to the runtime. +# +# You can create a rootfs with hooks by customizing the osbuilder scripts: +# https://github.com/kata-containers/osbuilder +# +# Hooks must be stored in a subdirectory of guest_hook_path according to their +# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}". +# The agent will scan these directories for executable files and add them, in +# lexicographical order, to the lifecycle of the guest container. +# Hooks are executed in the runtime namespace of the guest. See the official documentation: +# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks +# Warnings will be logged if any error is encountered will scanning for hooks, +# but it will not abort container execution. +#guest_hook_path = "/usr/share/oci/hooks" + +[proxy.@PROJECT_TYPE@] +path = "@PROXYPATH@" + +# If enabled, proxy messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +[shim.@PROJECT_TYPE@] +path = "@SHIMPATH@" + +# If enabled, shim messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +# If enabled, the shim will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# +# Note: By default, the shim runs in a separate network namespace. Therefore, +# to allow it to send trace details to the Jaeger agent running on the host, +# it is necessary to set 'disable_new_netns=true' so that it runs in the host +# network namespace. +# +# (default: disabled) +#enable_tracing = true + +[agent.@PROJECT_TYPE@] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +#enable_debug = true + +# Enable agent tracing. +# +# If enabled, the default trace mode is "dynamic" and the +# default trace type is "isolated". The trace mode and type are set +# explicity with the `trace_type=` and `trace_mode=` options. +# +# Notes: +# +# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly +# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` +# will NOT activate agent tracing. +# +# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for +# full details. +# +# (default: disabled) +#enable_tracing = true +# +#trace_mode = "dynamic" +#trace_type = "isolated" + +[netmon] +# If enabled, the network monitoring process gets started when the +# sandbox is created. This allows for the detection of some additional +# network being added to the existing network namespace, after the +# sandbox has been created. +# (default: disabled) +#enable_netmon = true + +# Specify the path to the netmon binary. +path = "@NETMONPATH@" + +# If enabled, netmon messages will be sent to the system log +# (default: disabled) +#enable_debug = true + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +#enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - bridged +# Uses a linux bridge to interconnect the container interface to +# the VM. Works for most cases except macvlan and ipvlan. +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="@DEFNETWORKMODEL_ACRN@" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `enable_netmon` +# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# If you are using docker, `disable_new_netns` only works with `docker run --net=none` +# (default: false) +#disable_new_netns = true + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# They may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# 1. "newstore": new persist storage driver which breaks backward compatibility, +# expected to move out of experimental in 2.0.0. +# (default: []) +experimental=@DEFAULTEXPFEATURES@ From adcac9368f2847d5371afacb6d34599dfdf3d97b Mon Sep 17 00:00:00 2001 From: Vijay Dhanraj Date: Sat, 8 Jun 2019 12:42:31 -0700 Subject: [PATCH 2/6] kata-check: Check and validate type-1 hypervisor for kata ACRN hypervisor is a type-1 hypervisor and this patch adds support to check and validate if the system is capable of running kata containers with ACRN hypervisor. Depends-on: github.com/kata-containers/tests#1793 v3->v4: Implemented a generic way to identify hypervisor and test VM creation. v2->v3: 1. Removed cgo structs and defined go structs. 2. Suppressed lint warnings due to unused createVM struct. v1->v2: 1. Created an issue #1784 to address TODO item. 2. Fixed formatting of the log message. 3. Currently ACRN is only supported on amd64. So moved ACRN specific code to kata-check_amd64.go. Fixes: #1778 Signed-off-by: Vijay Dhanraj --- cli/kata-check.go | 11 +- cli/kata-check_amd64.go | 207 ++++++++++++++++++++++++++------- cli/kata-check_amd64_test.go | 13 ++- cli/kata-check_arm64.go | 5 +- cli/kata-check_arm64_test.go | 4 + cli/kata-check_generic_test.go | 11 +- cli/kata-check_ppc64le.go | 9 +- cli/kata-check_ppc64le_test.go | 4 + cli/kata-check_s390x.go | 8 +- cli/kata-check_s390x_test.go | 4 + cli/kata-check_test.go | 4 + cli/kata-env.go | 5 +- cli/main.go | 102 ++++++++-------- 13 files changed, 279 insertions(+), 108 deletions(-) diff --git a/cli/kata-check.go b/cli/kata-check.go index 28c23b6dd..068c74373 100644 --- a/cli/kata-check.go +++ b/cli/kata-check.go @@ -16,6 +16,7 @@ const int ioctl_KVM_CHECK_EXTENSION = KVM_CHECK_EXTENSION; import "C" import ( + "errors" "fmt" "os" "os/exec" @@ -26,6 +27,7 @@ import ( "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -311,7 +313,12 @@ var kataCheckCLICommand = cli.Command{ span, _ := katautils.Trace(ctx, "kata-check") defer span.Finish() - err = setCPUtype() + runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig) + if !ok { + return errors.New("kata-check: cannot determine runtime config") + } + + err = setCPUtype(runtimeConfig.HypervisorType) if err != nil { return err } @@ -332,7 +339,7 @@ var kataCheckCLICommand = cli.Command{ kataLog.Info(successMessageCapable) if os.Geteuid() == 0 { - err = archHostCanCreateVMContainer() + err = archHostCanCreateVMContainer(runtimeConfig.HypervisorType) if err != nil { return err } diff --git a/cli/kata-check_amd64.go b/cli/kata-check_amd64.go index 21e210e89..a36a3ec16 100644 --- a/cli/kata-check_amd64.go +++ b/cli/kata-check_amd64.go @@ -7,11 +7,13 @@ package main import ( "fmt" - "github.com/sirupsen/logrus" "io/ioutil" "strings" + "syscall" + "unsafe" vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/sirupsen/logrus" ) const ( @@ -24,6 +26,17 @@ const ( msgKernelVirtio = "Host kernel accelerator for virtio" msgKernelVirtioNet = "Host kernel accelerator for virtio network" msgKernelVirtioVhostVsock = "Host Support for Linux VM Sockets" + cpuFlagVMX = "vmx" + cpuFlagLM = "lm" + cpuFlagSVM = "svm" + cpuFlagSSE4_1 = "sse4_1" + kernelModvhm = "vhm_dev" + kernelModvhost = "vhost" + kernelModvhostnet = "vhost_net" + kernelModvhostvsock = "vhost_vsock" + kernelModkvm = "kvm" + kernelModkvmintel = "kvm_intel" + kernelModkvmamd = "kvm_amd" ) // CPU types @@ -33,6 +46,28 @@ const ( cpuTypeUnknown = -1 ) +const acrnDevice = "/dev/acrn_vhm" + +// ioctl_ACRN_CREATE_VM is the IOCTL to create VM in ACRN. +// Current Linux mainstream kernel doesn't have support for ACRN. +// Due to this several macros are not defined in Linux headers. +// Until the support is available, directly use the value instead +// of macros. +//https://github.com/kata-containers/runtime/issues/1784 +const ioctl_ACRN_CREATE_VM = 0x43000010 //nolint +const ioctl_ACRN_DESTROY_VM = 0x43000011 //nolint + +type acrn_create_vm struct { //nolint + vmid uint16 //nolint + reserved0 uint16 //nolint + vcpu_num uint16 //nolint + reserved1 uint16 //nolint + uuid [16]uint8 + vm_flag uint64 //nolint + req_buf uint64 //nolint + reserved2 [16]uint8 //nolint +} + // cpuType save the CPU type var cpuType int @@ -49,7 +84,7 @@ var archRequiredCPUAttribs map[string]string // required module parameters. var archRequiredKernelModules map[string]kernelModule -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { cpuType = getCPUtype() if cpuType == cpuTypeUnknown { @@ -66,64 +101,88 @@ func setCPUtype() error { "unrestricted_guest": "Y", } } - archRequiredCPUFlags = map[string]string{ - "vmx": "Virtualization support", - "lm": "64Bit CPU", - "sse4_1": "SSE4.1", - } - archRequiredCPUAttribs = map[string]string{ - archGenuineIntel: "Intel Architecture CPU", - } - archRequiredKernelModules = map[string]kernelModule{ - "kvm": { - desc: msgKernelVM, - required: true, - }, - "kvm_intel": { - desc: "Intel KVM", - parameters: kvmIntelParams, - required: true, - }, - "vhost": { - desc: msgKernelVirtio, - required: true, - }, - "vhost_net": { - desc: msgKernelVirtioNet, - required: true, - }, - "vhost_vsock": { - desc: msgKernelVirtioVhostVsock, - required: false, - }, + + switch hypervisorType { + case "firecracker": + fallthrough + case "qemu": + archRequiredCPUFlags = map[string]string{ + cpuFlagVMX: "Virtualization support", + cpuFlagLM: "64Bit CPU", + cpuFlagSSE4_1: "SSE4.1", + } + archRequiredCPUAttribs = map[string]string{ + archGenuineIntel: "Intel Architecture CPU", + } + archRequiredKernelModules = map[string]kernelModule{ + kernelModkvm: { + desc: msgKernelVM, + }, + kernelModkvmintel: { + desc: "Intel KVM", + parameters: kvmIntelParams, + }, + kernelModvhost: { + desc: msgKernelVirtio, + }, + kernelModvhostnet: { + desc: msgKernelVirtioNet, + }, + kernelModvhostvsock: { + desc: msgKernelVirtioVhostVsock, + required: false, + }, + } + case "acrn": + archRequiredCPUFlags = map[string]string{ + cpuFlagLM: "64Bit CPU", + cpuFlagSSE4_1: "SSE4.1", + } + archRequiredCPUAttribs = map[string]string{ + archGenuineIntel: "Intel Architecture CPU", + } + archRequiredKernelModules = map[string]kernelModule{ + kernelModvhm: { + desc: "Intel ACRN", + }, + kernelModvhost: { + desc: msgKernelVirtio, + }, + kernelModvhostnet: { + desc: msgKernelVirtioNet, + }, + } + default: + return fmt.Errorf("setCPUtype: Unknown hypervisor type %s", hypervisorType) } + } else if cpuType == cpuTypeAMD { archRequiredCPUFlags = map[string]string{ - "svm": "Virtualization support", - "lm": "64Bit CPU", - "sse4_1": "SSE4.1", + cpuFlagSVM: "Virtualization support", + cpuFlagLM: "64Bit CPU", + cpuFlagSSE4_1: "SSE4.1", } archRequiredCPUAttribs = map[string]string{ archAuthenticAMD: "AMD Architecture CPU", } archRequiredKernelModules = map[string]kernelModule{ - "kvm": { + kernelModkvm: { desc: msgKernelVM, required: true, }, - "kvm_amd": { + kernelModkvmamd: { desc: "AMD KVM", required: true, }, - "vhost": { + kernelModvhost: { desc: msgKernelVirtio, required: true, }, - "vhost_net": { + kernelModvhostnet: { desc: msgKernelVirtioNet, required: true, }, - "vhost_vsock": { + kernelModvhostvsock: { desc: msgKernelVirtioVhostVsock, required: false, }, @@ -155,8 +214,72 @@ func kvmIsUsable() error { return genericKvmIsUsable() } -func archHostCanCreateVMContainer() error { - return kvmIsUsable() +// acrnIsUsable determines if it will be possible to create a full virtual machine +// by creating a minimal VM and then deleting it. +func acrnIsUsable() error { + flags := syscall.O_RDWR | syscall.O_CLOEXEC + + f, err := syscall.Open(acrnDevice, flags, 0) + if err != nil { + return err + } + defer syscall.Close(f) + + fieldLogger := kataLog.WithField("check-type", "full") + + fieldLogger.WithField("device", acrnDevice).Info("device available") + + createVM := acrn_create_vm{ + uuid: [16]uint8{ + 0xd2, 0x79, 0x54, 0x38, 0x25, 0xd6, 0x11, 0xe8, + 0x86, 0x4e, 0xcb, 0x7a, 0x18, 0xb3, 0x46, 0x43, + }, + } + + ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL, + uintptr(f), + uintptr(ioctl_ACRN_CREATE_VM), + uintptr(unsafe.Pointer(&createVM))) + if ret != 0 || errno != 0 { + if errno == syscall.EBUSY { + fieldLogger.WithField("reason", "another hypervisor running").Error("cannot create VM") + } + fieldLogger.WithFields(logrus.Fields{ + "ret": ret, + "errno": errno, + }).Info("Create VM Error") + return errno + } + + ret, _, errno = syscall.Syscall(syscall.SYS_IOCTL, + uintptr(f), + uintptr(ioctl_ACRN_DESTROY_VM), + 0) + if ret != 0 || errno != 0 { + fieldLogger.WithFields(logrus.Fields{ + "ret": ret, + "errno": errno, + }).Info("Destroy VM Error") + return errno + } + + fieldLogger.WithField("feature", "create-vm").Info("feature available") + + return nil +} + +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { + + switch hypervisorType { + case "qemu": + fallthrough + case "firecracker": + return kvmIsUsable() + case "acrn": + return acrnIsUsable() + default: + return fmt.Errorf("archHostCanCreateVMContainer: Unknown hypervisor type %s", hypervisorType) + } } // hostIsVMContainerCapable checks to see if the host is theoretically capable diff --git a/cli/kata-check_amd64_test.go b/cli/kata-check_amd64_test.go index bf2bbedd0..2fcba9ae2 100644 --- a/cli/kata-check_amd64_test.go +++ b/cli/kata-check_amd64_test.go @@ -55,6 +55,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -108,6 +111,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} @@ -514,6 +518,10 @@ foo : bar func TestSetCPUtype(t *testing.T) { assert := assert.New(t) + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + savedArchRequiredCPUFlags := archRequiredCPUFlags savedArchRequiredCPUAttribs := archRequiredCPUAttribs savedArchRequiredKernelModules := archRequiredKernelModules @@ -528,7 +536,10 @@ func TestSetCPUtype(t *testing.T) { archRequiredCPUAttribs = map[string]string{} archRequiredKernelModules = map[string]kernelModule{} - setCPUtype() + _, config, err := makeRuntimeConfig(tmpdir) + assert.NoError(err) + + setCPUtype(config.HypervisorType) assert.NotEmpty(archRequiredCPUFlags) assert.NotEmpty(archRequiredCPUAttribs) diff --git a/cli/kata-check_arm64.go b/cli/kata-check_arm64.go index 5f835fcab..959f312a4 100644 --- a/cli/kata-check_arm64.go +++ b/cli/kata-check_arm64.go @@ -8,6 +8,7 @@ package main import ( "fmt" + vc "github.com/kata-containers/runtime/virtcontainers" "github.com/sirupsen/logrus" ) @@ -56,7 +57,7 @@ var archRequiredKVMExtensions = map[string]kvmExtension{ }, } -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { return nil } @@ -84,7 +85,7 @@ func checkKVMExtensions() error { return nil } -func archHostCanCreateVMContainer() error { +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { if err := kvmIsUsable(); err != nil { return err } diff --git a/cli/kata-check_arm64_test.go b/cli/kata-check_arm64_test.go index 433c0996f..3c0588f29 100644 --- a/cli/kata-check_arm64_test.go +++ b/cli/kata-check_arm64_test.go @@ -36,6 +36,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -78,6 +81,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} diff --git a/cli/kata-check_generic_test.go b/cli/kata-check_generic_test.go index 5332d4e89..144126830 100644 --- a/cli/kata-check_generic_test.go +++ b/cli/kata-check_generic_test.go @@ -8,6 +8,8 @@ package main import ( + "io/ioutil" + "os" "testing" "github.com/stretchr/testify/assert" @@ -16,6 +18,10 @@ import ( func testSetCPUTypeGeneric(t *testing.T) { assert := assert.New(t) + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + savedArchRequiredCPUFlags := archRequiredCPUFlags savedArchRequiredCPUAttribs := archRequiredCPUAttribs savedArchRequiredKernelModules := archRequiredKernelModules @@ -30,7 +36,10 @@ func testSetCPUTypeGeneric(t *testing.T) { assert.Empty(archRequiredCPUAttribs) assert.NotEmpty(archRequiredKernelModules) - setCPUtype() + _, config, err := makeRuntimeConfig(tmpdir) + assert.NoError(err) + + setCPUtype(config.HypervisorType) assert.Equal(archRequiredCPUFlags, savedArchRequiredCPUFlags) assert.Equal(archRequiredCPUAttribs, savedArchRequiredCPUAttribs) diff --git a/cli/kata-check_ppc64le.go b/cli/kata-check_ppc64le.go index 4d7ec3494..b57c971ce 100644 --- a/cli/kata-check_ppc64le.go +++ b/cli/kata-check_ppc64le.go @@ -8,12 +8,13 @@ package main import ( "fmt" "os/exec" + "regexp" + "strconv" "strings" "github.com/kata-containers/runtime/pkg/katautils" + vc "github.com/kata-containers/runtime/virtcontainers" "github.com/sirupsen/logrus" - "regexp" - "strconv" ) const ( @@ -55,11 +56,11 @@ var archRequiredKernelModules = map[string]kernelModule{ }, } -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { return nil } -func archHostCanCreateVMContainer() error { +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { return kvmIsUsable() } diff --git a/cli/kata-check_ppc64le_test.go b/cli/kata-check_ppc64le_test.go index 29859c30c..d49ec445f 100644 --- a/cli/kata-check_ppc64le_test.go +++ b/cli/kata-check_ppc64le_test.go @@ -53,6 +53,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -98,6 +101,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} diff --git a/cli/kata-check_s390x.go b/cli/kata-check_s390x.go index 1e5f42d50..0c67f1520 100644 --- a/cli/kata-check_s390x.go +++ b/cli/kata-check_s390x.go @@ -7,8 +7,10 @@ package main import ( "fmt" - "github.com/sirupsen/logrus" "strings" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/sirupsen/logrus" ) const ( @@ -42,7 +44,7 @@ var archRequiredKernelModules = map[string]kernelModule{ }, } -func setCPUtype() error { +func setCPUtype(hypervisorType vc.HypervisorType) error { return nil } @@ -52,7 +54,7 @@ func kvmIsUsable() error { return genericKvmIsUsable() } -func archHostCanCreateVMContainer() error { +func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error { return kvmIsUsable() } diff --git a/cli/kata-check_s390x_test.go b/cli/kata-check_s390x_test.go index c8bc129ea..108d6e1e8 100644 --- a/cli/kata-check_s390x_test.go +++ b/cli/kata-check_s390x_test.go @@ -53,6 +53,9 @@ func TestCCCheckCLIFunction(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + savedSysModuleDir := sysModuleDir savedProcCPUInfo := procCPUInfo @@ -97,6 +100,7 @@ func TestCCCheckCLIFunction(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config // create buffer to save logger output buf := &bytes.Buffer{} diff --git a/cli/kata-check_test.go b/cli/kata-check_test.go index 29767a926..87605ef12 100644 --- a/cli/kata-check_test.go +++ b/cli/kata-check_test.go @@ -659,6 +659,9 @@ func TestCheckCLIFunctionFail(t *testing.T) { } defer os.RemoveAll(dir) + _, config, err := makeRuntimeConfig(dir) + assert.NoError(err) + oldProcCPUInfo := procCPUInfo // doesn't exist @@ -670,6 +673,7 @@ func TestCheckCLIFunctionFail(t *testing.T) { ctx := createCLIContext(nil) ctx.App.Name = "foo" + ctx.App.Metadata["runtimeConfig"] = config fn, ok := kataCheckCLICommand.Action.(func(context *cli.Context) error) assert.True(ok) diff --git a/cli/kata-env.go b/cli/kata-env.go index c8610daa4..f9c1e9d80 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -9,9 +9,8 @@ import ( "encoding/json" "errors" "os" - "strings" - runtim "runtime" + "strings" "github.com/BurntSushi/toml" "github.com/kata-containers/runtime/pkg/katautils" @@ -358,7 +357,7 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo { } func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) { - err = setCPUtype() + err = setCPUtype(config.HypervisorType) if err != nil { return EnvInfo{}, err } diff --git a/cli/main.go b/cli/main.go index a021a0a77..72c9165e7 100644 --- a/cli/main.go +++ b/cli/main.go @@ -254,54 +254,54 @@ func beforeSubcommands(c *cli.Context) error { handleShowConfig(c) - if userWantsUsage(c) || (c.NArg() == 1 && (c.Args()[0] == checkCmd)) { + if userWantsUsage(c) { // No setup required if the user just - // wants to see the usage statement or are - // running a command that does not manipulate - // containers. + // wants to see the usage statement. return nil } - if path := c.GlobalString("log"); path != "" { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) - if err != nil { - return err - } - kataLog.Logger.Out = f - } - - switch c.GlobalString("log-format") { - case "text": - // retain logrus's default. - case "json": - kataLog.Logger.Formatter = new(logrus.JSONFormatter) - default: - return fmt.Errorf("unknown log-format %q", c.GlobalString("log-format")) - } - + ignoreLogging := false var traceRootSpan string - // Add the name of the sub-command to each log entry for easier - // debugging. - cmdName := c.Args().First() - if c.App.Command(cmdName) != nil { - kataLog = kataLog.WithField("command", cmdName) + subCmdIsCheckCmd := (c.NArg() == 1 && (c.Args()[0] == checkCmd)) + if !subCmdIsCheckCmd { + if path := c.GlobalString("log"); path != "" { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) + if err != nil { + return err + } + kataLog.Logger.Out = f + } - // Name for the root span (used for tracing) now the - // sub-command name is known. - traceRootSpan = name + " " + cmdName - } + switch c.GlobalString("log-format") { + case "text": + // retain logrus's default. + case "json": + kataLog.Logger.Formatter = new(logrus.JSONFormatter) + default: + return fmt.Errorf("unknown log-format %q", c.GlobalString("log-format")) + } - // Since a context is required, pass a new (throw-away) one - we - // cannot use the main context as tracing hasn't been enabled yet - // (meaning any spans created at this point will be silently ignored). - setExternalLoggers(context.Background(), kataLog) + // Add the name of the sub-command to each log entry for easier + // debugging. + cmdName := c.Args().First() + if c.App.Command(cmdName) != nil { + kataLog = kataLog.WithField("command", cmdName) - ignoreLogging := false + // Name for the root span (used for tracing) now the + // sub-command name is known. + traceRootSpan = name + " " + cmdName + } - if c.NArg() == 1 && c.Args()[0] == envCmd { - // simply report the logging setup - ignoreLogging = true + // Since a context is required, pass a new (throw-away) one - we + // cannot use the main context as tracing hasn't been enabled yet + // (meaning any spans created at this point will be silently ignored). + setExternalLoggers(context.Background(), kataLog) + + if c.NArg() == 1 && c.Args()[0] == envCmd { + // simply report the logging setup + ignoreLogging = true + } } configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false) @@ -309,19 +309,21 @@ func beforeSubcommands(c *cli.Context) error { fatal(err) } - debug = runtimeConfig.Debug - crashOnError = runtimeConfig.Debug + if !subCmdIsCheckCmd { + debug = runtimeConfig.Debug + crashOnError = runtimeConfig.Debug - if traceRootSpan != "" { - // Create the tracer. - // - // Note: no spans are created until the command-line has been parsed. - // This delays collection of trace data slightly but benefits the user by - // ensuring the first span is the name of the sub-command being - // invoked from the command-line. - err = setupTracing(c, traceRootSpan) - if err != nil { - return err + if traceRootSpan != "" { + // Create the tracer. + // + // Note: no spans are created until the command-line has been parsed. + // This delays collection of trace data slightly but benefits the user by + // ensuring the first span is the name of the sub-command being + // invoked from the command-line. + err = setupTracing(c, traceRootSpan) + if err != nil { + return err + } } } From 828e0a22050f906580c16b2c26cdfc47520d579b Mon Sep 17 00:00:00 2001 From: Vijay Dhanraj Date: Sat, 8 Jun 2019 13:46:05 -0700 Subject: [PATCH 3/6] pkg/katautils: Add support for ACRN hypervisor config This patch adds support for, 1. Extracting and configuring ACRN hypervisor from toml. 2. Add ACRN hypervisor ctl for controlling ACRN hypervisor. This will be used for updating virtio-blk based container rootfs using blk rescan feature. v2->v3: Fixed acrnctl path. v1->v2: Trimmed hypervisor config options as needed by ACRN. Fixes: #1778 Signed-off-by: Vijay Dhanraj --- pkg/katautils/config-settings.go | 1 + pkg/katautils/config.go | 77 ++++++++++++++++++++++++++++++++ virtcontainers/hypervisor.go | 6 +++ 3 files changed, 84 insertions(+) diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index 7c1333955..ac3c44c99 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -9,6 +9,7 @@ package katautils var defaultHypervisorPath = "/usr/bin/qemu-lite-system-x86_64" +var defaultHypervisorCtlPath = "/usr/bin/acrnctl" var defaultImagePath = "/usr/share/kata-containers/kata-containers.img" var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container" var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 64ee30985..a7faacf31 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -50,6 +50,7 @@ const ( // supported hypervisor component types firecrackerHypervisorTableType = "firecracker" qemuHypervisorTableType = "qemu" + acrnHypervisorTableType = "acrn" // supported proxy component types kataProxyTableType = "kata" @@ -84,6 +85,7 @@ type factory struct { type hypervisor struct { Path string `toml:"path"` Kernel string `toml:"kernel"` + CtlPath string `toml:"ctlpath"` Initrd string `toml:"initrd"` Image string `toml:"image"` Firmware string `toml:"firmware"` @@ -163,6 +165,16 @@ func (h hypervisor) path() (string, error) { return ResolvePath(p) } +func (h hypervisor) ctlpath() (string, error) { + p := h.CtlPath + + if h.CtlPath == "" { + p = defaultHypervisorCtlPath + } + + return ResolvePath(p) +} + func (h hypervisor) kernel() (string, error) { p := h.Kernel @@ -602,6 +614,67 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + hypervisor, err := h.path() + if err != nil { + return vc.HypervisorConfig{}, err + } + + hypervisorctl, err := h.ctlpath() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err + } + + image, err := h.image() + if err != nil { + return vc.HypervisorConfig{}, err + } + + if image == "" { + return vc.HypervisorConfig{}, + errors.New("image must be defined in the configuration file") + } + + firmware, err := h.firmware() + if err != nil { + return vc.HypervisorConfig{}, err + } + + kernelParams := h.kernelParams() + + blockDriver, err := h.blockDeviceDriver() + if err != nil { + return vc.HypervisorConfig{}, err + } + + return vc.HypervisorConfig{ + HypervisorPath: hypervisor, + KernelPath: kernel, + ImagePath: image, + HypervisorCtlPath: hypervisorctl, + FirmwarePath: firmware, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), + HugePages: h.HugePages, + Mlock: !h.Swap, + Debug: h.Debug, + DisableNestingChecks: h.DisableNestingChecks, + BlockDeviceDriver: blockDriver, + DisableVhostNet: h.DisableVhostNet, + GuestHookPath: h.guestHookPath(), + }, nil +} + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { if f.TemplatePath == "" { f.TemplatePath = defaultTemplatePath @@ -642,11 +715,15 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case qemuHypervisorTableType: config.HypervisorType = vc.QemuHypervisor hConfig, err = newQemuHypervisorConfig(hypervisor) + case acrnHypervisorTableType: + config.HypervisorType = vc.AcrnHypervisor + hConfig, err = newAcrnHypervisorConfig(hypervisor) } if err != nil { return fmt.Errorf("%v: %v", configPath, err) } + config.HypervisorConfig = hConfig } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 9ad51ff06..91b340748 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -31,6 +31,9 @@ const ( // QemuHypervisor is the QEMU hypervisor. QemuHypervisor HypervisorType = "qemu" + // AcrnHypervisor is the ACRN hypervisor. + AcrnHypervisor HypervisorType = "acrn" + // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" ) @@ -212,6 +215,9 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string + // HypervisorCtlPath is the hypervisor ctl executable host path. + HypervisorCtlPath string + // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string From d9a415784199da97b0db91c59dcc6b5f019cb7d6 Mon Sep 17 00:00:00 2001 From: Vijay Dhanraj Date: Sat, 8 Jun 2019 14:08:34 -0700 Subject: [PATCH 4/6] virtcontainers: Add support for launching/managing ACRN based VMs This patch adds the following, 1. Implement Sandbox management APIs for ACRN. 2. Implement Sandbox operation APIs for ACRN. 3. Add support for hot-plugging virtio-blk based (using blk rescan feature) container rootfs to ACRN. 4. Prime devices, image and kernel parameters for launching VM using ACRN. v2->v3: Incrementing index to keep track of virtio-blk devices created. This change removes the workaround introduced in block.go. v1->v2: 1. Created issue #1785 to address the UUID TODO item. 2. Removed dead code. 3. Fixed formatting of log messages. 4. Fixed year in copyright message. 5. Removed acrn_amd64.go file as there are no amd64 specific changes. Moved the code to acrn_arch_base.go. Fixes: #1778 Signed-off-by: Vijay Dhanraj --- virtcontainers/acrn.go | 621 +++++++++++++++++++++++++ virtcontainers/acrn_arch_base.go | 772 +++++++++++++++++++++++++++++++ virtcontainers/hypervisor.go | 14 + virtcontainers/types/asset.go | 3 + 4 files changed, 1410 insertions(+) create mode 100644 virtcontainers/acrn.go create mode 100644 virtcontainers/acrn_arch_base.go diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go new file mode 100644 index 000000000..aaf401b86 --- /dev/null +++ b/virtcontainers/acrn.go @@ -0,0 +1,621 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" + opentracing "github.com/opentracing/opentracing-go" + "github.com/sirupsen/logrus" +) + +// AcrnState keeps Acrn's state +type AcrnState struct { + UUID string +} + +// AcrnInfo keeps PID of the hypervisor +type AcrnInfo struct { + PID int +} + +// acrn is an Hypervisor interface implementation for the Linux acrn hypervisor. +type acrn struct { + id string + + store *store.VCStore + config HypervisorConfig + acrnConfig Config + state AcrnState + info AcrnInfo + arch acrnArch + ctx context.Context +} + +const ( + acrnConsoleSocket = "console.sock" + acrnStopSandboxTimeoutSecs = 15 +) + +// agnostic list of kernel parameters +var acrnDefaultKernelParameters = []Param{ + {"panic", "1"}, +} + +func (a *acrn) kernelParameters() string { + // get a list of arch kernel parameters + params := a.arch.kernelParameters(a.config.Debug) + + // use default parameters + params = append(params, acrnDefaultKernelParameters...) + + // set the maximum number of vCPUs + params = append(params, Param{"maxcpus", fmt.Sprintf("%d", a.config.NumVCPUs)}) + + // add the params specified by the provided config. As the kernel + // honours the last parameter value set and since the config-provided + // params are added here, they will take priority over the defaults. + params = append(params, a.config.KernelParams...) + + paramsStr := SerializeParams(params, "=") + + return strings.Join(paramsStr, " ") +} + +// Adds all capabilities supported by acrn implementation of hypervisor interface +func (a *acrn) capabilities() types.Capabilities { + span, _ := a.trace("capabilities") + defer span.Finish() + + return a.arch.capabilities() +} + +func (a *acrn) hypervisorConfig() HypervisorConfig { + return a.config +} + +// get the acrn binary path +func (a *acrn) acrnPath() (string, error) { + p, err := a.config.HypervisorAssetPath() + if err != nil { + return "", err + } + + if p == "" { + p, err = a.arch.acrnPath() + if err != nil { + return "", err + } + } + + if _, err = os.Stat(p); os.IsNotExist(err) { + return "", fmt.Errorf("acrn path (%s) does not exist", p) + } + + return p, nil +} + +// get the ACRNCTL binary path +func (a *acrn) acrnctlPath() (string, error) { + ctlpath, err := a.config.HypervisorCtlAssetPath() + if err != nil { + return "", err + } + + if ctlpath == "" { + ctlpath, err = a.arch.acrnctlPath() + if err != nil { + return "", err + } + } + + if _, err = os.Stat(ctlpath); os.IsNotExist(err) { + return "", fmt.Errorf("acrnctl path (%s) does not exist", ctlpath) + } + + return ctlpath, nil +} + +// Logger returns a logrus logger appropriate for logging acrn messages +func (a *acrn) Logger() *logrus.Entry { + return virtLog.WithField("subsystem", "acrn") +} + +func (a *acrn) trace(name string) (opentracing.Span, context.Context) { + if a.ctx == nil { + a.Logger().WithField("type", "bug").Error("trace called before context set") + a.ctx = context.Background() + } + + span, ctx := opentracing.StartSpanFromContext(a.ctx, name) + + span.SetTag("subsystem", "hypervisor") + span.SetTag("type", "acrn") + + return span, ctx +} + +func (a *acrn) memoryTopology() (Memory, error) { + memMb := uint64(a.config.MemorySize) + + return a.arch.memoryTopology(memMb), nil +} + +func (a *acrn) appendImage(devices []Device, imagePath string) ([]Device, error) { + if imagePath == "" { + return nil, fmt.Errorf("Image path is empty: %s", imagePath) + } + + // Get sandbox and increment the globalIndex. + // This is to make sure the VM rootfs occupies + // the first Index which is /dev/vda. + sandbox, err := globalSandboxList.lookupSandbox(a.id) + if sandbox == nil && err != nil { + return nil, err + } + sandbox.GetAndSetSandboxBlockIndex() + + devices, err = a.arch.appendImage(devices, imagePath) + if err != nil { + return nil, err + } + + return devices, nil +} + +func (a *acrn) buildDevices(imagePath string) ([]Device, error) { + var devices []Device + + if imagePath == "" { + return nil, fmt.Errorf("Image Path should not be empty: %s", imagePath) + } + + console, err := a.getSandboxConsole(a.id) + if err != nil { + return nil, err + } + + // Add bridges before any other devices. This way we make sure that + // bridge gets the first available PCI address. + devices = a.arch.appendBridges(devices) + + //Add LPC device to the list of other devices. + devices = a.arch.appendLPC(devices) + + devices = a.arch.appendConsole(devices, console) + + devices, err = a.appendImage(devices, imagePath) + if err != nil { + return nil, err + } + + // Create virtio blk devices with dummy backend as a place + // holder for container rootfs (as acrn doesn't support hot-plug). + // Once the container rootfs is known, replace the dummy backend + // with actual path (using block rescan feature in acrn) + devices, err = a.createDummyVirtioBlkDev(devices) + if err != nil { + return nil, err + } + + return devices, nil +} + +// setup sets the Acrn structure up. +func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { + span, _ := a.trace("setup") + defer span.Finish() + + err := hypervisorConfig.valid() + if err != nil { + return err + } + + a.id = id + a.store = vcStore + a.config = *hypervisorConfig + a.arch = newAcrnArch(a.config) + if err = a.store.Load(store.Hypervisor, &a.state); err != nil { + // acrn currently supports only known UUIDs for security + // reasons (FuSa). When launching VM, only these pre-defined + // UUID should be used else VM launch will fail. acrn team is + // working on a solution to expose these pre-defined UUIDs + // to Kata in order for it to launch VMs successfully. + // Until this support is available, Kata is limited to + // launch 1 VM (using the default UUID). + // https://github.com/kata-containers/runtime/issues/1785 + + // The path might already exist, but in case of VM templating, + // we have to create it since the sandbox has not created it yet. + if err = os.MkdirAll(store.SandboxRuntimeRootPath(id), store.DirMode); err != nil { + return err + } + + if err = a.store.Store(store.Hypervisor, a.state); err != nil { + return err + } + } + + if err = a.store.Load(store.Hypervisor, &a.info); err != nil { + a.Logger().WithField("function", "setup").WithError(err).Info("No info could be fetched") + } + + return nil +} + +func (a *acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) { + span, _ := a.trace("createDummyVirtioBlkDev") + defer span.Finish() + + // Since acrn doesn't support hot-plug, dummy virtio-blk + // devices are added and later replaced with container-rootfs. + // Starting from driveIndex 1, as 0 is allocated for VM rootfs. + for driveIndex := 1; driveIndex <= AcrnBlkDevPoolSz; driveIndex++ { + drive := config.BlockDrive{ + File: "nodisk", + Index: driveIndex, + } + + devices = a.arch.appendBlockDevice(devices, drive) + } + + return devices, nil +} + +// createSandbox is the Hypervisor sandbox creation. +func (a *acrn) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { + // Save the tracing context + a.ctx = ctx + + span, _ := a.trace("createSandbox") + defer span.Finish() + + if err := a.setup(id, hypervisorConfig, store); err != nil { + return err + } + + memory, err := a.memoryTopology() + if err != nil { + return err + } + + kernelPath, err := a.config.KernelAssetPath() + if err != nil { + return err + } + + imagePath, err := a.config.ImageAssetPath() + if err != nil { + return err + } + + kernel := Kernel{ + Path: kernelPath, + ImagePath: imagePath, + Params: a.kernelParameters(), + } + + // Disabling UUID check until the below is fixed. + // https://github.com/kata-containers/runtime/issues/1785 + + devices, err := a.buildDevices(imagePath) + if err != nil { + return err + } + + acrnPath, err := a.acrnPath() + if err != nil { + return err + } + + acrnctlPath, err := a.acrnctlPath() + if err != nil { + return err + } + + acrnConfig := Config{ + ACPIVirt: true, + Path: acrnPath, + CtlPath: acrnctlPath, + Memory: memory, + NumCPU: a.config.NumVCPUs, + Devices: devices, + Kernel: kernel, + Name: fmt.Sprintf("sandbox-%s", a.id), + } + + a.acrnConfig = acrnConfig + + return nil +} + +// startSandbox will start the Sandbox's VM. +func (a *acrn) startSandbox(timeoutSecs int) error { + span, _ := a.trace("startSandbox") + defer span.Finish() + + if a.config.Debug { + params := a.arch.kernelParameters(a.config.Debug) + strParams := SerializeParams(params, "=") + formatted := strings.Join(strParams, " ") + + // The name of this field matches a similar one generated by + // the runtime and allows users to identify which parameters + // are set here, which come from the runtime and which are set + // by the user. + a.Logger().WithField("default-kernel-parameters", formatted).Debug() + } + + vmPath := filepath.Join(store.RunVMStoragePath, a.id) + err := os.MkdirAll(vmPath, store.DirMode) + if err != nil { + return err + } + defer func() { + if err != nil { + if err := os.RemoveAll(vmPath); err != nil { + a.Logger().WithError(err).Error("Failed to clean up vm directory") + } + } + }() + + var strErr string + var PID int + PID, strErr, err = LaunchAcrn(a.acrnConfig, virtLog.WithField("subsystem", "acrn-dm")) + if err != nil { + return fmt.Errorf("%s", strErr) + } + a.info.PID = PID + + if err = a.waitSandbox(timeoutSecs); err != nil { + a.Logger().WithField("acrn wait failed:", err).Debug() + return err + } + + //Store VMM information + return a.store.Store(store.Hypervisor, a.info) + +} + +// waitSandbox will wait for the Sandbox's VM to be up and running. +func (a *acrn) waitSandbox(timeoutSecs int) error { + span, _ := a.trace("waitSandbox") + defer span.Finish() + + if timeoutSecs < 0 { + return fmt.Errorf("Invalid timeout %ds", timeoutSecs) + } + + time.Sleep(time.Duration(timeoutSecs) * time.Second) + + return nil +} + +// stopSandbox will stop the Sandbox's VM. +func (a *acrn) stopSandbox() (err error) { + span, _ := a.trace("stopSandbox") + defer span.Finish() + + a.Logger().Info("Stopping acrn VM") + + defer func() { + if err != nil { + a.Logger().Info("stopSandbox failed") + } else { + a.Logger().Info("acrn VM stopped") + } + }() + + pid := a.info.PID + + // Check if VM process is running, in case it is not, let's + // return from here. + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + a.Logger().Info("acrn VM already stopped") + return nil + } + + // Send signal to the VM process to try to stop it properly + if err = syscall.Kill(pid, syscall.SIGINT); err != nil { + a.Logger().Info("Sending signal to stop acrn VM failed") + return err + } + + // Wait for the VM process to terminate + tInit := time.Now() + for { + if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { + a.Logger().Info("acrn VM stopped after sending signal") + return nil + } + + if time.Since(tInit).Seconds() >= acrnStopSandboxTimeoutSecs { + a.Logger().Warnf("VM still running after waiting %ds", acrnStopSandboxTimeoutSecs) + break + } + + // Let's avoid to run a too busy loop + time.Sleep(time.Duration(50) * time.Millisecond) + } + + // Let's try with a hammer now, a SIGKILL should get rid of the + // VM process. + return syscall.Kill(pid, syscall.SIGKILL) + +} + +func (a *acrn) updateBlockDevice(drive *config.BlockDrive) error { + var err error + if drive.File == "" || drive.Index >= AcrnBlkDevPoolSz { + return fmt.Errorf("Empty filepath or invalid drive index, Dive ID:%s, Drive Index:%d", + drive.ID, drive.Index) + } + + slot := AcrnBlkdDevSlot[drive.Index] + + //Explicitly set PCIAddr to NULL, so that VirtPath can be used + drive.PCIAddr = "" + + args := []string{"blkrescan", a.acrnConfig.Name, fmt.Sprintf("%d,%s", slot, drive.File)} + + a.Logger().WithFields(logrus.Fields{ + "drive": drive, + "path": a.config.HypervisorCtlPath, + }).Info("updateBlockDevice with acrnctl path") + cmd := exec.Command(a.config.HypervisorCtlPath, args...) + if err := cmd.Run(); err != nil { + a.Logger().WithError(err).Error("updating Block device with newFile path") + } + + return err +} + +func (a *acrn) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + span, _ := a.trace("hotplugAddDevice") + defer span.Finish() + + switch devType { + case blockDev: + //The drive placeholder has to exist prior to Update + return nil, a.updateBlockDevice(devInfo.(*config.BlockDrive)) + default: + return nil, fmt.Errorf("hotplugAddDevice: unsupported device: devInfo:%v, deviceType%v", + devInfo, devType) + } +} + +func (a *acrn) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { + span, _ := a.trace("hotplugRemoveDevice") + defer span.Finish() + + // Not supported. return success + + return nil, nil +} + +func (a *acrn) pauseSandbox() error { + span, _ := a.trace("pauseSandbox") + defer span.Finish() + + // Not supported. return success + + return nil +} + +func (a *acrn) resumeSandbox() error { + span, _ := a.trace("resumeSandbox") + defer span.Finish() + + // Not supported. return success + + return nil +} + +// addDevice will add extra devices to Acrn command line. +func (a *acrn) addDevice(devInfo interface{}, devType deviceType) error { + var err error + span, _ := a.trace("addDevice") + defer span.Finish() + + switch v := devInfo.(type) { + case types.Volume: + // Not supported. return success + err = nil + case types.Socket: + a.acrnConfig.Devices = a.arch.appendSocket(a.acrnConfig.Devices, v) + case kataVSOCK: + // Not supported. return success + err = nil + case Endpoint: + a.acrnConfig.Devices = a.arch.appendNetwork(a.acrnConfig.Devices, v) + case config.BlockDrive: + a.acrnConfig.Devices = a.arch.appendBlockDevice(a.acrnConfig.Devices, v) + case config.VhostUserDeviceAttrs: + // Not supported. return success + err = nil + case config.VFIODev: + // Not supported. return success + err = nil + default: + err = nil + a.Logger().WithField("unknown-device-type", devInfo).Error("Adding device") + } + + return err +} + +// getSandboxConsole builds the path of the console where we can read +// logs coming from the sandbox. +func (a *acrn) getSandboxConsole(id string) (string, error) { + span, _ := a.trace("getSandboxConsole") + defer span.Finish() + + return utils.BuildSocketPath(store.RunVMStoragePath, id, acrnConsoleSocket) +} + +func (a *acrn) saveSandbox() error { + a.Logger().Info("save sandbox") + + // Not supported. return success + + return nil +} + +func (a *acrn) disconnect() { + span, _ := a.trace("disconnect") + defer span.Finish() + + // Not supported. +} + +func (a *acrn) getThreadIDs() (vcpuThreadIDs, error) { + span, _ := a.trace("getThreadIDs") + defer span.Finish() + + // Not supported. return success + //Just allocating an empty map + + return vcpuThreadIDs{}, nil +} + +func (a *acrn) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil +} + +func (a *acrn) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { + return 0, 0, nil +} + +func (a *acrn) cleanup() error { + span, _ := a.trace("cleanup") + defer span.Finish() + + return nil +} + +func (a *acrn) pid() int { + return a.info.PID +} + +func (a *acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("acrn is not supported by VM cache") +} + +func (a *acrn) toGrpc() ([]byte, error) { + return nil, errors.New("acrn is not supported by VM cache") +} diff --git a/virtcontainers/acrn_arch_base.go b/virtcontainers/acrn_arch_base.go new file mode 100644 index 000000000..a9882576a --- /dev/null +++ b/virtcontainers/acrn_arch_base.go @@ -0,0 +1,772 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" +) + +type acrnArch interface { + + // acrnPath returns the path to the acrn binary + acrnPath() (string, error) + + // acrnctlPath returns the path to the acrnctl binary + acrnctlPath() (string, error) + + // kernelParameters returns the kernel parameters + // if debug is true then kernel debug parameters are included + kernelParameters(debug bool) []Param + + //capabilities returns the capabilities supported by acrn + capabilities() types.Capabilities + + // memoryTopology returns the memory topology using the given amount of memoryMb and hostMemoryMb + memoryTopology(memMb uint64) Memory + + // appendConsole appends a console to devices + appendConsole(devices []Device, path string) []Device + + // appendImage appends an image to devices + appendImage(devices []Device, path string) ([]Device, error) + + // appendBridges appends bridges to devices + appendBridges(devices []Device) []Device + + // appendLPC appends LPC to devices + // UART device emulated by the acrn-dm is connected to the system by the LPC bus + appendLPC(devices []Device) []Device + + // appendSocket appends a socket to devices + appendSocket(devices []Device, socket types.Socket) []Device + + // appendNetwork appends a endpoint device to devices + appendNetwork(devices []Device, endpoint Endpoint) []Device + + // appendBlockDevice appends a block drive to devices + appendBlockDevice(devices []Device, drive config.BlockDrive) []Device + + // handleImagePath handles the Hypervisor Config image path + handleImagePath(config HypervisorConfig) +} + +type acrnArchBase struct { + path string + ctlpath string + kernelParamsNonDebug []Param + kernelParamsDebug []Param + kernelParams []Param +} + +const acrnPath = "/usr/bin/acrn-dm" +const acrnctlPath = "/usr/bin/acrnctl" + +// acrn GVT-g slot is harded code to 2 as there is +// no simple way to pass arguments of PCI slots from +// device model (acrn-dm) to ACRNGT module. +const acrnGVTgReservedSlot = 2 + +const acrnLPCDev = "lpc" +const acrnHostBridge = "hostbridge" + +var baselogger *logrus.Entry + +// AcrnBlkDevPoolSz defines the number of dummy virtio-blk +// device that will be created for hot-plugging container +// rootfs. Since acrn doesn't support hot-plug, dummy virtio-blk +// devices are added and later replaced with container-rootfs. +var AcrnBlkDevPoolSz = 8 + +// AcrnBlkdDevSlot array provides translation between +// the vitio-blk device index and slot it is currently +// attached. +// Allocating extra 1 to accommodate for VM rootfs +// which is at driveIndex 0 +var AcrnBlkdDevSlot = make([]int, AcrnBlkDevPoolSz+1) + +// acrnKernelParamsNonDebug is a list of the default kernel +// parameters that will be used in standard (non-debug) mode. +var acrnKernelParamsNonDebug = []Param{ + {"quiet", ""}, +} + +// acrnKernelParamsSystemdNonDebug is a list of the default systemd related +// kernel parameters that will be used in standard (non-debug) mode. +var acrnKernelParamsSystemdNonDebug = []Param{ + {"systemd.show_status", "false"}, +} + +// acrnKernelParamsDebug is a list of the default kernel +// parameters that will be used in debug mode (as much boot output as +// possible). +var acrnKernelParamsDebug = []Param{ + {"debug", ""}, +} + +// acrnKernelParamsSystemdDebug is a list of the default systemd related kernel +// parameters that will be used in debug mode (as much boot output as +// possible). +var acrnKernelParamsSystemdDebug = []Param{ + {"systemd.show_status", "true"}, + {"systemd.log_level", "debug"}, + {"systemd.log_target", "kmsg"}, + {"printk.devkmsg", "on"}, +} + +var acrnKernelRootParams = []Param{ + {"root", "/dev/vda1 rw rootwait"}, +} + +var acrnKernelParams = []Param{ + {"tsc", "reliable"}, + {"no_timer_check", ""}, + {"nohpet", ""}, + {"console", "tty0"}, + {"console", "ttyS0"}, + {"console", "hvc0"}, + {"log_buf_len", "16M"}, + {"consoleblank", "0"}, + {"iommu", "off"}, + {"i915.avail_planes_per_pipe", "0x070F00"}, + {"i915.enable_hangcheck", "0"}, + {"i915.nuclear_pageflip", "1"}, + {"i915.enable_guc_loading", "0"}, + {"i915.enable_guc_submission", "0"}, + {"i915.enable_guc", "0"}, +} + +// Device is the acrn device interface. +type Device interface { + Valid() bool + AcrnParams(slot int, config *Config) []string +} + +// ConsoleDeviceBackend is the character device backend for acrn +type ConsoleDeviceBackend string + +const ( + + // Socket creates a 2 way stream socket (TCP or Unix). + Socket ConsoleDeviceBackend = "socket" + + // Stdio sends traffic from the guest to acrn's standard output. + Stdio ConsoleDeviceBackend = "console" + + // File backend only supports console output to a file (no input). + File ConsoleDeviceBackend = "file" + + // TTY is an alias for Serial. + TTY ConsoleDeviceBackend = "tty" + + // PTY creates a new pseudo-terminal on the host and connect to it. + PTY ConsoleDeviceBackend = "pty" +) + +// BEPortType marks the port as console port or virtio-serial port +type BEPortType int + +const ( + // SerialBE marks the port as serial port + SerialBE BEPortType = iota + + //ConsoleBE marks the port as console port (append @) + ConsoleBE +) + +// ConsoleDevice represents a acrn console device. +type ConsoleDevice struct { + // Name of the socket + Name string + + //Backend device used for virtio-console + Backend ConsoleDeviceBackend + + // PortType marks the port as serial or console port (@) + PortType BEPortType + + //Path to virtio-console backend (can be omitted for pty, tty, stdio) + Path string +} + +// NetDeviceType is a acrn networking device type. +type NetDeviceType string + +const ( + // TAP is a TAP networking device type. + TAP NetDeviceType = "tap" + + // MACVTAP is a macvtap networking device type. + MACVTAP NetDeviceType = "macvtap" +) + +// NetDevice represents a guest networking device +type NetDevice struct { + // Type is the netdev type (e.g. tap). + Type NetDeviceType + + // IfName is the interface name + IFName string + + //MACAddress is the networking device interface MAC address + MACAddress string +} + +// BlockDevice represents a acrn block device. +type BlockDevice struct { + + // mem path to block device + FilePath string + + //BlkIndex - Blk index corresponding to slot + Index int +} + +// BridgeDevice represents a acrn bridge device like pci-bridge, pxb, etc. +type BridgeDevice struct { + + // Function is PCI function. Func can be from 0 to 7 + Function int + + // Emul is a string describing the type of PCI device e.g. virtio-net + Emul string + + // Config is an optional string, depending on the device, that can be + // used for configuration + Config string +} + +// LPCDevice represents a acrn LPC device +type LPCDevice struct { + + // Function is PCI function. Func can be from 0 to 7 + Function int + + // Emul is a string describing the type of PCI device e.g. virtio-net + Emul string +} + +// Memory is the guest memory configuration structure. +type Memory struct { + // Size is the amount of memory made available to the guest. + // It should be suffixed with M or G for sizes in megabytes or + // gigabytes respectively. + Size string +} + +// Kernel is the guest kernel configuration structure. +type Kernel struct { + // Path is the guest kernel path on the host filesystem. + Path string + + // InitrdPath is the guest initrd path on the host filesystem. + ImagePath string + + // Params is the kernel parameters string. + Params string +} + +// Config is the acrn configuration structure. +// It allows for passing custom settings and parameters to the acrn-dm API. +type Config struct { + + // Path is the acrn binary path. + Path string + + // Path is the acrn binary path. + CtlPath string + + // Name is the acrn guest name + Name string + + // UUID is the acrn process UUID. + UUID string + + // Devices is a list of devices for acrn to create and drive. + Devices []Device + + // Kernel is the guest kernel configuration. + Kernel Kernel + + // Memory is the guest memory configuration. + Memory Memory + + acrnParams []string + + // ACPI virtualization support + ACPIVirt bool + + // NumCPU is the number of CPUs for guest + NumCPU uint32 +} + +// MaxAcrnVCPUs returns the maximum number of vCPUs supported +func MaxAcrnVCPUs() uint32 { + return uint32(8) +} + +func newAcrnArch(config HypervisorConfig) acrnArch { + a := &acrnArchBase{ + path: acrnPath, + ctlpath: acrnctlPath, + kernelParamsNonDebug: acrnKernelParamsNonDebug, + kernelParamsDebug: acrnKernelParamsDebug, + kernelParams: acrnKernelParams, + } + + a.handleImagePath(config) + return a +} + +func (a *acrnArchBase) acrnPath() (string, error) { + p := a.path + return p, nil +} + +func (a *acrnArchBase) acrnctlPath() (string, error) { + ctlpath := a.ctlpath + return ctlpath, nil +} + +func (a *acrnArchBase) kernelParameters(debug bool) []Param { + params := a.kernelParams + + if debug { + params = append(params, a.kernelParamsDebug...) + } else { + params = append(params, a.kernelParamsNonDebug...) + } + + return params +} + +func (a *acrnArchBase) memoryTopology(memoryMb uint64) Memory { + mem := fmt.Sprintf("%dM", memoryMb) + memory := Memory{ + Size: mem, + } + + return memory +} + +func (a *acrnArchBase) capabilities() types.Capabilities { + var caps types.Capabilities + + // For devicemapper disable support for filesystem sharing + caps.SetFsSharingUnsupported() + caps.SetBlockDeviceSupport() + caps.SetBlockDeviceHotplugSupport() + + return caps +} + +// Valid returns true if the CharDevice structure is valid and complete. +func (cdev ConsoleDevice) Valid() bool { + if cdev.Backend != "tty" && cdev.Backend != "pty" && + cdev.Backend != "console" && cdev.Backend != "socket" && + cdev.Backend != "file" { + return false + } else if cdev.PortType != ConsoleBE && cdev.PortType != SerialBE { + return false + } else if cdev.Path == "" { + return false + } else { + return true + } +} + +// AcrnParams returns the acrn parameters built out of this console device. +func (cdev ConsoleDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + var deviceParams []string + + acrnParams = append(acrnParams, "-s") + deviceParams = append(deviceParams, fmt.Sprintf("%d,virtio-console,", slot)) + + if cdev.PortType == ConsoleBE { + deviceParams = append(deviceParams, "@") + } + + switch cdev.Backend { + case "pty": + deviceParams = append(deviceParams, "pty:pty_port") + case "tty": + deviceParams = append(deviceParams, fmt.Sprintf("tty:tty_port=%s", cdev.Path)) + case "socket": + deviceParams = append(deviceParams, fmt.Sprintf("socket:%s=%s", cdev.Name, cdev.Path)) + case "file": + deviceParams = append(deviceParams, fmt.Sprintf("file:file_port=%s", cdev.Path)) + case "stdio": + deviceParams = append(deviceParams, "stdio:stdio_port") + default: + // do nothing. Error should be already caught + } + + acrnParams = append(acrnParams, strings.Join(deviceParams, "")) + return acrnParams +} + +// AcrnNetdevParam converts to the acrn type to string +func (netdev NetDevice) AcrnNetdevParam() []string { + var deviceParams []string + + switch netdev.Type { + case TAP: + deviceParams = append(deviceParams, netdev.IFName) + deviceParams = append(deviceParams, fmt.Sprintf(",mac=%s", netdev.MACAddress)) + case MACVTAP: + deviceParams = append(deviceParams, netdev.IFName) + default: + deviceParams = append(deviceParams, netdev.IFName) + + } + + return deviceParams +} + +// Valid returns true if the NetDevice structure is valid and complete. +func (netdev NetDevice) Valid() bool { + if netdev.IFName == "" { + return false + } else if netdev.MACAddress == "" { + return false + } else if netdev.Type != TAP && netdev.Type != MACVTAP { + return false + } else { + return true + } +} + +// AcrnParams returns the acrn parameters built out of this network device. +func (netdev NetDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,virtio-net,%s", slot, strings.Join(netdev.AcrnNetdevParam(), ""))) + + return acrnParams +} + +// Valid returns true if the BlockDevice structure is valid and complete. +func (blkdev BlockDevice) Valid() bool { + return blkdev.FilePath != "" +} + +// AcrnParams returns the acrn parameters built out of this block device. +func (blkdev BlockDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + device := "virtio-blk" + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d,%s,%s", + slot, device, blkdev.FilePath)) + + // Update the global array (BlkIndex<->slot) + // Used to identify slots for the hot-plugged virtio-blk devices + if blkdev.Index <= AcrnBlkDevPoolSz { + AcrnBlkdDevSlot[blkdev.Index] = slot + } else { + baselogger.WithFields(logrus.Fields{ + "device": device, + "index": blkdev.Index, + }).Info("Invalid index device") + } + + return acrnParams +} + +// Valid returns true if the BridgeDevice structure is valid and complete. +func (bridgeDev BridgeDevice) Valid() bool { + if bridgeDev.Function != 0 || bridgeDev.Emul != acrnHostBridge { + return false + } + return true +} + +// AcrnParams returns the acrn parameters built out of this bridge device. +func (bridgeDev BridgeDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d:%d,%s", slot, + bridgeDev.Function, bridgeDev.Emul)) + + return acrnParams +} + +// Valid returns true if the BridgeDevice structure is valid and complete. +func (lpcDev LPCDevice) Valid() bool { + return lpcDev.Emul == acrnLPCDev +} + +// AcrnParams returns the acrn parameters built out of this bridge device. +func (lpcDev LPCDevice) AcrnParams(slot int, config *Config) []string { + var acrnParams []string + var deviceParams []string + + acrnParams = append(acrnParams, "-s") + acrnParams = append(acrnParams, fmt.Sprintf("%d:%d,%s", slot, + lpcDev.Function, lpcDev.Emul)) + + //define UART port + deviceParams = append(deviceParams, "-l") + deviceParams = append(deviceParams, "com1,stdio") + acrnParams = append(acrnParams, strings.Join(deviceParams, "")) + + return acrnParams +} + +func (config *Config) appendName() { + if config.Name != "" { + config.acrnParams = append(config.acrnParams, config.Name) + } +} + +func (config *Config) appendDevices() { + slot := 0 + for _, d := range config.Devices { + if !d.Valid() { + continue + } + + if slot == acrnGVTgReservedSlot { + slot++ /*Slot 2 is assigned for GVT-g in acrn, so skip 2 */ + baselogger.Info("Slot 2 is assigned for GVT-g in acrn, so skipping this slot") + + } + config.acrnParams = append(config.acrnParams, d.AcrnParams(slot, config)...) + slot++ + } +} + +func (config *Config) appendUUID() { + if config.UUID != "" { + config.acrnParams = append(config.acrnParams, "-U") + config.acrnParams = append(config.acrnParams, config.UUID) + } +} + +func (config *Config) appendACPI() { + if config.ACPIVirt { + config.acrnParams = append(config.acrnParams, "-A") + } +} + +func (config *Config) appendMemory() { + if config.Memory.Size != "" { + config.acrnParams = append(config.acrnParams, "-m") + config.acrnParams = append(config.acrnParams, config.Memory.Size) + } +} + +func (config *Config) appendCPUs() { + if config.NumCPU != 0 { + config.acrnParams = append(config.acrnParams, "-c") + config.acrnParams = append(config.acrnParams, fmt.Sprintf("%d", config.NumCPU)) + } + +} + +func (config *Config) appendKernel() { + if config.Kernel.Path == "" { + return + } + config.acrnParams = append(config.acrnParams, "-k") + config.acrnParams = append(config.acrnParams, config.Kernel.Path) + + if config.Kernel.Params == "" { + return + } + config.acrnParams = append(config.acrnParams, "-B") + config.acrnParams = append(config.acrnParams, config.Kernel.Params) +} + +// LaunchAcrn can be used to launch a new acrn instance. +// +// The Config parameter contains a set of acrn parameters and settings. +// +// This function writes its log output via logger parameter. +func LaunchAcrn(config Config, logger *logrus.Entry) (int, string, error) { + baselogger = logger + config.appendUUID() + config.appendACPI() + config.appendMemory() + config.appendCPUs() + config.appendDevices() + config.appendKernel() + config.appendName() + + return LaunchCustomAcrn(context.Background(), config.Path, config.acrnParams, logger) +} + +// LaunchCustomAcrn can be used to launch a new acrn instance. +// +// The path parameter is used to pass the acrn executable path. +// +// params is a slice of options to pass to acrn-dm +// +// This function writes its log output via logger parameter. +func LaunchCustomAcrn(ctx context.Context, path string, params []string, + logger *logrus.Entry) (int, string, error) { + + errStr := "" + + if path == "" { + path = "acrn-dm" + } + + /* #nosec */ + cmd := exec.CommandContext(ctx, path, params...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + logger.WithFields(logrus.Fields{ + "Path": path, + "Params": params, + }).Info("launching acrn with:") + + err := cmd.Start() + if err != nil { + logger.Errorf("Unable to launch %s: %v", path, err) + errStr = stderr.String() + logger.Errorf("%s", errStr) + } + return cmd.Process.Pid, errStr, err +} + +func (a *acrnArchBase) appendImage(devices []Device, path string) ([]Device, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, err + } + + ImgBlkdevice := BlockDevice{ + FilePath: path, + Index: 0, + } + + devices = append(devices, ImgBlkdevice) + + return devices, nil +} + +// appendBridges appends to devices the given bridges +func (a *acrnArchBase) appendBridges(devices []Device) []Device { + devices = append(devices, + BridgeDevice{ + Function: 0, + Emul: acrnHostBridge, + Config: "", + }, + ) + + return devices +} + +// appendBridges appends to devices the given bridges +func (a *acrnArchBase) appendLPC(devices []Device) []Device { + devices = append(devices, + LPCDevice{ + Function: 0, + Emul: acrnLPCDev, + }, + ) + + return devices +} + +func (a *acrnArchBase) appendConsole(devices []Device, path string) []Device { + console := ConsoleDevice{ + Name: "console0", + Backend: Socket, + PortType: ConsoleBE, + Path: path, + } + + devices = append(devices, console) + return devices +} + +func (a *acrnArchBase) appendSocket(devices []Device, socket types.Socket) []Device { + serailsocket := ConsoleDevice{ + Name: socket.Name, + Backend: Socket, + PortType: SerialBE, + Path: socket.HostPath, + } + + devices = append(devices, serailsocket) + return devices +} + +func networkModelToAcrnType(model NetInterworkingModel) NetDeviceType { + switch model { + case NetXConnectBridgedModel: + return TAP + case NetXConnectMacVtapModel: + return MACVTAP + default: + //TAP should work for most other cases + return TAP + } +} + +func (a *acrnArchBase) appendNetwork(devices []Device, endpoint Endpoint) []Device { + switch ep := endpoint.(type) { + case *VethEndpoint: + netPair := ep.NetworkPair() + devices = append(devices, + NetDevice{ + Type: networkModelToAcrnType(netPair.NetInterworkingModel), + IFName: netPair.TAPIface.Name, + MACAddress: netPair.TAPIface.HardAddr, + }, + ) + case *MacvtapEndpoint: + devices = append(devices, + NetDevice{ + Type: MACVTAP, + IFName: ep.Name(), + MACAddress: ep.HardwareAddr(), + }, + ) + default: + // Return devices as is for unsupported endpoint. + baselogger.WithField("Endpoint", endpoint).Error("Unsupported N/W Endpoint") + } + + return devices +} + +func (a *acrnArchBase) appendBlockDevice(devices []Device, drive config.BlockDrive) []Device { + if drive.File == "" { + return devices + } + + devices = append(devices, + BlockDevice{ + FilePath: drive.File, + Index: drive.Index, + }, + ) + + return devices +} + +func (a *acrnArchBase) handleImagePath(config HypervisorConfig) { + if config.ImagePath != "" { + a.kernelParams = append(a.kernelParams, acrnKernelRootParams...) + a.kernelParamsNonDebug = append(a.kernelParamsNonDebug, acrnKernelParamsSystemdNonDebug...) + a.kernelParamsDebug = append(a.kernelParamsDebug, acrnKernelParamsSystemdDebug...) + } +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 91b340748..15ce20e93 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -121,6 +121,9 @@ func (hType *HypervisorType) Set(value string) error { case "firecracker": *hType = FirecrackerHypervisor return nil + case "acrn": + *hType = AcrnHypervisor + return nil case "mock": *hType = MockHypervisor return nil @@ -136,6 +139,8 @@ func (hType *HypervisorType) String() string { return string(QemuHypervisor) case FirecrackerHypervisor: return string(FirecrackerHypervisor) + case AcrnHypervisor: + return string(AcrnHypervisor) case MockHypervisor: return string(MockHypervisor) default: @@ -150,6 +155,8 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { return &qemu{}, nil case FirecrackerHypervisor: return &firecracker{}, nil + case AcrnHypervisor: + return &acrn{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: @@ -436,6 +443,8 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) { return conf.InitrdPath, nil case types.HypervisorAsset: return conf.HypervisorPath, nil + case types.HypervisorCtlAsset: + return conf.HypervisorCtlPath, nil case types.FirmwareAsset: return conf.FirmwarePath, nil default: @@ -483,6 +492,11 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } +// HypervisorCtlAssetPath returns the VM hypervisor ctl path +func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) { + return conf.assetPath(types.HypervisorCtlAsset) +} + // CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise. func (conf *HypervisorConfig) CustomHypervisorAsset() bool { return conf.isCustomAsset(types.HypervisorAsset) diff --git a/virtcontainers/types/asset.go b/virtcontainers/types/asset.go index dd9bab3b7..7c9afd374 100644 --- a/virtcontainers/types/asset.go +++ b/virtcontainers/types/asset.go @@ -49,6 +49,9 @@ const ( // HypervisorAsset is an hypervisor asset. HypervisorAsset AssetType = "hypervisor" + // HypervisorCtlAsset is an hypervisor control asset. + HypervisorCtlAsset AssetType = "hypervisorctl" + // FirmwareAsset is a firmware asset. FirmwareAsset AssetType = "firmware" ) From f246a799aa2c71f2385fd859115705a991d0d1d3 Mon Sep 17 00:00:00 2001 From: Vijay Dhanraj Date: Tue, 11 Jun 2019 09:46:08 -0700 Subject: [PATCH 5/6] virtcontainers: Add support for updating virtio-blk based container rootfs Thist patch adds the following, 1. ACRN only supports virtio-blk and so the rootfs for the VM sits at /dev/vda. So to get the container rootfs increment the globalIndex by 1. 2. ACRN doesn't hot-plug container rootfs (but uses blkrescan) to update the container rootfs. So the agent can be provided the virtpath rather than the PCIaddr avoiding unneccessary rescaning to find the virthpath. v1->v2: Removed the workaround of incrementing index for virtio-blk device and addressed it acrn. Fixes: #1778 Signed-off-by: Vijay Dhanraj --- virtcontainers/device/api/interface.go | 4 ++-- virtcontainers/device/api/mockDeviceReceiver.go | 5 +++++ virtcontainers/kata_agent.go | 12 ++++++++---- virtcontainers/sandbox.go | 17 +++++++++++------ 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go index 828049c6f..f4060c046 100644 --- a/virtcontainers/device/api/interface.go +++ b/virtcontainers/device/api/interface.go @@ -7,10 +7,9 @@ package api import ( - "github.com/sirupsen/logrus" - "github.com/kata-containers/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/sirupsen/logrus" ) var devLogger = logrus.WithField("subsystem", "device") @@ -36,6 +35,7 @@ type DeviceReceiver interface { // this is only for virtio-blk and virtio-scsi support GetAndSetSandboxBlockIndex() (int, error) DecrementSandboxBlockIndex() error + GetHypervisorType() string // this is for appending device to hypervisor boot params AppendDevice(Device) error diff --git a/virtcontainers/device/api/mockDeviceReceiver.go b/virtcontainers/device/api/mockDeviceReceiver.go index c080d072d..e7d937ded 100644 --- a/virtcontainers/device/api/mockDeviceReceiver.go +++ b/virtcontainers/device/api/mockDeviceReceiver.go @@ -36,3 +36,8 @@ func (mockDC *MockDeviceReceiver) DecrementSandboxBlockIndex() error { func (mockDC *MockDeviceReceiver) AppendDevice(Device) error { return nil } + +// GetHypervisorType is used for getting Hypervisor name currently used. +func (mockDC *MockDeviceReceiver) GetHypervisorType() string { + return "" +} diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 61e739d17..d11f65c21 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "github.com/gogo/protobuf/proto" aTypes "github.com/kata-containers/agent/pkg/types" kataclient "github.com/kata-containers/agent/protocols/client" "github.com/kata-containers/agent/protocols/grpc" @@ -30,10 +31,8 @@ import ( "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" - opentracing "github.com/opentracing/opentracing-go" - - "github.com/gogo/protobuf/proto" "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/net/context" @@ -1084,7 +1083,12 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat rootfs.Source = blockDrive.VirtPath } else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock { rootfs.Driver = kataBlkDevType - rootfs.Source = blockDrive.PCIAddr + if blockDrive.PCIAddr == "" { + rootfs.Source = blockDrive.VirtPath + } else { + rootfs.Source = blockDrive.PCIAddr + } + } else { rootfs.Driver = kataSCSIDevType rootfs.Source = blockDrive.SCSIAddr diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 8be04e9c0..e44e103e7 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -15,12 +15,6 @@ import ( "syscall" "github.com/containernetworking/plugins/pkg/ns" - specs "github.com/opencontainers/runtime-spec/specs-go" - opentracing "github.com/opentracing/opentracing-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" - "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" @@ -34,6 +28,11 @@ import ( "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" ) const ( @@ -1842,3 +1841,9 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 { } return utils.CalculateVCpusFromMilliCpus(mCPU) } + +// GetHypervisorType is used for getting Hypervisor name currently used. +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) GetHypervisorType() string { + return string(s.config.HypervisorType) +} From 98a69736c5b30bd9ff0e84902dfb368dd8286a20 Mon Sep 17 00:00:00 2001 From: Vijay Dhanraj Date: Thu, 27 Jun 2019 10:41:45 -0700 Subject: [PATCH 6/6] virtcontainers: Add ACRN unit test cases This patch adds unit test cases for acrn specific changes. Fixes: #1778 Signed-off-by: Vijay Dhanraj --- virtcontainers/acrn_arch_base_test.go | 299 ++++++++++++++++++++++++++ virtcontainers/acrn_test.go | 285 ++++++++++++++++++++++++ virtcontainers/virtcontainers_test.go | 24 +++ 3 files changed, 608 insertions(+) create mode 100644 virtcontainers/acrn_arch_base_test.go create mode 100644 virtcontainers/acrn_test.go diff --git a/virtcontainers/acrn_arch_base_test.go b/virtcontainers/acrn_arch_base_test.go new file mode 100644 index 000000000..1ba57ac63 --- /dev/null +++ b/virtcontainers/acrn_arch_base_test.go @@ -0,0 +1,299 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "io/ioutil" + "net" + "path/filepath" + "testing" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" +) + +const ( + acrnArchBaseAcrnPath = "/usr/bin/acrn" + acrnArchBaseAcrnCtlPath = "/usr/bin/acrnctl" +) + +var acrnArchBaseKernelParamsNonDebug = []Param{ + {"quiet", ""}, +} + +var acrnArchBaseKernelParamsDebug = []Param{ + {"debug", ""}, +} + +var acrnArchBaseKernelParams = []Param{ + {"root", "/dev/vda"}, +} + +func newAcrnArchBase() *acrnArchBase { + return &acrnArchBase{ + path: acrnArchBaseAcrnPath, + ctlpath: acrnArchBaseAcrnCtlPath, + kernelParamsNonDebug: acrnArchBaseKernelParamsNonDebug, + kernelParamsDebug: acrnArchBaseKernelParamsDebug, + kernelParams: acrnArchBaseKernelParams, + } +} + +func TestAcrnArchBaseAcrnPaths(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + p, err := acrnArchBase.acrnPath() + assert.NoError(err) + assert.Equal(p, acrnArchBaseAcrnPath) + + ctlp, err := acrnArchBase.acrnctlPath() + assert.NoError(err) + assert.Equal(ctlp, acrnArchBaseAcrnCtlPath) +} + +func TestAcrnArchBaseKernelParameters(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + // with debug params + expectedParams := acrnArchBaseKernelParams + debugParams := acrnArchBaseKernelParamsDebug + expectedParams = append(expectedParams, debugParams...) + p := acrnArchBase.kernelParameters(true) + assert.Equal(expectedParams, p) + + // with non-debug params + expectedParams = acrnArchBaseKernelParams + nonDebugParams := acrnArchBaseKernelParamsNonDebug + expectedParams = append(expectedParams, nonDebugParams...) + p = acrnArchBase.kernelParameters(false) + assert.Equal(expectedParams, p) +} + +func TestAcrnArchBaseCapabilities(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + c := acrnArchBase.capabilities() + assert.True(c.IsBlockDeviceSupported()) + assert.True(c.IsBlockDeviceHotplugSupported()) + assert.False(c.IsFsSharingSupported()) +} + +func TestAcrnArchBaseMemoryTopology(t *testing.T) { + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + mem := uint64(8192) + + expectedMemory := Memory{ + Size: fmt.Sprintf("%dM", mem), + } + + m := acrnArchBase.memoryTopology(mem) + assert.Equal(expectedMemory, m) +} + +func TestAcrnArchBaseAppendConsoles(t *testing.T) { + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + path := filepath.Join(store.SandboxRuntimeRootPath(sandboxID), consoleSocket) + + expectedOut := []Device{ + ConsoleDevice{ + Name: "console0", + Backend: Socket, + PortType: ConsoleBE, + Path: path, + }, + } + + devices = acrnArchBase.appendConsole(devices, path) + assert.Equal(expectedOut, devices) +} + +func TestAcrnArchBaseAppendImage(t *testing.T) { + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + image, err := ioutil.TempFile("", "img") + assert.NoError(err) + err = image.Close() + assert.NoError(err) + + devices, err = acrnArchBase.appendImage(devices, image.Name()) + assert.NoError(err) + assert.Len(devices, 1) + + expectedOut := []Device{ + BlockDevice{ + FilePath: image.Name(), + Index: 0, + }, + } + + assert.Equal(expectedOut, devices) +} + +func TestAcrnArchBaseAppendBridges(t *testing.T) { + function := 0 + emul := acrnHostBridge + config := "" + + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + devices = acrnArchBase.appendBridges(devices) + assert.Len(devices, 1) + + expectedOut := []Device{ + BridgeDevice{ + Function: function, + Emul: emul, + Config: config, + }, + } + + assert.Equal(expectedOut, devices) +} + +func TestAcrnArchBaseAppendLpcDevice(t *testing.T) { + function := 0 + emul := acrnLPCDev + + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + devices = acrnArchBase.appendLPC(devices) + assert.Len(devices, 1) + + expectedOut := []Device{ + LPCDevice{ + Function: function, + Emul: emul, + }, + } + + assert.Equal(expectedOut, devices) +} + +func testAcrnArchBaseAppend(t *testing.T, structure interface{}, expected []Device) { + var devices []Device + var err error + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + switch s := structure.(type) { + case types.Socket: + devices = acrnArchBase.appendSocket(devices, s) + case config.BlockDrive: + devices = acrnArchBase.appendBlockDevice(devices, s) + } + + assert.NoError(err) + assert.Equal(devices, expected) +} + +func TestAcrnArchBaseAppendSocket(t *testing.T) { + name := "archserial.test" + hostPath := "/tmp/archserial.sock" + + expectedOut := []Device{ + ConsoleDevice{ + Name: name, + Backend: Socket, + PortType: SerialBE, + Path: hostPath, + }, + } + + socket := types.Socket{ + HostPath: hostPath, + Name: name, + } + + testAcrnArchBaseAppend(t, socket, expectedOut) +} + +func TestAcrnArchBaseAppendBlockDevice(t *testing.T) { + path := "/tmp/archtest.img" + index := 5 + + expectedOut := []Device{ + BlockDevice{ + FilePath: path, + Index: index, + }, + } + + drive := config.BlockDrive{ + File: path, + Index: index, + } + + testAcrnArchBaseAppend(t, drive, expectedOut) +} + +func TestAcrnArchBaseAppendNetwork(t *testing.T) { + var devices []Device + assert := assert.New(t) + acrnArchBase := newAcrnArchBase() + + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + vethEp := &VethEndpoint{ + NetPair: NetworkInterfacePair{ + TapInterface: TapInterface{ + ID: "uniqueTestID0", + Name: "br0_kata", + TAPIface: NetworkInterface{ + Name: "tap0_kata", + }, + }, + VirtIface: NetworkInterface{ + Name: "eth0", + HardAddr: macAddr.String(), + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: VethEndpointType, + } + + macvtapEp := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: NetworkInfo{ + Iface: NetlinkIface{ + Type: "macvtap", + }, + }, + } + + expectedOut := []Device{ + NetDevice{ + Type: TAP, + IFName: vethEp.NetPair.TAPIface.Name, + MACAddress: vethEp.NetPair.TAPIface.HardAddr, + }, + NetDevice{ + Type: MACVTAP, + IFName: macvtapEp.Name(), + MACAddress: macvtapEp.HardwareAddr(), + }, + } + + devices = acrnArchBase.appendNetwork(devices, vethEp) + devices = acrnArchBase.appendNetwork(devices, macvtapEp) + assert.Equal(expectedOut, devices) +} diff --git a/virtcontainers/acrn_test.go b/virtcontainers/acrn_test.go new file mode 100644 index 000000000..c57f68240 --- /dev/null +++ b/virtcontainers/acrn_test.go @@ -0,0 +1,285 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" +) + +func newAcrnConfig() HypervisorConfig { + return HypervisorConfig{ + KernelPath: testAcrnKernelPath, + ImagePath: testAcrnImagePath, + HypervisorPath: testAcrnPath, + HypervisorCtlPath: testAcrnCtlPath, + NumVCPUs: defaultVCPUs, + MemorySize: defaultMemSzMiB, + BlockDeviceDriver: config.VirtioBlock, + DefaultBridges: defaultBridges, + DefaultMaxVCPUs: MaxAcrnVCPUs(), + // Adding this here, as hypervisorconfig.valid() + // forcefully adds it even when 9pfs is not supported + Msize9p: defaultMsize9p, + } +} + +func testAcrnKernelParameters(t *testing.T, kernelParams []Param, debug bool) { + acrnConfig := newAcrnConfig() + acrnConfig.KernelParams = kernelParams + + if debug == true { + acrnConfig.Debug = true + } + + a := &acrn{ + config: acrnConfig, + arch: &acrnArchBase{}, + } + + expected := fmt.Sprintf("panic=1 maxcpus=%d foo=foo bar=bar", a.config.NumVCPUs) + + params := a.kernelParameters() + if params != expected { + t.Fatalf("Got: %v, Expecting: %v", params, expected) + } +} + +func TestAcrnKernelParameters(t *testing.T) { + params := []Param{ + { + Key: "foo", + Value: "foo", + }, + { + Key: "bar", + Value: "bar", + }, + } + + testAcrnKernelParameters(t, params, true) + testAcrnKernelParameters(t, params, false) +} + +func TestAcrnCapabilities(t *testing.T) { + a := &acrn{ + ctx: context.Background(), + arch: &acrnArchBase{}, + } + + caps := a.capabilities() + if !caps.IsBlockDeviceSupported() { + t.Fatal("Block device should be supported") + } + + if !caps.IsBlockDeviceHotplugSupported() { + t.Fatal("Block device hotplug should be supported") + } +} + +func testAcrnAddDevice(t *testing.T, devInfo interface{}, devType deviceType, expected []Device) { + a := &acrn{ + ctx: context.Background(), + arch: &acrnArchBase{}, + } + + err := a.addDevice(devInfo, devType) + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(a.acrnConfig.Devices, expected) == false { + t.Fatalf("Got %v\nExpecting %v", a.acrnConfig.Devices, expected) + } +} + +func TestAcrnAddDeviceSerialPortDev(t *testing.T) { + name := "serial.test" + hostPath := "/tmp/serial.sock" + + expectedOut := []Device{ + ConsoleDevice{ + Name: name, + Backend: Socket, + PortType: SerialBE, + Path: hostPath, + }, + } + + socket := types.Socket{ + HostPath: hostPath, + Name: name, + } + + testAcrnAddDevice(t, socket, serialPortDev, expectedOut) +} + +func TestAcrnAddDeviceBlockDev(t *testing.T) { + path := "/tmp/test.img" + index := 1 + + expectedOut := []Device{ + BlockDevice{ + FilePath: path, + Index: index, + }, + } + + drive := config.BlockDrive{ + File: path, + Index: index, + } + + testAcrnAddDevice(t, drive, blockDev, expectedOut) +} + +func TestAcrnHotplugUnsupportedDeviceType(t *testing.T) { + assert := assert.New(t) + + acrnConfig := newAcrnConfig() + a := &acrn{ + ctx: context.Background(), + id: "acrnTest", + config: acrnConfig, + } + + _, err := a.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) + assert.Error(err) +} + +func TestAcrnUpdateBlockDeviceInvalidPath(t *testing.T) { + assert := assert.New(t) + + path := "" + index := 1 + + acrnConfig := newAcrnConfig() + a := &acrn{ + ctx: context.Background(), + id: "acrnBlkTest", + config: acrnConfig, + } + + drive := &config.BlockDrive{ + File: path, + Index: index, + } + + err := a.updateBlockDevice(drive) + assert.Error(err) +} + +func TestAcrnUpdateBlockDeviceInvalidIdx(t *testing.T) { + assert := assert.New(t) + + path := "/tmp/test.img" + index := AcrnBlkDevPoolSz + 1 + + acrnConfig := newAcrnConfig() + a := &acrn{ + ctx: context.Background(), + id: "acrnBlkTest", + config: acrnConfig, + } + + drive := &config.BlockDrive{ + File: path, + Index: index, + } + + err := a.updateBlockDevice(drive) + assert.Error(err) +} + +func TestAcrnGetSandboxConsole(t *testing.T) { + a := &acrn{ + ctx: context.Background(), + } + sandboxID := "testSandboxID" + expected := filepath.Join(store.RunVMStoragePath, sandboxID, consoleSocket) + + result, err := a.getSandboxConsole(sandboxID) + if err != nil { + t.Fatal(err) + } + + if result != expected { + t.Fatalf("Got %s\nExpecting %s", result, expected) + } +} + +func TestAcrnCreateSandbox(t *testing.T) { + acrnConfig := newAcrnConfig() + a := &acrn{} + + sandbox := &Sandbox{ + ctx: context.Background(), + id: "testSandbox", + config: &SandboxConfig{ + HypervisorConfig: acrnConfig, + }, + } + + vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + if err != nil { + t.Fatal(err) + } + sandbox.store = vcStore + + if err = globalSandboxList.addSandbox(sandbox); err != nil { + t.Fatalf("Could not add sandbox to global list: %v", err) + } + + defer globalSandboxList.removeSandbox(sandbox.id) + + // Create the hypervisor fake binary + testAcrnPath := filepath.Join(testDir, testHypervisor) + _, err = os.Create(testAcrnPath) + if err != nil { + t.Fatalf("Could not create hypervisor file %s: %v", testAcrnPath, err) + } + + if err := a.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(acrnConfig, a.config) == false { + t.Fatalf("Got %v\nExpecting %v", a.config, acrnConfig) + } +} + +func TestAcrnMemoryTopology(t *testing.T) { + mem := uint32(1000) + + a := &acrn{ + arch: &acrnArchBase{}, + config: HypervisorConfig{ + MemorySize: mem, + }, + } + + expectedOut := Memory{ + Size: fmt.Sprintf("%dM", mem), + } + + memory, err := a.memoryTopology() + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(memory, expectedOut) == false { + t.Fatalf("Got %v\nExpecting %v", memory, expectedOut) + } +} diff --git a/virtcontainers/virtcontainers_test.go b/virtcontainers/virtcontainers_test.go index 424a83ed6..21c499d52 100644 --- a/virtcontainers/virtcontainers_test.go +++ b/virtcontainers/virtcontainers_test.go @@ -25,6 +25,7 @@ const testKernel = "kernel" const testInitrd = "initrd" const testImage = "image" const testHypervisor = "hypervisor" +const testHypervisorCtl = "hypervisorctl" const testBundle = "bundle" const testDisabledAsNonRoot = "Test disabled as requires root privileges" @@ -41,6 +42,10 @@ var testQemuKernelPath = "" var testQemuInitrdPath = "" var testQemuImagePath = "" var testQemuPath = "" +var testAcrnKernelPath = "" +var testAcrnImagePath = "" +var testAcrnPath = "" +var testAcrnCtlPath = "" var testHyperstartCtlSocket = "" var testHyperstartTtySocket = "" @@ -67,6 +72,18 @@ func setup() { } } +func setupAcrn() { + os.Mkdir(filepath.Join(testDir, testBundle), store.DirMode) + + for _, filename := range []string{testAcrnKernelPath, testAcrnImagePath, testAcrnPath, testAcrnCtlPath} { + _, err := os.Create(filename) + if err != nil { + fmt.Printf("Could not recreate %s:%v", filename, err) + os.Exit(1) + } + } +} + // TestMain is the common main function used by ALL the test functions // for this package. func TestMain(m *testing.M) { @@ -102,6 +119,13 @@ func TestMain(m *testing.M) { setup() + testAcrnKernelPath = filepath.Join(testDir, testKernel) + testAcrnImagePath = filepath.Join(testDir, testImage) + testAcrnPath = filepath.Join(testDir, testHypervisor) + testAcrnCtlPath = filepath.Join(testDir, testHypervisorCtl) + + setupAcrn() + // allow the tests to run without affecting the host system. store.ConfigStoragePath = filepath.Join(testDir, store.StoragePathSuffix, "config") store.RunStoragePath = filepath.Join(testDir, store.StoragePathSuffix, "run")