runtime: move all code to src/runtime

To prepare for merging into kata-containers repository.

Signed-off-by: Peng Tao <bergwolf@hyper.sh>

src/runtime/cli/config-generated.go.in

@@ -0,0 +1,49 @@
+//
+// Copyright (c) 2018-2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// WARNING: This file is auto-generated - DO NOT EDIT!
+//
+// Note that some variables are "var" to allow them to be modified
+// by the tests.
+package main
+
+import (
+	"fmt"
+)
+
+// name is the name of the runtime
+const name = "@RUNTIME_NAME@"
+
+// name of the project
+const project = "@PROJECT_NAME@"
+
+// prefix used to denote non-standard CLI commands and options.
+const projectPrefix = "@PROJECT_TYPE@"
+
+// original URL for this project
+const projectURL = "@PROJECT_URL@"
+
+const defaultRootDirectory = "@PKGRUNDIR@"
+
+// commit is the git commit the runtime is compiled from.
+var commit = "@COMMIT@"
+
+// version is the runtime version.
+var version = "@VERSION@"
+
+// project-specific command names
+var envCmd = fmt.Sprintf("%s-env", projectPrefix)
+var checkCmd = fmt.Sprintf("%s-check", projectPrefix)
+
+// project-specific option names
+var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix)
+var showConfigPathsOption = fmt.Sprintf("%s-show-default-config-paths", projectPrefix)
+
+// Default config file used by stateless systems.
+var defaultRuntimeConfiguration = "@CONFIG_PATH@"
+
+// Alternate config file that takes precedence over
+// defaultRuntimeConfiguration.
+var defaultSysConfRuntimeConfiguration = "@SYSCONFIG@"

src/runtime/cli/config/configuration-acrn.toml.in

@@ -0,0 +1,237 @@
+# Copyright (c) 2017-2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_ACRN_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.acrn]
+path = "@ACRNPATH@"
+ctlpath = "@ACRNCTLPATH@"
+kernel = "@KERNELPATH_ACRN@"
+image = "@IMAGEPATH@"
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that acrn uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS_ACRN@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. ACRN only supports virtio-blk.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. This extra output is added
+# to the proxy logs, but only when proxy debug is also enabled.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics.
+# Default false
+#disable_vhost_net = true
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered will scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+[proxy.@PROJECT_TYPE@]
+path = "@PROXYPATH@"
+
+# If enabled, proxy messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[shim.@PROJECT_TYPE@]
+path = "@SHIMPATH@"
+
+# If enabled, shim messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+# If enabled, the shim will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+#
+# Note: By default, the shim runs in a separate network namespace. Therefore,
+# to allow it to send trace details to the Jaeger agent running on the host,
+# it is necessary to set 'disable_new_netns=true' so that it runs in the host
+# network namespace.
+#
+# (default: disabled)
+#enable_tracing = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - bridged (Deprecated)
+#     Uses a linux bridge to interconnect the container interface to
+#     the VM. Works for most cases except macvlan and ipvlan.
+#     ***NOTE: This feature has been deprecated with plans to remove this
+#     feature in the future. Please use other network models listed below.
+#
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_ACRN@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
+# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

src/runtime/cli/config/configuration-clh.toml.in

@@ -0,0 +1,213 @@
+# Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_CLH_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.clh]
+path = "@CLHPATH@"
+kernel = "@KERNELPATH_CLH@"
+image = "@IMAGEPATH@"
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# cloud-hypervisor prefers virtiofs caching (dax) for performance reasons
+virtio_fs_cache = "always"
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. This extra output is added
+# to the proxy logs, but only when proxy debug is also enabled.
+#
+# Default false
+#enable_debug = true
+
+[proxy.@PROJECT_TYPE@]
+path = "@PROXYPATH@"
+
+# If enabled, proxy messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[shim.@PROJECT_TYPE@]
+path = "@SHIMPATH@"
+
+# If enabled, shim messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+# If enabled, the shim will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+#
+# Note: By default, the shim runs in a separate network namespace. Therefore,
+# to allow it to send trace details to the Jaeger agent running on the host,
+# it is necessary to set 'disable_new_netns=true' so that it runs in the host
+# network namespace.
+#
+# (default: disabled)
+#enable_tracing = true
+
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
+
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - bridged (Deprecated)
+#     Uses a linux bridge to interconnect the container interface to
+#     the VM. Works for most cases except macvlan and ipvlan.
+#     ***NOTE: This feature has been deprecated with plans to remove this
+#     feature in the future. Please use other network models listed below.
+#
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_CLH@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
+# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

src/runtime/cli/config/configuration-fc.toml.in

@@ -0,0 +1,339 @@
+# Copyright (c) 2017-2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_FC_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.firecracker]
+path = "@FCPATH@"
+# Path for the jailer specific to firecracker
+# If the jailer path is not set kata will launch firecracker
+# without a jail. If the jailer is set firecracker will be
+# launched in a jailed enviornment created by the jailer
+# This is disabled by default as additional setup is required
+# for this feature today.
+#jailer_path = "@FCJAILERPATH@"
+kernel = "@KERNELPATH_FC@"
+image = "@IMAGEPATH@"
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's 
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons. 
+# This flag prevents the block device from being passed to the hypervisor, 
+# 9pfs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically 
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable swap of vm memory. Default false.
+# The behaviour is undefined if mem_prealloc is also set to true
+#enable_swap = true
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. This extra output is added
+# to the proxy logs, but only when proxy debug is also enabled.
+# 
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+# 
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes 
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If true and vsocks are supported, use vsocks to communicate directly
+# with the agent (no proxy is started).
+# Default true
+use_vsock = true
+
+# VFIO devices are hotplugged on a bridge by default. 
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on 
+# a bridge. This value is valid for "pc" machine type.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered will scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+[shim.@PROJECT_TYPE@]
+path = "@SHIMPATH@"
+
+# If enabled, shim messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+# If enabled, the shim will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+#
+# Note: By default, the shim runs in a separate network namespace. Therefore,
+# to allow it to send trace details to the Jaeger agent running on the host,
+# it is necessary to set 'disable_new_netns=true' so that it runs in the host
+# network namespace.
+#
+# (default: disabled)
+#enable_tracing = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_FC@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
+# (default: false)
+#disable_new_netns = true
+
+# if enable, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

src/runtime/cli/config/configuration-qemu-virtiofs.toml.in

@@ -0,0 +1,452 @@
+# Copyright (c) 2017-2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_VIRTIOFS_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUVIRTIOFSPATH@"
+kernel = "@KERNELVIRTIOFSPATH@"
+image = "@IMAGEPATH@"
+machine_type = "@MACHINETYPE@"
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators="@MACHINEACCELERATORS@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons.
+# This flag prevents the block device from being passed to the hypervisor,
+# 9pfs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-fs (default)
+#   - virtio-9p
+shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+#file_mem_backend = ""
+
+# Enable swap of vm memory. Default false.
+# The behaviour is undefined if mem_prealloc is also set to true
+#enable_swap = true
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. This extra output is added
+# to the proxy logs, but only when proxy debug is also enabled.
+#
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+#
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If true and vsocks are supported, use vsocks to communicate directly
+# with the agent and no proxy is started, otherwise use unix
+# sockets and start a proxy to communicate with the agent.
+# Default false
+#use_vsock = true
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+# Default false
+#disable_image_nvdimm = true
+
+# VFIO devices are hotplugged on a bridge by default.
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on
+# a bridge. This value is valid for "pc" machine type.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance. 
+#disable_vhost_net = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered will scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+#template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[proxy.@PROJECT_TYPE@]
+path = "@PROXYPATH@"
+
+# If enabled, proxy messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[shim.@PROJECT_TYPE@]
+path = "@SHIMPATH@"
+
+# If enabled, shim messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+# If enabled, the shim will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+#
+# Note: By default, the shim runs in a separate network namespace. Therefore,
+# to allow it to send trace details to the Jaeger agent running on the host,
+# it is necessary to set 'disable_new_netns=true' so that it runs in the host
+# network namespace.
+#
+# (default: disabled)
+#enable_tracing = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - bridged (Deprecated)
+#     Uses a linux bridge to interconnect the container interface to
+#     the VM. Works for most cases except macvlan and ipvlan.
+#     ***NOTE: This feature has been deprecated with plans to remove this
+#     feature in the future. Please use other network models listed below.
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
+# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

src/runtime/cli/config/configuration-qemu.toml.in

@@ -0,0 +1,460 @@
+# Copyright (c) 2017-2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# XXX: WARNING: this file is auto-generated.
+# XXX:
+# XXX: Source file: "@CONFIG_QEMU_IN@"
+# XXX: Project:
+# XXX:   Name: @PROJECT_NAME@
+# XXX:   Type: @PROJECT_TYPE@
+
+[hypervisor.qemu]
+path = "@QEMUPATH@"
+kernel = "@KERNELPATH@"
+initrd = "@INITRDPATH@"
+image = "@IMAGEPATH@"
+machine_type = "@MACHINETYPE@"
+
+# Optional space-separated list of options to pass to the guest kernel.
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
+# trouble running pre-2.15 glibc.
+#
+# WARNING: - any parameter specified here will take priority over the default
+# parameter value of the same name used to start the virtual machine.
+# Do not set values here unless you understand the impact of doing so as you
+# may stop the virtual machine from booting.
+# To see the list of default parameters, enable hypervisor debug, create a
+# container and look for 'default-kernel-parameters' log entries.
+kernel_params = "@KERNELPARAMS@"
+
+# Path to the firmware.
+# If you want that qemu uses the default firmware leave this option empty
+firmware = "@FIRMWAREPATH@"
+
+# Machine accelerators
+# comma-separated list of machine accelerators to pass to the hypervisor.
+# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
+machine_accelerators="@MACHINEACCELERATORS@"
+
+# Default number of vCPUs per SB/VM:
+# unspecified or 0                --> will be set to @DEFVCPUS@
+# < 0                             --> will be set to the actual number of physical cores
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores
+default_vcpus = 1
+
+# Default maximum number of vCPUs per SB/VM:
+# unspecified or == 0             --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# > 0 <= number of physical cores --> will be set to the specified number
+# > number of physical cores      --> will be set to the actual number of physical cores or to the maximum number
+#                                     of vCPUs supported by KVM if that number is exceeded
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
+# the actual number of physical cores is greater than it.
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
+# unless you know what are you doing.
+default_maxvcpus = @DEFMAXVCPUS@
+
+# Bridges can be used to hot plug devices.
+# Limitations:
+# * Currently only pci bridges are supported
+# * Until 30 devices per bridge can be hot plugged.
+# * Until 5 PCI bridges can be cold plugged per VM.
+#   This limitation could be a bug in qemu or in the kernel
+# Default number of bridges per SB/VM:
+# unspecified or 0   --> will be set to @DEFBRIDGES@
+# > 1 <= 5           --> will be set to the specified number
+# > 5                --> will be set to 5
+default_bridges = @DEFBRIDGES@
+
+# Default memory size in MiB for SB/VM.
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
+default_memory = @DEFMEMSZ@
+#
+# Default memory slots per SB/VM.
+# If unspecified then it will be set @DEFMEMSLOTS@.
+# This is will determine the times that memory will be hotadded to sandbox/VM.
+#memory_slots = @DEFMEMSLOTS@
+
+# The size in MiB will be plused to max memory of hypervisor.
+# It is the memory address space for the NVDIMM devie.
+# If set block storage driver (block_device_driver) to "nvdimm",
+# should set memory_offset to the size of block device.
+# Default 0
+#memory_offset = 0
+
+# Specifies virtio-mem will be enabled or not.
+# Please note that this option should be used with the command
+# "echo 1 > /proc/sys/vm/overcommit_memory".
+# Default false
+#enable_virtio_mem = true
+
+# Disable block device from being used for a container's rootfs.
+# In case of a storage driver like devicemapper where a container's 
+# root file system is backed by a block device, the block device is passed
+# directly to the hypervisor for performance reasons. 
+# This flag prevents the block device from being passed to the hypervisor, 
+# 9pfs is used instead to pass the rootfs.
+disable_block_device_use = @DEFDISABLEBLOCK@
+
+# Shared file system type:
+#   - virtio-9p (default)
+#   - virtio-fs
+shared_fs = "@DEFSHAREDFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Extra args for virtiofsd daemon
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
+
+# Specifies cache-related options will be set to block devices or not.
+# Default false
+#block_device_cache_set = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
+# Default false
+#block_device_cache_direct = true
+
+# Specifies cache-related options for block devices.
+# Denotes whether flush requests for the device are ignored.
+# Default false
+#block_device_cache_noflush = true
+
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
+# Enable pre allocation of VM RAM, default false
+# Enabling this will result in lower container density
+# as all of the memory will be allocated and locked
+# This is useful when you want to reserve all the memory
+# upfront or in the cases where you want memory latencies
+# to be very predictable
+# Default false
+#enable_mem_prealloc = true
+
+# Enable huge pages for VM RAM, default false
+# Enabling this will result in the VM memory
+# being allocated using huge pages.
+# This is useful when you want to use vhost-user network
+# stacks within the container. This will automatically 
+# result in memory pre allocation
+#enable_hugepages = true
+
+# Enable vhost-user storage device, default false
+# Enabling this will result in some Linux reserved block type
+# major range 240-254 being chosen to represent vhost-user devices.
+enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
+
+# The base directory specifically used for vhost-user devices.
+# Its sub-path "block" is used for block devices; "block/sockets" is
+# where we expect vhost-user sockets to live; "block/devices" is where
+# simulated block device nodes for vhost-user devices to live.
+vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
+
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+#file_mem_backend = ""
+
+# Enable swap of vm memory. Default false.
+# The behaviour is undefined if mem_prealloc is also set to true
+#enable_swap = true
+
+# This option changes the default hypervisor and kernel parameters
+# to enable debug output where available. This extra output is added
+# to the proxy logs, but only when proxy debug is also enabled.
+# 
+# Default false
+#enable_debug = true
+
+# Disable the customizations done in the runtime when it detects
+# that it is running on top a VMM. This will result in the runtime
+# behaving as it would when running on bare metal.
+# 
+#disable_nesting_checks = true
+
+# This is the msize used for 9p shares. It is the number of bytes 
+# used for 9p packet payload.
+#msize_9p = @DEFMSIZE9P@
+
+# If true and vsocks are supported, use vsocks to communicate directly
+# with the agent and no proxy is started, otherwise use unix
+# sockets and start a proxy to communicate with the agent.
+# Default false
+#use_vsock = true
+
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+# Default is false
+#disable_image_nvdimm = true
+
+# VFIO devices are hotplugged on a bridge by default. 
+# Enable hotplugging on root bus. This may be required for devices with
+# a large PCI bar, as this is a current limitation with hotplugging on 
+# a bridge. This value is valid for "pc" machine type.
+# Default false
+#hotplug_vfio_on_root_bus = true
+
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
+# The value means the number of pcie_root_port
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Default 0
+#pcie_root_port = 2
+
+# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
+# security (vhost-net runs ring0) for network I/O performance. 
+#disable_vhost_net = true
+
+#
+# Default entropy source.
+# The path to a host source of entropy (including a real hardware RNG)
+# /dev/urandom and /dev/random are two main options.
+# Be aware that /dev/random is a blocking source of entropy.  If the host
+# runs out of entropy, the VMs boot time will increase leading to get startup
+# timeouts.
+# The source of entropy /dev/urandom is non-blocking and provides a
+# generally acceptable source of entropy. It should work well for pretty much
+# all practical purposes.
+#entropy_source= "@DEFENTROPYSOURCE@"
+
+# Path to OCI hook binaries in the *guest rootfs*.
+# This does not affect host-side hooks which must instead be added to
+# the OCI spec passed to the runtime.
+#
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
+# https://github.com/kata-containers/osbuilder
+#
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
+# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}".
+# The agent will scan these directories for executable files and add them, in
+# lexicographical order, to the lifecycle of the guest container.
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+# Warnings will be logged if any error is encountered will scanning for hooks,
+# but it will not abort container execution.
+#guest_hook_path = "/usr/share/oci/hooks"
+
+[factory]
+# VM templating support. Once enabled, new VMs are created from template
+# using vm cloning. They will share the same initial kernel, initramfs and
+# agent memory by mapping it readonly. It helps speeding up new container
+# creation and saves a lot of memory if there are many kata containers running
+# on the same host.
+#
+# When disabled, new VMs are created from scratch.
+#
+# Note: Requires "initrd=" to be set ("image=" is not supported).
+#
+# Default false
+#enable_template = true
+
+# Specifies the path of template.
+#
+# Default "/run/vc/vm/template"
+#template_path = "/run/vc/vm/template"
+
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
+[proxy.@PROJECT_TYPE@]
+path = "@PROXYPATH@"
+
+# If enabled, proxy messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[shim.@PROJECT_TYPE@]
+path = "@SHIMPATH@"
+
+# If enabled, shim messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+# If enabled, the shim will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+#
+# Note: By default, the shim runs in a separate network namespace. Therefore,
+# to allow it to send trace details to the Jaeger agent running on the host,
+# it is necessary to set 'disable_new_netns=true' so that it runs in the host
+# network namespace.
+#
+# (default: disabled)
+#enable_tracing = true
+
+[agent.@PROJECT_TYPE@]
+# If enabled, make the agent display debug-level messages.
+# (default: disabled)
+#enable_debug = true
+
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+[runtime]
+# If enabled, the runtime will log additional debug messages to the
+# system log
+# (default: disabled)
+#enable_debug = true
+#
+# Internetworking model
+# Determines how the VM should be connected to the
+# the container network interface
+# Options:
+#
+#   - macvtap
+#     Used when the Container network interface can be bridged using
+#     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
+#   - tcfilter
+#     Uses tc filter rules to redirect traffic from the network interface
+#     provided by plugin to a tap interface connected to the VM.
+#
+internetworking_model="@DEFNETWORKMODEL_QEMU@"
+
+# disable guest seccomp
+# Determines whether container seccomp profiles are passed to the virtual
+# machine and applied by the kata agent. If set to true, seccomp is not applied
+# within the guest
+# (default: true)
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
+
+# If enabled, the runtime will create opentracing.io traces and spans.
+# (See https://www.jaegertracing.io/docs/getting-started).
+# (default: disabled)
+#enable_tracing = true
+
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
+# (like OVS) directly.
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
+# (default: false)
+#disable_new_netns = true
+
+# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+# The sandbox cgroup is constrained if there is no container type annotation.
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# they may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

src/runtime/cli/console.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+var ptmxPath = "/dev/ptmx"
+
+// Console represents a pseudo TTY.
+type Console struct {
+	io.ReadWriteCloser
+
+	master    *os.File
+	slavePath string
+}
+
+// isTerminal returns true if fd is a terminal, else false
+func isTerminal(fd uintptr) bool {
+	var termios syscall.Termios
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(&termios)))
+	return err == 0
+}
+
+// ConsoleFromFile creates a console from a file
+func ConsoleFromFile(f *os.File) *Console {
+	return &Console{
+		master: f,
+	}
+}
+
+// NewConsole returns an initialized console that can be used within a container by copying bytes
+// from the master side to the slave that is attached as the tty for the container's init process.
+func newConsole() (*Console, error) {
+	master, err := os.OpenFile(ptmxPath, unix.O_RDWR|unix.O_NOCTTY|unix.O_CLOEXEC, 0)
+	if err != nil {
+		return nil, err
+	}
+	if err := saneTerminal(master); err != nil {
+		return nil, err
+	}
+	console, err := ptsname(master)
+	if err != nil {
+		return nil, err
+	}
+	if err := unlockpt(master); err != nil {
+		return nil, err
+	}
+	return &Console{
+		slavePath: console,
+		master:    master,
+	}, nil
+}
+
+// File returns master
+func (c *Console) File() *os.File {
+	return c.master
+}
+
+// Path to slave
+func (c *Console) Path() string {
+	return c.slavePath
+}
+
+// Read from master
+func (c *Console) Read(b []byte) (int, error) {
+	return c.master.Read(b)
+}
+
+// Write to master
+func (c *Console) Write(b []byte) (int, error) {
+	return c.master.Write(b)
+}
+
+// Close master
+func (c *Console) Close() error {
+	if m := c.master; m != nil {
+		return m.Close()
+	}
+	return nil
+}
+
+// unlockpt unlocks the slave pseudoterminal device corresponding to the master pseudoterminal referred to by f.
+// unlockpt should be called before opening the slave side of a pty.
+func unlockpt(f *os.File) error {
+	var u int32
+	if _, _, err := unix.Syscall(unix.SYS_IOCTL, f.Fd(), unix.TIOCSPTLCK, uintptr(unsafe.Pointer(&u))); err != 0 {
+		return err
+	}
+	return nil
+}
+
+// ptsname retrieves the name of the first available pts for the given master.
+func ptsname(f *os.File) (string, error) {
+	var u uint32
+	if _, _, err := unix.Syscall(unix.SYS_IOCTL, f.Fd(), unix.TIOCGPTN, uintptr(unsafe.Pointer(&u))); err != 0 {
+		return "", err
+	}
+	return fmt.Sprintf("/dev/pts/%d", u), nil
+}
+
+// saneTerminal sets the necessary tty_ioctl(4)s to ensure that a pty pair
+// created by us acts normally. In particular, a not-very-well-known default of
+// Linux unix98 ptys is that they have +onlcr by default. While this isn't a
+// problem for terminal emulators, because we relay data from the terminal we
+// also relay that funky line discipline.
+func saneTerminal(terminal *os.File) error {
+	// Go doesn't have a wrapper for any of the termios ioctls.
+	var termios unix.Termios
+
+	if _, _, err := unix.Syscall(unix.SYS_IOCTL, terminal.Fd(), unix.TCGETS, uintptr(unsafe.Pointer(&termios))); err != 0 {
+		return fmt.Errorf("ioctl(tty, tcgets): %s", err.Error())
+	}
+
+	// Set -onlcr so we don't have to deal with \r.
+	termios.Oflag &^= unix.ONLCR
+
+	if _, _, err := unix.Syscall(unix.SYS_IOCTL, terminal.Fd(), unix.TCSETS, uintptr(unsafe.Pointer(&termios))); err != 0 {
+		return fmt.Errorf("ioctl(tty, tcsets): %s", err.Error())
+	}
+
+	return nil
+}

src/runtime/cli/console_test.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConsoleFromFile(t *testing.T) {
+	assert := assert.New(t)
+
+	console := ConsoleFromFile(os.Stdout)
+
+	assert.NotNil(console.File(), "console file is nil")
+}
+
+func TestNewConsole(t *testing.T) {
+	assert := assert.New(t)
+
+	console, err := newConsole()
+	assert.NoError(err, "failed to create a new console: %s", err)
+	defer console.Close()
+
+	assert.NotEmpty(console.Path(), "console path is empty")
+
+	assert.NotNil(console.File(), "console file is nil")
+}
+
+func TestIsTerminal(t *testing.T) {
+	assert := assert.New(t)
+
+	var fd uintptr = 4
+	assert.False(isTerminal(fd), "Fd %d is not a terminal", fd)
+
+	console, err := newConsole()
+	assert.NoError(err, "failed to create a new console: %s", err)
+	defer console.Close()
+
+	fd = console.File().Fd()
+	assert.True(isTerminal(fd), "Fd %d is a terminal", fd)
+}
+
+func TestReadWrite(t *testing.T) {
+	assert := assert.New(t)
+
+	// write operation
+	f, err := ioutil.TempFile(os.TempDir(), ".tty")
+	assert.NoError(err, "failed to create a temporal file")
+	defer os.Remove(f.Name())
+
+	console := ConsoleFromFile(f)
+	assert.NotNil(console)
+	defer console.Close()
+
+	msgWrite := "hello"
+	l, err := console.Write([]byte(msgWrite))
+	assert.NoError(err, "failed to write message: %s", msgWrite)
+	assert.Equal(len(msgWrite), l)
+
+	console.master.Sync()
+	console.master.Seek(0, 0)
+
+	// Read operation
+	msgRead := make([]byte, len(msgWrite))
+	l, err = console.Read(msgRead)
+	assert.NoError(err, "failed to read message: %s", msgWrite)
+	assert.Equal(len(msgWrite), l)
+	assert.Equal(msgWrite, string(msgRead))
+}
+
+func TestNewConsoleFail(t *testing.T) {
+	assert := assert.New(t)
+
+	orgPtmxPath := ptmxPath
+	defer func() { ptmxPath = orgPtmxPath }()
+
+	// OpenFile failure
+	ptmxPath = "/this/file/does/not/exist"
+	c, err := newConsole()
+	assert.Error(err)
+	assert.Nil(c)
+
+	// saneTerminal failure
+	f, err := ioutil.TempFile("", "")
+	assert.NoError(err)
+	assert.NoError(f.Close())
+	defer os.Remove(f.Name())
+	ptmxPath = f.Name()
+	c, err = newConsole()
+	assert.Error(err)
+	assert.Nil(c)
+}
+
+func TestConsoleClose(t *testing.T) {
+	assert := assert.New(t)
+
+	// nil master
+	c := &Console{}
+	assert.NoError(c.Close())
+
+	f, err := ioutil.TempFile("", "")
+	assert.NoError(err)
+	defer os.Remove(f.Name())
+
+	c.master = f
+	assert.NoError(c.Close())
+}
+
+func TestConsolePtsnameFail(t *testing.T) {
+	assert := assert.New(t)
+
+	pts, err := ptsname(nil)
+	assert.Error(err)
+	assert.Empty(pts)
+}

src/runtime/cli/containerd-shim-kata-v2/main.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/kata-containers/runtime/containerd-shim-v2"
+)
+
+func shimConfig(config *shim.Config) {
+	config.NoReaper = true
+	config.NoSubreaper = true
+}
+
+func main() {
+	shim.Run("io.containerd.kata.v2", containerdshim.New, shimConfig)
+}

src/runtime/cli/create.go

@@ -0,0 +1,185 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/urfave/cli"
+)
+
+var createCLICommand = cli.Command{
+	Name:  "create",
+	Usage: "Create a container",
+	ArgsUsage: `<container-id>
+
+   <container-id> is your name for the instance of the container that you
+   are starting. The name you provide for the container instance must be unique
+   on your host.`,
+	Description: `The create command creates an instance of a container for a bundle. The
+   bundle is a directory with a specification file named "` + specConfig + `" and a
+   root filesystem.
+   The specification file includes an args parameter. The args parameter is
+   used to specify command(s) that get run when the container is started.
+   To change the command(s) that get executed on start, edit the args
+   parameter of the spec.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "bundle, b",
+			Value: "",
+			Usage: `path to the root of the bundle directory, defaults to the current directory`,
+		},
+		cli.StringFlag{
+			Name:  "console",
+			Value: "",
+			Usage: "path to a pseudo terminal",
+		},
+		cli.StringFlag{
+			Name:  "console-socket",
+			Value: "",
+			Usage: "path to an AF_UNIX socket which will receive a file descriptor referencing the master end of the console's pseudoterminal",
+		},
+		cli.StringFlag{
+			Name:  "pid-file",
+			Value: "",
+			Usage: "specify the file to write the process id to",
+		},
+		cli.BoolFlag{
+			Name:  "no-pivot",
+			Usage: "warning: this flag is meaningless to kata-runtime, just defined in order to be compatible with docker in ramdisk",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("invalid runtime config")
+		}
+
+		console, err := setupConsole(context.String("console"), context.String("console-socket"))
+		if err != nil {
+			return err
+		}
+
+		return create(ctx, context.Args().First(),
+			context.String("bundle"),
+			console,
+			context.String("pid-file"),
+			true,
+			context.Bool("systemd-cgroup"),
+			runtimeConfig,
+		)
+	},
+}
+
+func create(ctx context.Context, containerID, bundlePath, console, pidFilePath string, detach, systemdCgroup bool,
+	runtimeConfig oci.RuntimeConfig) error {
+	var err error
+
+	span, ctx := katautils.Trace(ctx, "create")
+	defer span.Finish()
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	if bundlePath == "" {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
+		kataLog.WithField("directory", cwd).Debug("Defaulting bundle path to current directory")
+
+		bundlePath = cwd
+	}
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	if bundlePath, err = validCreateParams(ctx, containerID, bundlePath); err != nil {
+		return err
+	}
+
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	if err != nil {
+		return err
+	}
+
+	containerType, err := oci.ContainerType(ociSpec)
+	if err != nil {
+		return err
+	}
+
+	katautils.HandleFactory(ctx, vci, &runtimeConfig)
+
+	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
+
+	//rootfs has been mounted by containerd shim
+	rootFs := vc.RootFs{Mounted: true}
+
+	var process vc.Process
+	switch containerType {
+	case vc.PodSandbox:
+		_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
+		if err != nil {
+			return err
+		}
+	case vc.PodContainer:
+		process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Creation of PID file has to be the last thing done in the create
+	// because containerd considers the create complete after this file
+	// is created.
+	return createPIDFile(ctx, pidFilePath, process.Pid)
+}
+
+func createPIDFile(ctx context.Context, pidFilePath string, pid int) error {
+	span, _ := katautils.Trace(ctx, "createPIDFile")
+	defer span.Finish()
+
+	if pidFilePath == "" {
+		// runtime should not fail since pid file is optional
+		return nil
+	}
+
+	if err := os.RemoveAll(pidFilePath); err != nil {
+		return err
+	}
+
+	f, err := os.Create(pidFilePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	pidStr := fmt.Sprintf("%d", pid)
+
+	n, err := f.WriteString(pidStr)
+	if err != nil {
+		return err
+	}
+
+	if n < len(pidStr) {
+		return fmt.Errorf("Could not write pid to '%s': only %d bytes written out of %d", pidFilePath, n, len(pidStr))
+	}
+
+	return nil
+}

src/runtime/cli/create_test.go

@@ -0,0 +1,724 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	ktu "github.com/kata-containers/runtime/pkg/katatestutils"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+const (
+	testPID                     = 100
+	testConsole                 = "/dev/pts/999"
+	testContainerTypeAnnotation = "io.kubernetes.cri-o.ContainerType"
+	testContainerTypeSandbox    = "sandbox"
+	testContainerTypeContainer  = "container"
+)
+
+var (
+	testStrPID      = fmt.Sprintf("%d", testPID)
+	ctrsMapTreePath = "/var/run/kata-containers/containers-mapping"
+)
+
+func TestCreatePIDFileSuccessful(t *testing.T) {
+	pidDirPath, err := ioutil.TempDir(testDir, "pid-path-")
+	if err != nil {
+		t.Fatalf("Could not create temporary PID directory: %s", err)
+	}
+
+	pidFilePath := filepath.Join(pidDirPath, "pid-file-path")
+	if err := createPIDFile(context.Background(), pidFilePath, testPID); err != nil {
+		t.Fatal(err)
+	}
+
+	fileBytes, err := ioutil.ReadFile(pidFilePath)
+	if err != nil {
+		os.RemoveAll(pidFilePath)
+		t.Fatalf("Could not read %q: %s", pidFilePath, err)
+	}
+
+	if string(fileBytes) != testStrPID {
+		os.RemoveAll(pidFilePath)
+		t.Fatalf("PID %s read from %q different from expected PID %s", string(fileBytes), pidFilePath, testStrPID)
+	}
+
+	os.RemoveAll(pidFilePath)
+}
+
+func TestCreatePIDFileEmptyPathSuccessful(t *testing.T) {
+	file := ""
+	if err := createPIDFile(context.Background(), file, testPID); err != nil {
+		t.Fatalf("This test should not fail (pidFilePath %q, pid %d)", file, testPID)
+	}
+}
+
+func TestCreatePIDFileUnableToRemove(t *testing.T) {
+	if tc.NotValid(ktu.NeedNonRoot()) {
+		// The os.FileMode(0000) trick doesn't work for root.
+		t.Skip(ktu.TestDisabledNeedNonRoot)
+	}
+
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	subdir := filepath.Join(tmpdir, "dir")
+	file := filepath.Join(subdir, "pidfile")
+
+	// stop non-root user from removing the directory later
+	err = os.MkdirAll(subdir, os.FileMode(0000))
+	assert.NoError(err)
+
+	err = createPIDFile(context.Background(), file, testPID)
+	assert.Error(err)
+
+	// let it be deleted
+	os.Chmod(subdir, testDirMode)
+}
+
+func TestCreatePIDFileUnableToCreate(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	subdir := filepath.Join(tmpdir, "dir")
+	file := filepath.Join(subdir, "pidfile")
+
+	err = createPIDFile(context.Background(), file, testPID)
+
+	// subdir doesn't exist
+	assert.Error(err)
+	os.Chmod(subdir, testDirMode)
+}
+
+func TestCreateCLIFunctionNoRuntimeConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+	ctx.App.Metadata["foo"] = "bar"
+
+	fn, ok := createCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err := fn(ctx)
+
+	// no runtime config in the Metadata
+	assert.Error(err)
+}
+
+func TestCreateCLIFunctionSetupConsoleFail(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	subdir := filepath.Join(tmpdir, "dir")
+
+	// does not exist
+	consoleSocketPath := filepath.Join(subdir, "console")
+
+	set := flag.NewFlagSet("", 0)
+
+	set.String("console-socket", consoleSocketPath, "")
+
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	fn, ok := createCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+
+	// failed to setup console
+	assert.Error(err)
+}
+
+func TestCreateCLIFunctionCreateFail(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	set := flag.NewFlagSet("", 0)
+
+	set.String("console-socket", "", "")
+
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	fn, ok := createCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+
+	// create() failed
+	assert.Error(err)
+}
+
+func TestCreateInvalidArgs(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+			{MockID: testContainerID},
+			{MockID: testContainerID},
+		},
+	}
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	type testData struct {
+		containerID   string
+		bundlePath    string
+		console       string
+		pidFilePath   string
+		detach        bool
+		systemdCgroup bool
+		runtimeConfig oci.RuntimeConfig
+	}
+
+	data := []testData{
+		{"", "", "", "", false, false, oci.RuntimeConfig{}},
+		{"", "", "", "", true, true, oci.RuntimeConfig{}},
+		{"foo", "", "", "", true, false, oci.RuntimeConfig{}},
+		{testContainerID, bundlePath, testConsole, pidFilePath, false, false, runtimeConfig},
+		{testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig},
+	}
+
+	for i, d := range data {
+		err := create(context.Background(), d.containerID, d.bundlePath, d.console, d.pidFilePath, d.detach, d.systemdCgroup, d.runtimeConfig)
+		assert.Errorf(err, "test %d (%+v)", i, d)
+	}
+}
+
+func TestCreateInvalidConfigJSON(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	f, err := os.OpenFile(ociConfigFile, os.O_APPEND|os.O_WRONLY, testFileMode)
+	assert.NoError(err)
+
+	// invalidate the JSON
+	_, err = f.WriteString("{")
+	assert.NoError(err)
+	f.Close()
+
+	for detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig)
+		assert.Errorf(err, "%+v", detach)
+		assert.False(vcmock.IsMockError(err))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateInvalidContainerType(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	// Force an invalid container type
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = "I-am-not-a-valid-container-type"
+
+	// rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig)
+		assert.Errorf(err, "%+v", detach)
+		assert.False(vcmock.IsMockError(err))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateContainerInvalid(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := compatoci.ParseConfigJSON(bundlePath)
+
+	assert.NoError(err)
+
+	// Force createContainer() to be called.
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
+
+	// rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig)
+		assert.Errorf(err, "%+v", detach)
+		assert.False(vcmock.IsMockError(err))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateProcessCgroupsPathSuccessful(t *testing.T) {
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip(ktu.TestDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+		},
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	// Force sandbox-type container
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeSandbox
+
+	// Set a limit to ensure processCgroupsPath() considers the
+	// cgroup part of the spec
+	limit := int64(1024 * 1024)
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	// Set an absolute path
+	spec.Linux.CgroupsPath = "/this/is/a/cgroup/path"
+
+	var mounts []specs.Mount
+	foundMount := false
+
+	// Replace the standard cgroup destination with a temporary one.
+	for _, mount := range spec.Mounts {
+		if mount.Type == "cgroup" {
+			foundMount = true
+			cgroupDir, err := ioutil.TempDir("", "cgroup")
+			assert.NoError(err)
+
+			defer os.RemoveAll(cgroupDir)
+			mount.Destination = cgroupDir
+		}
+
+		mounts = append(mounts, mount)
+	}
+
+	assert.True(foundMount)
+
+	// Replace mounts with the newly created one.
+	spec.Mounts = mounts
+
+	// Rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	err = create(context.Background(), testContainerID, "", testConsole, pidFilePath, false, true, runtimeConfig)
+	assert.Error(err, "bundle path not set")
+
+	re := regexp.MustCompile("config.json.*no such file or directory")
+	matches := re.FindAllStringSubmatch(err.Error(), -1)
+	assert.NotEmpty(matches)
+
+	for _, detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, detach, true, runtimeConfig)
+		assert.NoError(err, "detached: %+v", detach)
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateCreateCgroupsFilesFail(t *testing.T) {
+	if tc.NotValid(ktu.NeedNonRoot()) {
+		// The os.FileMode(0000) trick doesn't work for root.
+		t.Skip(ktu.TestDisabledNeedNonRoot)
+	}
+
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+		},
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	// Force sandbox-type container
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeSandbox
+
+	// Set a limit to ensure processCgroupsPath() considers the
+	// cgroup part of the spec
+	limit := int64(1024 * 1024)
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	// Override
+	cgroupsDirPath = filepath.Join(tmpdir, "cgroups")
+	err = os.MkdirAll(cgroupsDirPath, testDirMode)
+	assert.NoError(err)
+
+	// Set a relative path
+	spec.Linux.CgroupsPath = "./a/relative/path"
+
+	dir := filepath.Join(cgroupsDirPath, "memory")
+
+	// Stop directory from being created
+	err = os.MkdirAll(dir, os.FileMode(0000))
+	assert.NoError(err)
+
+	// Rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig)
+		assert.Errorf(err, "%+v", detach)
+		assert.False(vcmock.IsMockError(err))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateCreateCreatePidFileFail(t *testing.T) {
+	if tc.NotValid(ktu.NeedNonRoot()) {
+		// The os.FileMode(0000) trick doesn't work for root.
+		t.Skip(ktu.TestDisabledNeedNonRoot)
+	}
+
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+		},
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidDir := filepath.Join(tmpdir, "pid")
+	pidFilePath := filepath.Join(pidDir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	// Force sandbox-type container
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeSandbox
+
+	// Set a limit to ensure processCgroupsPath() considers the
+	// cgroup part of the spec
+	limit := int64(1024 * 1024)
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	// Rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	// stop the pidfile from being created
+	err = os.MkdirAll(pidDir, os.FileMode(0000))
+	assert.NoError(err)
+
+	for detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig)
+		assert.Errorf(err, "%+v", detach)
+		assert.False(vcmock.IsMockError(err))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreate(t *testing.T) {
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip(ktu.TestDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+		},
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	// Force sandbox-type container
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeSandbox
+
+	// Set a limit to ensure processCgroupsPath() considers the
+	// cgroup part of the spec
+	limit := int64(1024 * 1024)
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	// Rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for detach := range []bool{true, false} {
+		err := create(context.Background(), testContainerID, bundlePath, testConsole, pidFilePath, true, true, runtimeConfig)
+		assert.NoError(err, "%+v", detach)
+		os.RemoveAll(path)
+	}
+}

src/runtime/cli/delete.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnot "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var deleteCLICommand = cli.Command{
+	Name:  "delete",
+	Usage: "Delete any resources held by one or more containers",
+	ArgsUsage: `<container-id> [container-id...]
+
+   <container-id> is the name for the instance of the container.
+
+EXAMPLE:
+   If the container id is "ubuntu01" and ` + name + ` list currently shows the
+   status of "ubuntu01" as "stopped" the following will delete resources held
+   for "ubuntu01" removing "ubuntu01" from the ` + name + ` list of containers:
+
+       # ` + name + ` delete ubuntu01`,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "force, f",
+			Usage: "Forcibly deletes the container if it is still running (uses SIGKILL)",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		args := context.Args()
+		if !args.Present() {
+			return fmt.Errorf("Missing container ID, should at least provide one")
+		}
+
+		force := context.Bool("force")
+		for _, cID := range []string(args) {
+			if err := delete(ctx, cID, force); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
+func delete(ctx context.Context, containerID string, force bool) error {
+	span, ctx := katautils.Trace(ctx, "delete")
+	defer span.Finish()
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		if force {
+			kataLog.Warnf("Failed to get container, force will not fail: %s", err)
+			return nil
+		}
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+
+	span.SetTag("container", containerID)
+	span.SetTag("sandbox", sandboxID)
+
+	containerType, err := oci.GetContainerType(status.Annotations)
+	if err != nil {
+		return err
+	}
+
+	// Retrieve OCI spec configuration.
+	ociSpec, err := oci.GetOCIConfig(status)
+	if err != nil {
+		return err
+	}
+
+	forceStop := false
+	if oci.StateToOCIState(status.State.State) == oci.StateRunning {
+		if !force {
+			return fmt.Errorf("Container still running, should be stopped")
+		}
+
+		forceStop = true
+	}
+
+	switch containerType {
+	case vc.PodSandbox:
+		if err := deleteSandbox(ctx, sandboxID, force); err != nil {
+			return err
+		}
+	case vc.PodContainer:
+		if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("Invalid container type found")
+	}
+
+	// Run post-stop OCI hooks.
+	if err := katautils.PostStopHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]); err != nil {
+		return err
+	}
+
+	return katautils.DelContainerIDMapping(ctx, containerID)
+}
+
+func deleteSandbox(ctx context.Context, sandboxID string, force bool) error {
+	span, _ := katautils.Trace(ctx, "deleteSandbox")
+	defer span.Finish()
+
+	status, err := vci.StatusSandbox(ctx, sandboxID)
+	if err != nil {
+		return err
+	}
+
+	if oci.StateToOCIState(status.State.State) != oci.StateStopped {
+		if _, err := vci.StopSandbox(ctx, sandboxID, force); err != nil {
+			return err
+		}
+	}
+
+	if _, err := vci.DeleteSandbox(ctx, sandboxID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deleteContainer(ctx context.Context, sandboxID, containerID string, forceStop bool) error {
+	span, _ := katautils.Trace(ctx, "deleteContainer")
+	defer span.Finish()
+
+	if forceStop {
+		if _, err := vci.StopContainer(ctx, sandboxID, containerID); err != nil {
+			return err
+		}
+	}
+
+	if _, err := vci.DeleteContainer(ctx, sandboxID, containerID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeCgroupsPath(ctx context.Context, containerID string, cgroupsPathList []string) error {
+	span, _ := katautils.Trace(ctx, "removeCgroupsPath")
+	defer span.Finish()
+
+	if len(cgroupsPathList) == 0 {
+		kataLog.WithField("container", containerID).Info("Cgroups files not removed because cgroupsPath was empty")
+		return nil
+	}
+
+	for _, cgroupsPath := range cgroupsPathList {
+		if err := os.RemoveAll(cgroupsPath); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

src/runtime/cli/delete_test.go

@@ -0,0 +1,626 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+
+	ktu "github.com/kata-containers/runtime/pkg/katatestutils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+)
+
+func testRemoveCgroupsPathSuccessful(t *testing.T, cgroupsPathList []string) {
+	if err := removeCgroupsPath(context.Background(), "foo", cgroupsPathList); err != nil {
+		t.Fatalf("This test should succeed (cgroupsPathList = %v): %s", cgroupsPathList, err)
+	}
+}
+
+func TestRemoveCgroupsPathEmptyPathSuccessful(t *testing.T) {
+	testRemoveCgroupsPathSuccessful(t, []string{})
+}
+
+func TestRemoveCgroupsPathNonEmptyPathSuccessful(t *testing.T) {
+	cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
+	if err != nil {
+		t.Fatalf("Could not create temporary cgroups directory: %s", err)
+	}
+
+	if err := os.MkdirAll(cgroupsPath, testDirMode); err != nil {
+		t.Fatalf("CgroupsPath directory %q could not be created: %s", cgroupsPath, err)
+	}
+
+	testRemoveCgroupsPathSuccessful(t, []string{cgroupsPath})
+
+	if _, err := os.Stat(cgroupsPath); err == nil {
+		t.Fatalf("CgroupsPath directory %q should have been removed: %s", cgroupsPath, err)
+	}
+}
+
+func TestDeleteInvalidContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	// Missing container id
+	err := delete(context.Background(), "", false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	// Container missing in ListSandbox
+	err = delete(context.Background(), testContainerID, false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	// Force to delete missing container
+	err = delete(context.Background(), "non-existing-test", true)
+	assert.NoError(err)
+}
+
+func TestDeleteMissingContainerTypeAnnotation(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          sandbox.ID(),
+			Annotations: map[string]string{},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestDeleteInvalidConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func testConfigSetup(t *testing.T) (rootPath string, bundlePath string) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+
+	bundlePath = filepath.Join(tmpdir, "bundle")
+	err = os.MkdirAll(bundlePath, testDirMode)
+	assert.NoError(err)
+
+	err = createOCIConfig(bundlePath)
+	assert.NoError(err)
+
+	return tmpdir, bundlePath
+}
+
+func TestDeleteSandbox(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+			},
+			State: types.ContainerState{
+				State: "ready",
+			},
+			Spec: &ociSpec,
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
+		return vc.SandboxStatus{
+			ID: sandbox.ID(),
+			State: types.SandboxState{
+				State: types.StateReady,
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusSandboxFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StopSandboxFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.DeleteSandboxFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Nil(err)
+}
+
+func TestDeleteInvalidContainerType(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: "InvalidType",
+			},
+			State: types.ContainerState{
+				State: "created",
+			},
+			Spec: &ociSpec,
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	// Delete an invalid container type
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestDeleteSandboxRunning(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+			},
+			State: types.ContainerState{
+				State: "running",
+			},
+			Spec: &ociSpec,
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	// Delete on a running sandbox should fail
+	err = delete(context.Background(), sandbox.ID(), false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
+		return vc.SandboxStatus{
+			ID: sandbox.ID(),
+			State: types.SandboxState{
+				State: types.StateRunning,
+			},
+		}, nil
+	}
+
+	testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StatusSandboxFunc = nil
+		testingImpl.StopSandboxFunc = nil
+	}()
+
+	// Force delete a running sandbox
+	err = delete(context.Background(), sandbox.ID(), true)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.DeleteSandboxFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.ID(), true)
+	assert.Nil(err)
+}
+
+func TestDeleteRunningContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	path, err := createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.MockContainers[0].ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			State: types.ContainerState{
+				State: "running",
+			},
+			Spec: &ociSpec,
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	// Delete on a running container should fail.
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), false)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err = createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	// force delete
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), true)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.StopContainerFunc = testStopContainerFuncReturnNil
+	defer func() {
+		testingImpl.StopContainerFunc = nil
+	}()
+
+	path, err = createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), true)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return &vcmock.Container{}, nil
+	}
+
+	defer func() {
+		testingImpl.DeleteContainerFunc = nil
+	}()
+
+	path, err = createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), true)
+	assert.Nil(err)
+}
+
+func TestDeleteContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	path, err := createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.MockContainers[0].ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			State: types.ContainerState{
+				State: "ready",
+			},
+			Spec: &ociSpec,
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), false)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	path, err = createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StopContainerFunc = testStopContainerFuncReturnNil
+	defer func() {
+		testingImpl.StopContainerFunc = nil
+	}()
+
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), false)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return &vcmock.Container{}, nil
+	}
+
+	defer func() {
+		testingImpl.DeleteContainerFunc = nil
+	}()
+
+	path, err = createTempContainerIDMapping(sandbox.MockContainers[0].ID(), sandbox.MockContainers[0].ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	err = delete(context.Background(), sandbox.MockContainers[0].ID(), false)
+	assert.Nil(err)
+}
+
+func TestDeleteCLIFunction(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := &flag.FlagSet{}
+	ctx := createCLIContext(flagSet)
+
+	fn, ok := deleteCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	// no container id in the Metadata
+	err := fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err := createTempContainerIDMapping("xyz", "xyz")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	flagSet = flag.NewFlagSet("container-id", flag.ContinueOnError)
+	flagSet.Parse([]string{"xyz"})
+	ctx = createCLIContext(flagSet)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestDeleteCLIFunctionSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+			},
+			State: types.ContainerState{
+				State: "ready",
+			},
+			Spec: &ociSpec,
+		}, nil
+	}
+
+	testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
+		return vc.SandboxStatus{
+			ID: sandbox.ID(),
+			State: types.SandboxState{
+				State: types.StateReady,
+			},
+		}, nil
+	}
+
+	testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StopSandboxFunc = nil
+		testingImpl.DeleteSandboxFunc = nil
+	}()
+
+	flagSet := &flag.FlagSet{}
+	ctx := createCLIContext(flagSet)
+
+	fn, ok := deleteCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	flagSet = flag.NewFlagSet("container-id", flag.ContinueOnError)
+	flagSet.Parse([]string{sandbox.ID()})
+	ctx = createCLIContext(flagSet)
+	assert.NotNil(ctx)
+
+	err = fn(ctx)
+	assert.NoError(err)
+}
+
+func TestRemoveCGroupsPath(t *testing.T) {
+	if tc.NotValid(ktu.NeedNonRoot()) {
+		t.Skip(ktu.TestDisabledNeedNonRoot)
+	}
+
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	dir := filepath.Join(tmpdir, "dir")
+
+	err = os.Mkdir(dir, testDirMode)
+	assert.NoError(err)
+
+	// make directory unreadable by non-root user
+	err = os.Chmod(tmpdir, 0000)
+	assert.NoError(err)
+	defer func() {
+		_ = os.Chmod(tmpdir, 0755)
+	}()
+
+	err = removeCgroupsPath(context.Background(), "foo", []string{dir})
+	assert.Error(err)
+}

src/runtime/cli/events.go

@@ -0,0 +1,295 @@
+// Copyright (c) 2014,2015,2016,2017 Docker, Inc.
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type event struct {
+	Type string      `json:"type"`
+	ID   string      `json:"id"`
+	Data interface{} `json:"data,omitempty"`
+}
+
+// stats is the runc specific stats structure for stability when encoding and decoding stats.
+type stats struct {
+	CPU      cpu                `json:"cpu"`
+	Memory   memory             `json:"memory"`
+	Pids     pids               `json:"pids"`
+	Blkio    blkio              `json:"blkio"`
+	Hugetlb  map[string]hugetlb `json:"hugetlb"`
+	IntelRdt intelRdt           `json:"intel_rdt"`
+}
+
+type hugetlb struct {
+	Usage   uint64 `json:"usage,omitempty"`
+	Max     uint64 `json:"max,omitempty"`
+	Failcnt uint64 `json:"failcnt"`
+}
+
+type blkioEntry struct {
+	Major uint64 `json:"major,omitempty"`
+	Minor uint64 `json:"minor,omitempty"`
+	Op    string `json:"op,omitempty"`
+	Value uint64 `json:"value,omitempty"`
+}
+
+type blkio struct {
+	IoServiceBytesRecursive []blkioEntry `json:"ioServiceBytesRecursive,omitempty"`
+	IoServicedRecursive     []blkioEntry `json:"ioServicedRecursive,omitempty"`
+	IoQueuedRecursive       []blkioEntry `json:"ioQueueRecursive,omitempty"`
+	IoServiceTimeRecursive  []blkioEntry `json:"ioServiceTimeRecursive,omitempty"`
+	IoWaitTimeRecursive     []blkioEntry `json:"ioWaitTimeRecursive,omitempty"`
+	IoMergedRecursive       []blkioEntry `json:"ioMergedRecursive,omitempty"`
+	IoTimeRecursive         []blkioEntry `json:"ioTimeRecursive,omitempty"`
+	SectorsRecursive        []blkioEntry `json:"sectorsRecursive,omitempty"`
+}
+
+type pids struct {
+	Current uint64 `json:"current,omitempty"`
+	Limit   uint64 `json:"limit,omitempty"`
+}
+
+type throttling struct {
+	Periods          uint64 `json:"periods,omitempty"`
+	ThrottledPeriods uint64 `json:"throttledPeriods,omitempty"`
+	ThrottledTime    uint64 `json:"throttledTime,omitempty"`
+}
+
+type cpuUsage struct {
+	// Units: nanoseconds.
+	Total  uint64   `json:"total,omitempty"`
+	Percpu []uint64 `json:"percpu,omitempty"`
+	Kernel uint64   `json:"kernel"`
+	User   uint64   `json:"user"`
+}
+
+type cpu struct {
+	Usage      cpuUsage   `json:"usage,omitempty"`
+	Throttling throttling `json:"throttling,omitempty"`
+}
+
+type memoryEntry struct {
+	Limit   uint64 `json:"limit"`
+	Usage   uint64 `json:"usage,omitempty"`
+	Max     uint64 `json:"max,omitempty"`
+	Failcnt uint64 `json:"failcnt"`
+}
+
+type memory struct {
+	Cache     uint64            `json:"cache,omitempty"`
+	Usage     memoryEntry       `json:"usage,omitempty"`
+	Swap      memoryEntry       `json:"swap,omitempty"`
+	Kernel    memoryEntry       `json:"kernel,omitempty"`
+	KernelTCP memoryEntry       `json:"kernelTCP,omitempty"`
+	Raw       map[string]uint64 `json:"raw,omitempty"`
+}
+
+type l3CacheInfo struct {
+	CbmMask    string `json:"cbm_mask,omitempty"`
+	MinCbmBits uint64 `json:"min_cbm_bits,omitempty"`
+	NumClosids uint64 `json:"num_closids,omitempty"`
+}
+
+type intelRdt struct {
+	// The read-only L3 cache information
+	L3CacheInfo *l3CacheInfo `json:"l3_cache_info,omitempty"`
+
+	// The read-only L3 cache schema in root
+	L3CacheSchemaRoot string `json:"l3_cache_schema_root,omitempty"`
+
+	// The L3 cache schema in 'container_id' group
+	L3CacheSchema string `json:"l3_cache_schema,omitempty"`
+}
+
+var eventsCLICommand = cli.Command{
+	Name:  "events",
+	Usage: "display container events such as OOM notifications, cpu, memory, and IO usage statistics",
+	ArgsUsage: `<container-id>
+
+Where "<container-id>" is the name for the instance of the container.`,
+	Description: `The events command displays information about the container. By default the
+information is displayed once every 5 seconds.`,
+	Flags: []cli.Flag{
+		cli.DurationFlag{
+			Name:  "interval",
+			Value: 5 * time.Second,
+			Usage: "set the stats collection interval",
+		},
+		cli.BoolFlag{
+			Name:  "stats",
+			Usage: "display the container's stats then exit",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, _ := katautils.Trace(ctx, "events")
+		defer span.Finish()
+
+		containerID := context.Args().First()
+		if containerID == "" {
+			return fmt.Errorf("container id cannot be empty")
+		}
+
+		kataLog = kataLog.WithField("container", containerID)
+		setExternalLoggers(ctx, kataLog)
+		span.SetTag("container", containerID)
+
+		duration := context.Duration("interval")
+		if duration <= 0 {
+			return fmt.Errorf("duration interval must be greater than 0")
+		}
+
+		status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+		if err != nil {
+			return err
+		}
+
+		containerID = status.ID
+
+		kataLog = kataLog.WithFields(logrus.Fields{
+			"container": containerID,
+			"sandbox":   sandboxID,
+		})
+
+		setExternalLoggers(ctx, kataLog)
+		span.SetTag("container", containerID)
+		span.SetTag("sandbox", sandboxID)
+
+		if status.State.State == types.StateStopped {
+			return fmt.Errorf("container with id %s is not running", status.ID)
+		}
+
+		var (
+			events = make(chan *event, 1024)
+			group  = &sync.WaitGroup{}
+		)
+		group.Add(1)
+
+		go func() {
+			defer group.Done()
+			enc := json.NewEncoder(os.Stdout)
+			for e := range events {
+				if err := enc.Encode(e); err != nil {
+					logrus.Error(err)
+				}
+			}
+		}()
+
+		if context.Bool("stats") {
+			s, err := vci.StatsContainer(ctx, sandboxID, containerID)
+			if err != nil {
+				return err
+			}
+			events <- &event{Type: "stats", ID: status.ID, Data: convertVirtcontainerStats(&s)}
+			close(events)
+			group.Wait()
+			return nil
+		}
+
+		go func() {
+			for range time.Tick(context.Duration("interval")) {
+				s, err := vci.StatsContainer(ctx, sandboxID, containerID)
+				if err != nil {
+					logrus.Error(err)
+					continue
+				}
+				events <- &event{Type: "stats", ID: status.ID, Data: convertVirtcontainerStats(&s)}
+			}
+		}()
+
+		group.Wait()
+		return nil
+	},
+}
+
+func convertVirtcontainerStats(containerStats *vc.ContainerStats) *stats {
+	cg := containerStats.CgroupStats
+	if cg == nil {
+		return nil
+	}
+	var s stats
+	s.Pids.Current = cg.PidsStats.Current
+	s.Pids.Limit = cg.PidsStats.Limit
+
+	s.CPU.Usage.Kernel = cg.CPUStats.CPUUsage.UsageInKernelmode
+	s.CPU.Usage.User = cg.CPUStats.CPUUsage.UsageInUsermode
+	s.CPU.Usage.Total = cg.CPUStats.CPUUsage.TotalUsage
+	s.CPU.Usage.Percpu = cg.CPUStats.CPUUsage.PercpuUsage
+	s.CPU.Throttling.Periods = cg.CPUStats.ThrottlingData.Periods
+	s.CPU.Throttling.ThrottledPeriods = cg.CPUStats.ThrottlingData.ThrottledPeriods
+	s.CPU.Throttling.ThrottledTime = cg.CPUStats.ThrottlingData.ThrottledTime
+
+	s.Memory.Cache = cg.MemoryStats.Cache
+	s.Memory.Kernel = convertMemoryEntry(cg.MemoryStats.KernelUsage)
+	s.Memory.KernelTCP = convertMemoryEntry(cg.MemoryStats.KernelTCPUsage)
+	s.Memory.Swap = convertMemoryEntry(cg.MemoryStats.SwapUsage)
+	s.Memory.Usage = convertMemoryEntry(cg.MemoryStats.Usage)
+	s.Memory.Raw = cg.MemoryStats.Stats
+
+	s.Blkio.IoServiceBytesRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceBytesRecursive)
+	s.Blkio.IoServicedRecursive = convertBlkioEntry(cg.BlkioStats.IoServicedRecursive)
+	s.Blkio.IoQueuedRecursive = convertBlkioEntry(cg.BlkioStats.IoQueuedRecursive)
+	s.Blkio.IoServiceTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceTimeRecursive)
+	s.Blkio.IoWaitTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoWaitTimeRecursive)
+	s.Blkio.IoMergedRecursive = convertBlkioEntry(cg.BlkioStats.IoMergedRecursive)
+	s.Blkio.IoTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoTimeRecursive)
+	s.Blkio.SectorsRecursive = convertBlkioEntry(cg.BlkioStats.SectorsRecursive)
+
+	s.Hugetlb = make(map[string]hugetlb)
+	for k, v := range cg.HugetlbStats {
+		s.Hugetlb[k] = convertHugtlb(v)
+	}
+
+	return &s
+}
+
+func convertHugtlb(c vc.HugetlbStats) hugetlb {
+	return hugetlb{
+		Usage:   c.Usage,
+		Max:     c.MaxUsage,
+		Failcnt: c.Failcnt,
+	}
+}
+
+func convertMemoryEntry(c vc.MemoryData) memoryEntry {
+	return memoryEntry{
+		Limit:   c.Limit,
+		Usage:   c.Usage,
+		Max:     c.MaxUsage,
+		Failcnt: c.Failcnt,
+	}
+}
+
+func convertBlkioEntry(c []vc.BlkioStatEntry) []blkioEntry {
+	var out []blkioEntry
+	for _, e := range c {
+		out = append(out, blkioEntry{
+			Major: e.Major,
+			Minor: e.Minor,
+			Op:    e.Op,
+			Value: e.Value,
+		})
+	}
+	return out
+}

src/runtime/cli/events_test.go

@@ -0,0 +1,144 @@
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+	"time"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func TestEventsCliAction(t *testing.T) {
+	assert := assert.New(t)
+
+	// get Action function
+	actionFunc, ok := eventsCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	flagSet := flag.NewFlagSet("events", flag.ContinueOnError)
+
+	// create a new fake context
+	ctx := createCLIContext(flagSet)
+
+	err := actionFunc(ctx)
+	assert.Error(err, "Missing container ID")
+}
+
+func TestEventsCLIFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := flag.NewFlagSet("events", flag.ContinueOnError)
+	ctx := createCLIContext(flagSet)
+
+	actionFunc, ok := eventsCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	// missing container ID
+	err := actionFunc(ctx)
+	assert.Error(err)
+
+	// interval is negative
+	flagSet.Parse([]string{testContainerID})
+	flagSet.Duration("interval", (-1)*time.Second, "")
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// interval is zero
+	flagSet = flag.NewFlagSet("events", flag.ContinueOnError)
+	flagSet.Parse([]string{testContainerID})
+	flagSet.Duration("interval", 0*time.Second, "")
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// not running
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+	err = actionFunc(ctx)
+	assert.Error(err)
+}
+
+func TestEventsCLISuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			State: types.ContainerState{
+				State: types.StateRunning,
+			},
+		}, nil
+	}
+
+	testingImpl.StatsContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) {
+		return vc.ContainerStats{}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StatsContainerFunc = nil
+	}()
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	actionFunc, ok := eventsCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	flagSet := flag.NewFlagSet("events", flag.ContinueOnError)
+	flagSet.Parse([]string{testContainerID})
+	flagSet.Duration("interval", 5*time.Second, "")
+	flagSet.Bool("stats", true, "")
+	ctx := createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.NoError(err)
+}

src/runtime/cli/exec.go

@@ -0,0 +1,291 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"syscall"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/urfave/cli"
+)
+
+type execParams struct {
+	ociProcess   specs.Process
+	cID          string
+	pidFile      string
+	console      string
+	consoleSock  string
+	processLabel string
+	detach       bool
+	noSubreaper  bool
+}
+
+var execCLICommand = cli.Command{
+	Name:  "exec",
+	Usage: "Execute new process inside the container",
+	ArgsUsage: `<container-id> <command> [command options]  || -p process.json <container-id>
+
+   <container-id> is the name for the instance of the container and <command>
+   is the command to be executed in the container. <command> can't be empty
+   unless a "-p" flag provided.
+
+EXAMPLE:
+   If the container is configured to run the linux ps command the following
+   will output a list of processes running in the container:
+
+       # ` + name + ` <container-id> ps`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "console",
+			Usage: "path to a pseudo terminal",
+		},
+		cli.StringFlag{
+			Name:  "console-socket",
+			Value: "",
+			Usage: "path to an AF_UNIX socket which will receive a file descriptor referencing the master end of the console's pseudoterminal",
+		},
+		cli.StringFlag{
+			Name:  "cwd",
+			Usage: "current working directory in the container",
+		},
+		cli.StringSliceFlag{
+			Name:  "env, e",
+			Usage: "set environment variables",
+		},
+		cli.BoolFlag{
+			Name:  "tty, t",
+			Usage: "allocate a pseudo-TTY",
+		},
+		cli.StringFlag{
+			Name:  "user, u",
+			Usage: "UID (format: <uid>[:<gid>])",
+		},
+		cli.StringFlag{
+			Name:  "process, p",
+			Usage: "path to the process.json",
+		},
+		cli.BoolFlag{
+			Name:  "detach,d",
+			Usage: "detach from the container's process",
+		},
+		cli.StringFlag{
+			Name:  "pid-file",
+			Value: "",
+			Usage: "specify the file to write the process id to",
+		},
+		cli.StringFlag{
+			Name:  "process-label",
+			Usage: "set the asm process label for the process commonly used with selinux",
+		},
+		cli.StringFlag{
+			Name:  "apparmor",
+			Usage: "set the apparmor profile for the process",
+		},
+		cli.BoolFlag{
+			Name:  "no-new-privs",
+			Usage: "set the no new privileges value for the process",
+		},
+		cli.StringSliceFlag{
+			Name:  "cap, c",
+			Value: &cli.StringSlice{},
+			Usage: "add a capability to the bounding set for the process",
+		},
+		cli.BoolFlag{
+			Name:   "no-subreaper",
+			Usage:  "disable the use of the subreaper used to reap reparented processes",
+			Hidden: true,
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		return execute(ctx, context)
+	},
+}
+
+func generateExecParams(context *cli.Context, specProcess *specs.Process) (execParams, error) {
+	ctxArgs := context.Args()
+
+	params := execParams{
+		cID:          ctxArgs.First(),
+		pidFile:      context.String("pid-file"),
+		console:      context.String("console"),
+		consoleSock:  context.String("console-socket"),
+		detach:       context.Bool("detach"),
+		processLabel: context.String("process-label"),
+		noSubreaper:  context.Bool("no-subreaper"),
+	}
+
+	if context.String("process") != "" {
+		var ociProcess specs.Process
+
+		fileContent, err := ioutil.ReadFile(context.String("process"))
+		if err != nil {
+			return execParams{}, err
+		}
+
+		if err := json.Unmarshal(fileContent, &ociProcess); err != nil {
+			return execParams{}, err
+		}
+
+		params.ociProcess = ociProcess
+	} else {
+		params.ociProcess = *specProcess
+
+		// Override terminal
+		if context.IsSet("tty") {
+			params.ociProcess.Terminal = context.Bool("tty")
+		}
+
+		// Override user
+		if context.String("user") != "" {
+			params.ociProcess.User = specs.User{
+				// This field is a Windows-only field
+				// according to the specification. However, it
+				// is abused here to allow the username
+				// specified in the OCI runtime configuration
+				// file to be overridden by a CLI request.
+				Username: context.String("user"),
+			}
+		}
+
+		// Override env
+		params.ociProcess.Env = append(params.ociProcess.Env, context.StringSlice("env")...)
+
+		// Override cwd
+		if context.String("cwd") != "" {
+			params.ociProcess.Cwd = context.String("cwd")
+		}
+
+		// Override no-new-privs
+		if context.IsSet("no-new-privs") {
+			params.ociProcess.NoNewPrivileges = context.Bool("no-new-privs")
+		}
+
+		// Override apparmor
+		if context.String("apparmor") != "" {
+			params.ociProcess.ApparmorProfile = context.String("apparmor")
+		}
+
+		params.ociProcess.Args = ctxArgs.Tail()
+	}
+
+	return params, nil
+}
+
+func execute(ctx context.Context, context *cli.Context) error {
+	span, ctx := katautils.Trace(ctx, "execute")
+	defer span.Finish()
+
+	containerID := context.Args().First()
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	kataLog = kataLog.WithField("sandbox", sandboxID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("sandbox", sandboxID)
+
+	// Retrieve OCI spec configuration.
+	ociSpec, err := oci.GetOCIConfig(status)
+	if err != nil {
+		return err
+	}
+
+	params, err := generateExecParams(context, ociSpec.Process)
+	if err != nil {
+		return err
+	}
+
+	params.cID = status.ID
+	containerID = params.cID
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	// container MUST be ready or running.
+	if status.State.State != types.StateReady &&
+		status.State.State != types.StateRunning {
+		return fmt.Errorf("Container %s is not ready or running",
+			params.cID)
+	}
+
+	envVars, err := oci.EnvVars(params.ociProcess.Env)
+	if err != nil {
+		return err
+	}
+
+	consolePath, err := setupConsole(params.console, params.consoleSock)
+	if err != nil {
+		return err
+	}
+
+	user := fmt.Sprintf("%d:%d", params.ociProcess.User.UID, params.ociProcess.User.GID)
+
+	if params.ociProcess.User.Username != "" {
+		user = params.ociProcess.User.Username
+	}
+
+	cmd := types.Cmd{
+		Args:        params.ociProcess.Args,
+		Envs:        envVars,
+		WorkDir:     params.ociProcess.Cwd,
+		User:        user,
+		Interactive: params.ociProcess.Terminal,
+		Console:     consolePath,
+		Detach:      noNeedForOutput(params.detach, params.ociProcess.Terminal),
+	}
+
+	_, _, process, err := vci.EnterContainer(ctx, sandboxID, params.cID, cmd)
+	if err != nil {
+		return err
+	}
+
+	// Creation of PID file has to be the last thing done in the exec
+	// because containerd considers the exec to have finished starting
+	// after this file is created.
+	if err := createPIDFile(ctx, params.pidFile, process.Pid); err != nil {
+		return err
+	}
+
+	if params.detach {
+		return nil
+	}
+
+	p, err := os.FindProcess(process.Pid)
+	if err != nil {
+		return err
+	}
+
+	ps, err := p.Wait()
+	if err != nil {
+		return fmt.Errorf("Process state %s, container info %+v: %v",
+			ps.String(), status, err)
+	}
+
+	// Exit code has to be forwarded in this case.
+	return cli.NewExitError("", ps.Sys().(syscall.WaitStatus).ExitStatus())
+}

src/runtime/cli/exec_test.go

@@ -0,0 +1,787 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+)
+
+func TestExecCLIFunction(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := &flag.FlagSet{}
+	ctx := createCLIContext(flagSet)
+
+	fn, ok := startCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	// no container-id in the Metadata
+	err := fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err := createTempContainerIDMapping("xyz", "xyz")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	// pass container-id
+	flagSet = flag.NewFlagSet("container-id", flag.ContinueOnError)
+	flagSet.Parse([]string{"xyz"})
+	ctx = createCLIContext(flagSet)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestExecuteErrors(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := flag.NewFlagSet("", 0)
+	ctx := createCLIContext(flagSet)
+
+	// missing container id
+	err := execute(context.Background(), ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	// StatusSandbox error
+	flagSet.Parse([]string{testContainerID})
+	err = execute(context.Background(), ctx)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	// Config path missing in annotations
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, types.ContainerState{}, annotations, &specs.Spec{Process: &specs.Process{}}), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = execute(context.Background(), ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	// Container state undefined
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations = map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	containerState := types.ContainerState{}
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, containerState, annotations, &ociSpec), nil
+	}
+
+	err = execute(context.Background(), ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	// Container paused
+	containerState = types.ContainerState{
+		State: types.StatePaused,
+	}
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, containerState, annotations, &ociSpec), nil
+	}
+
+	err = execute(context.Background(), ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	// Container stopped
+	containerState = types.ContainerState{
+		State: types.StateStopped,
+	}
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, containerState, annotations, &ociSpec), nil
+	}
+
+	err = execute(context.Background(), ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestExecuteErrorReadingProcessJson(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	// non-existent path
+	processPath := filepath.Join(tmpdir, "process.json")
+
+	flagSet := flag.NewFlagSet("", 0)
+	flagSet.String("process", processPath, "")
+	flagSet.Parse([]string{testContainerID})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	// Note: flags can only be tested with the CLI command function
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestExecuteErrorOpeningConsole(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	consoleSock := filepath.Join(tmpdir, "console-sock")
+
+	flagSet := flag.NewFlagSet("", 0)
+	flagSet.String("console-socket", consoleSock, "")
+	flagSet.Parse([]string{testContainerID})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	// Note: flags can only be tested with the CLI command function
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func testExecParamsSetup(t *testing.T, pidFilePath, consolePath string, detach bool) *flag.FlagSet {
+	flagSet := flag.NewFlagSet("", 0)
+
+	flagSet.String("pid-file", pidFilePath, "")
+	flagSet.String("console", consolePath, "")
+	flagSet.String("console-socket", "", "")
+	flagSet.Bool("detach", detach, "")
+	flagSet.String("process-label", "testlabel", "")
+	flagSet.Bool("no-subreaper", false, "")
+
+	return flagSet
+}
+
+func TestExecuteWithFlags(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+
+	flagSet := testExecParamsSetup(t, pidFilePath, consolePath, false)
+	flagSet.String("user", "root", "")
+	flagSet.String("cwd", "/home/root", "")
+	flagSet.String("apparmor", "/tmp/profile", "")
+	flagSet.Bool("no-new-privs", false, "")
+
+	flagSet.Parse([]string{testContainerID, "/tmp/foo"})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	// EnterContainer error
+	err = fn(ctx)
+	assert.Error(err)
+
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.EnterContainerFunc = func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) {
+		return &vcmock.Sandbox{}, &vcmock.Container{}, &vc.Process{}, nil
+	}
+
+	defer func() {
+		testingImpl.EnterContainerFunc = nil
+		os.Remove(pidFilePath)
+	}()
+
+	// Process not running error
+	err = fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	os.Remove(pidFilePath)
+
+	// Process ran and exited successfully
+	testingImpl.EnterContainerFunc = func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) {
+		// create a fake container process
+		workload := []string{"cat", "/dev/null"}
+		command := exec.Command(workload[0], workload[1:]...)
+		err := command.Start()
+		assert.NoError(err, "Unable to start process %v: %s", workload, err)
+
+		vcProcess := vc.Process{}
+		vcProcess.Pid = command.Process.Pid
+		return &vcmock.Sandbox{}, &vcmock.Container{}, &vcProcess, nil
+	}
+
+	defer func() {
+		testingImpl.EnterContainerFunc = nil
+		os.Remove(pidFilePath)
+	}()
+
+	// Should get an exit code when run in non-detached mode.
+	err = fn(ctx)
+	_, ok = err.(*cli.ExitError)
+	assert.True(ok, true, "Exit code not received for fake workload process")
+}
+
+func TestExecuteWithFlagsDetached(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+	detach := true
+
+	flagSet := testExecParamsSetup(t, pidFilePath, consolePath, detach)
+	flagSet.Parse([]string{testContainerID, "/tmp/foo"})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	testingImpl.EnterContainerFunc = func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) {
+		// create a fake container process
+		workload := []string{"cat", "/dev/null"}
+		command := exec.Command(workload[0], workload[1:]...)
+		err := command.Start()
+		assert.NoError(err, "Unable to start process %v: %s", workload, err)
+
+		vcProcess := vc.Process{}
+		vcProcess.Pid = command.Process.Pid
+		return &vcmock.Sandbox{}, &vcmock.Container{}, &vcProcess, nil
+	}
+
+	defer func() {
+		testingImpl.EnterContainerFunc = nil
+		os.Remove(pidFilePath)
+	}()
+
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.NoError(err)
+}
+
+func TestExecuteWithInvalidProcessJson(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+	detach := false
+
+	flagSet := testExecParamsSetup(t, pidFilePath, consolePath, detach)
+
+	processPath := filepath.Join(tmpdir, "process.json")
+	flagSet.String("process", processPath, "")
+
+	f, err := os.OpenFile(processPath, os.O_RDWR|os.O_CREATE, testFileMode)
+	assert.NoError(err)
+
+	// invalidate the JSON
+	_, err = f.WriteString("{")
+	assert.NoError(err)
+	f.Close()
+
+	defer os.Remove(processPath)
+
+	flagSet.Parse([]string{testContainerID})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestExecuteWithValidProcessJson(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+
+	flagSet := testExecParamsSetup(t, pidFilePath, consolePath, false)
+
+	processPath := filepath.Join(tmpdir, "process.json")
+	flagSet.String("process", processPath, "")
+
+	flagSet.Parse([]string{testContainerID, "/tmp/foo"})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	processJSON := `{
+				"consoleSize": {
+					"height": 15,
+					"width": 15
+				},
+				"terminal": true,
+				"user": {
+					"uid": 0,
+					"gid": 0
+				},
+				"args": [
+					"sh"
+				],
+				"env": [
+					"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+					"TERM=xterm"
+				],
+				"cwd": "/"
+			}`
+
+	f, err := os.OpenFile(processPath, os.O_RDWR|os.O_CREATE, testFileMode)
+	assert.NoError(err)
+
+	_, err = f.WriteString(processJSON)
+	assert.NoError(err)
+	f.Close()
+
+	defer os.Remove(processPath)
+
+	workload := []string{"cat", "/dev/null"}
+
+	testingImpl.EnterContainerFunc = func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) {
+		// create a fake container process
+		command := exec.Command(workload[0], workload[1:]...)
+		err := command.Start()
+		assert.NoError(err, "Unable to start process %v: %s", workload, err)
+
+		vcProcess := vc.Process{}
+		vcProcess.Pid = command.Process.Pid
+
+		return &vcmock.Sandbox{}, &vcmock.Container{}, &vcProcess, nil
+	}
+
+	defer func() {
+		testingImpl.EnterContainerFunc = nil
+		os.Remove(pidFilePath)
+	}()
+
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	exitErr, ok := err.(*cli.ExitError)
+	assert.True(ok, true, "Exit code not received for fake workload process")
+	assert.Equal(exitErr.ExitCode(), 0, "Exit code should have been 0 for fake workload %s", workload)
+}
+
+func TestExecuteWithEmptyEnvironmentValue(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+
+	flagSet := testExecParamsSetup(t, pidFilePath, consolePath, false)
+
+	processPath := filepath.Join(tmpdir, "process.json")
+
+	flagSet.String("process", processPath, "")
+	flagSet.Parse([]string{testContainerID})
+	ctx := createCLIContext(flagSet)
+
+	rootPath, bundlePath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+	}
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &ociSpec), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	processJSON := `{
+				"consoleSize": {
+					"height": 15,
+					"width": 15
+				},
+				"terminal": true,
+				"user": {
+					"uid": 0,
+					"gid": 0
+				},
+				"args": [
+					"sh"
+				],
+				"env": [
+					"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+					"TERM="
+				],
+				"cwd": "/"
+
+			}`
+
+	f, err := os.OpenFile(processPath, os.O_RDWR|os.O_CREATE, testFileMode)
+	assert.NoError(err)
+
+	_, err = f.WriteString(processJSON)
+	assert.NoError(err)
+	f.Close()
+
+	defer os.Remove(processPath)
+
+	workload := []string{"cat", "/dev/null"}
+
+	testingImpl.EnterContainerFunc = func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) {
+		// create a fake container process
+		command := exec.Command(workload[0], workload[1:]...)
+		err := command.Start()
+		assert.NoError(err, "Unable to start process %v: %s", workload, err)
+
+		vcProcess := vc.Process{}
+		vcProcess.Pid = command.Process.Pid
+
+		return &vcmock.Sandbox{}, &vcmock.Container{}, &vcProcess, nil
+	}
+
+	defer func() {
+		testingImpl.EnterContainerFunc = nil
+		os.Remove(pidFilePath)
+	}()
+
+	fn, ok := execCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	// vcAnnotations.EnvVars error due to incorrect environment
+	err = fn(ctx)
+	exitErr, ok := err.(*cli.ExitError)
+	assert.True(ok, true, "Exit code not received for fake workload process")
+	assert.Equal(exitErr.ExitCode(), 0, "Exit code should have been 0 for empty environment variable value")
+}
+
+func TestGenerateExecParams(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+	consoleSocket := "/tmp/console-sock"
+	processLabel := "testlabel"
+	user := "root"
+	cwd := "cwd"
+	apparmor := "apparmorProf"
+
+	flagSet := flag.NewFlagSet("", 0)
+	flagSet.String("pid-file", pidFilePath, "")
+	flagSet.String("console", consolePath, "")
+	flagSet.String("console-socket", consoleSocket, "")
+	flagSet.String("process-label", processLabel, "")
+
+	flagSet.String("user", user, "")
+	flagSet.String("cwd", cwd, "")
+	flagSet.String("apparmor", apparmor, "")
+
+	ctx := createCLIContext(flagSet)
+	process := &specs.Process{}
+	params, err := generateExecParams(ctx, process)
+	assert.NoError(err)
+
+	assert.Equal(params.pidFile, pidFilePath)
+	assert.Equal(params.console, consolePath)
+	assert.Equal(params.consoleSock, consoleSocket)
+	assert.Equal(params.processLabel, processLabel)
+	assert.Equal(params.noSubreaper, false)
+	assert.Equal(params.detach, false)
+
+	assert.Equal(params.ociProcess.Terminal, false)
+	assert.Equal(params.ociProcess.User.UID, uint32(0))
+	assert.Equal(params.ociProcess.User.GID, uint32(0))
+	assert.Equal(params.ociProcess.Cwd, cwd)
+	assert.Equal(params.ociProcess.ApparmorProfile, apparmor)
+}
+
+func TestGenerateExecParamsWithProcessJsonFile(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	pidFilePath := filepath.Join(tmpdir, "pid")
+	consolePath := "/dev/ptmx"
+	consoleSocket := "/tmp/console-sock"
+	detach := true
+	processLabel := "testlabel"
+
+	flagSet := flag.NewFlagSet("", 0)
+	flagSet.String("pid-file", pidFilePath, "")
+	flagSet.String("console", consolePath, "")
+	flagSet.String("console-socket", consoleSocket, "")
+	flagSet.Bool("detach", detach, "")
+	flagSet.String("process-label", processLabel, "")
+
+	processPath := filepath.Join(tmpdir, "process.json")
+	flagSet.String("process", processPath, "")
+
+	flagSet.Parse([]string{testContainerID})
+	ctx := createCLIContext(flagSet)
+
+	processJSON := `{
+				"consoleSize": {
+					"height": 15,
+					"width": 15
+				},
+				"terminal": true,
+				"user": {
+					"uid": 0,
+					"gid": 0
+				},
+				"args": [
+					"sh"
+				],
+				"env": [
+					"TERM=xterm",
+					"foo=bar"
+				],
+				"cwd": "/"
+			}`
+
+	f, err := os.OpenFile(processPath, os.O_RDWR|os.O_CREATE, testFileMode)
+	assert.NoError(err)
+
+	_, err = f.WriteString(processJSON)
+	assert.NoError(err)
+	f.Close()
+
+	defer os.Remove(processPath)
+
+	process := &specs.Process{}
+	params, err := generateExecParams(ctx, process)
+	assert.NoError(err)
+
+	assert.Equal(params.pidFile, pidFilePath)
+	assert.Equal(params.console, consolePath)
+	assert.Equal(params.consoleSock, consoleSocket)
+	assert.Equal(params.processLabel, processLabel)
+	assert.Equal(params.noSubreaper, false)
+	assert.Equal(params.detach, detach)
+
+	assert.Equal(params.ociProcess.Terminal, true)
+	assert.Equal(params.ociProcess.ConsoleSize.Height, uint(15))
+	assert.Equal(params.ociProcess.ConsoleSize.Width, uint(15))
+	assert.Equal(params.ociProcess.User.UID, uint32(0))
+	assert.Equal(params.ociProcess.User.GID, uint32(0))
+	assert.Equal(params.ociProcess.Cwd, "/")
+	assert.Equal(params.ociProcess.Env[0], "TERM=xterm")
+	assert.Equal(params.ociProcess.Env[1], "foo=bar")
+}

src/runtime/cli/exit.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import "os"
+
+var atexitFuncs []func()
+
+var exitFunc = os.Exit
+
+// atexit registers a function f that will be run when exit is called. The
+// handlers so registered will be called the in reverse order of their
+// registration.
+func atexit(f func()) {
+	atexitFuncs = append(atexitFuncs, f)
+}
+
+// exit calls all atexit handlers before exiting the process with status.
+func exit(status int) {
+	for i := len(atexitFuncs) - 1; i >= 0; i-- {
+		f := atexitFuncs[i]
+		f()
+	}
+	exitFunc(status)
+}

src/runtime/cli/exit_test.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var testFoo string
+
+func testFunc() {
+	testFoo = "bar"
+}
+
+func TestExit(t *testing.T) {
+	assert := assert.New(t)
+
+	var testExitStatus int
+	exitFunc = func(status int) {
+		testExitStatus = status
+	}
+
+	defer func() {
+		exitFunc = os.Exit
+	}()
+
+	// test with no atexit functions added.
+	exit(1)
+	assert.Equal(testExitStatus, 1)
+
+	// test with a function added to the atexit list.
+	atexit(testFunc)
+	exit(0)
+	assert.Equal(testFoo, "bar")
+	assert.Equal(testExitStatus, 0)
+}

src/runtime/cli/factory.go

@@ -0,0 +1,335 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+	pb "github.com/kata-containers/runtime/protocols/cache"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vf "github.com/kata-containers/runtime/virtcontainers/factory"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+	"golang.org/x/sys/unix"
+	"google.golang.org/grpc"
+)
+
+var factorySubCmds = []cli.Command{
+	initFactoryCommand,
+	destroyFactoryCommand,
+	statusFactoryCommand,
+}
+
+var factoryCLICommand = cli.Command{
+	Name:        "factory",
+	Usage:       "manage vm factory",
+	Subcommands: factorySubCmds,
+	Action: func(context *cli.Context) {
+		cli.ShowSubcommandHelp(context)
+	},
+}
+
+type cacheServer struct {
+	rpc     *grpc.Server
+	factory vc.Factory
+	done    chan struct{}
+}
+
+var jsonVMConfig *pb.GrpcVMConfig
+
+// Config requests base factory config and convert it to gRPC protocol.
+func (s *cacheServer) Config(ctx context.Context, empty *types.Empty) (*pb.GrpcVMConfig, error) {
+	if jsonVMConfig == nil {
+		config := s.factory.Config()
+
+		var err error
+		jsonVMConfig, err = config.ToGrpc()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return jsonVMConfig, nil
+}
+
+// GetBaseVM requests a paused VM and convert it to gRPC protocol.
+func (s *cacheServer) GetBaseVM(ctx context.Context, empty *types.Empty) (*pb.GrpcVM, error) {
+	config := s.factory.Config()
+
+	vm, err := s.factory.GetBaseVM(ctx, config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to GetBaseVM")
+	}
+
+	return vm.ToGrpc(config)
+}
+
+func (s *cacheServer) quit() {
+	s.rpc.GracefulStop()
+	close(s.done)
+}
+
+// Quit will stop VMCache server after 1 second.
+func (s *cacheServer) Quit(ctx context.Context, empty *types.Empty) (*types.Empty, error) {
+	go func() {
+		kataLog.Info("VM cache server will stop after 1 second")
+		time.Sleep(time.Second)
+		s.quit()
+	}()
+	return &types.Empty{}, nil
+}
+
+func (s *cacheServer) Status(ctx context.Context, empty *types.Empty) (*pb.GrpcStatus, error) {
+	stat := pb.GrpcStatus{
+		Pid:      int64(os.Getpid()),
+		Vmstatus: s.factory.GetVMStatus(),
+	}
+	return &stat, nil
+}
+
+func getUnixListener(path string) (net.Listener, error) {
+	err := os.MkdirAll(filepath.Dir(path), 0755)
+	if err != nil {
+		return nil, err
+	}
+	_, err = os.Stat(path)
+	if err == nil {
+		return nil, fmt.Errorf("%s already exist.  Please stop running VMCache server and remove %s", path, path)
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	}
+	l, err := net.Listen("unix", path)
+	if err != nil {
+		return nil, err
+	}
+	if err = os.Chmod(path, 0600); err != nil {
+		l.Close()
+		return nil, err
+	}
+	return l, nil
+}
+
+var handledSignals = []os.Signal{
+	syscall.SIGTERM,
+	syscall.SIGINT,
+	syscall.SIGPIPE,
+}
+
+func handleSignals(s *cacheServer, signals chan os.Signal) {
+	s.done = make(chan struct{}, 1)
+	go func() {
+		for {
+			sig := <-signals
+			kataLog.WithField("signal", sig).Debug("received signal")
+			switch sig {
+			case unix.SIGPIPE:
+				continue
+			default:
+				s.quit()
+				return
+			}
+		}
+	}()
+}
+
+var initFactoryCommand = cli.Command{
+	Name:  "init",
+	Usage: "initialize a VM factory based on kata-runtime configuration",
+	Action: func(c *cli.Context) error {
+		ctx, err := cliContextToContext(c)
+		if err != nil {
+			return err
+		}
+
+		runtimeConfig, ok := c.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("invalid runtime config")
+		}
+
+		factoryConfig := vf.Config{
+			Template:     runtimeConfig.FactoryConfig.Template,
+			TemplatePath: runtimeConfig.FactoryConfig.TemplatePath,
+			Cache:        runtimeConfig.FactoryConfig.VMCacheNumber,
+			VMCache:      runtimeConfig.FactoryConfig.VMCacheNumber > 0,
+			VMConfig: vc.VMConfig{
+				HypervisorType:   runtimeConfig.HypervisorType,
+				HypervisorConfig: runtimeConfig.HypervisorConfig,
+				AgentType:        runtimeConfig.AgentType,
+				AgentConfig:      runtimeConfig.AgentConfig,
+				ProxyType:        runtimeConfig.ProxyType,
+				ProxyConfig:      runtimeConfig.ProxyConfig,
+			},
+		}
+
+		if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
+			f, err := vf.NewFactory(ctx, factoryConfig, false)
+			if err != nil {
+				return err
+			}
+			defer f.CloseFactory(ctx)
+
+			s := &cacheServer{
+				rpc:     grpc.NewServer(),
+				factory: f,
+			}
+			pb.RegisterCacheServiceServer(s.rpc, s)
+
+			l, err := getUnixListener(runtimeConfig.FactoryConfig.VMCacheEndpoint)
+			if err != nil {
+				return err
+			}
+			defer l.Close()
+
+			signals := make(chan os.Signal, 8)
+			handleSignals(s, signals)
+			signal.Notify(signals, handledSignals...)
+
+			kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server start")
+			s.rpc.Serve(l)
+
+			<-s.done
+
+			kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server stop")
+			return nil
+		}
+
+		if runtimeConfig.FactoryConfig.Template {
+			kataLog.WithField("factory", factoryConfig).Info("create vm factory")
+			_, err := vf.NewFactory(ctx, factoryConfig, false)
+			if err != nil {
+				kataLog.WithError(err).Error("create vm factory failed")
+				return err
+			}
+			fmt.Fprintln(defaultOutputFile, "vm factory initialized")
+		} else {
+			const errstring = "vm factory or VMCache is not enabled"
+			kataLog.Error(errstring)
+			fmt.Fprintln(defaultOutputFile, errstring)
+		}
+
+		return nil
+	},
+}
+
+var destroyFactoryCommand = cli.Command{
+	Name:  "destroy",
+	Usage: "destroy the VM factory",
+	Action: func(c *cli.Context) error {
+		ctx, err := cliContextToContext(c)
+		if err != nil {
+			return err
+		}
+
+		runtimeConfig, ok := c.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("invalid runtime config")
+		}
+
+		if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
+			conn, err := grpc.Dial(fmt.Sprintf("unix://%s", runtimeConfig.FactoryConfig.VMCacheEndpoint), grpc.WithInsecure())
+			if err != nil {
+				return errors.Wrapf(err, "failed to connect %q", runtimeConfig.FactoryConfig.VMCacheEndpoint)
+			}
+			defer conn.Close()
+			_, err = pb.NewCacheServiceClient(conn).Quit(ctx, &types.Empty{})
+			if err != nil {
+				return errors.Wrapf(err, "failed to call gRPC Quit")
+			}
+			// Wait VMCache server stop
+			time.Sleep(time.Second)
+		} else if runtimeConfig.FactoryConfig.Template {
+			factoryConfig := vf.Config{
+				Template:     true,
+				TemplatePath: runtimeConfig.FactoryConfig.TemplatePath,
+				VMConfig: vc.VMConfig{
+					HypervisorType:   runtimeConfig.HypervisorType,
+					HypervisorConfig: runtimeConfig.HypervisorConfig,
+					AgentType:        runtimeConfig.AgentType,
+					AgentConfig:      runtimeConfig.AgentConfig,
+					ProxyType:        runtimeConfig.ProxyType,
+					ProxyConfig:      runtimeConfig.ProxyConfig,
+				},
+			}
+			kataLog.WithField("factory", factoryConfig).Info("load vm factory")
+			f, err := vf.NewFactory(ctx, factoryConfig, true)
+			if err != nil {
+				kataLog.WithError(err).Error("load vm factory failed")
+				// ignore error
+			} else {
+				f.CloseFactory(ctx)
+			}
+		}
+		fmt.Fprintln(defaultOutputFile, "vm factory destroyed")
+		return nil
+	},
+}
+
+var statusFactoryCommand = cli.Command{
+	Name:  "status",
+	Usage: "query the status of VM factory",
+	Action: func(c *cli.Context) error {
+		ctx, err := cliContextToContext(c)
+		if err != nil {
+			return err
+		}
+
+		runtimeConfig, ok := c.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("invalid runtime config")
+		}
+
+		if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
+			conn, err := grpc.Dial(fmt.Sprintf("unix://%s", runtimeConfig.FactoryConfig.VMCacheEndpoint), grpc.WithInsecure())
+			if err != nil {
+				fmt.Fprintln(defaultOutputFile, errors.Wrapf(err, "failed to connect %q", runtimeConfig.FactoryConfig.VMCacheEndpoint))
+			} else {
+				defer conn.Close()
+				status, err := pb.NewCacheServiceClient(conn).Status(ctx, &types.Empty{})
+				if err != nil {
+					fmt.Fprintln(defaultOutputFile, errors.Wrapf(err, "failed to call gRPC Status\n"))
+				} else {
+					fmt.Fprintf(defaultOutputFile, "VM cache server pid = %d\n", status.Pid)
+					for _, vs := range status.Vmstatus {
+						fmt.Fprintf(defaultOutputFile, "VM pid = %d Cpu = %d Memory = %dMiB\n", vs.Pid, vs.Cpu, vs.Memory)
+					}
+				}
+			}
+		}
+		if runtimeConfig.FactoryConfig.Template {
+			factoryConfig := vf.Config{
+				Template:     true,
+				TemplatePath: runtimeConfig.FactoryConfig.TemplatePath,
+				VMConfig: vc.VMConfig{
+					HypervisorType:   runtimeConfig.HypervisorType,
+					HypervisorConfig: runtimeConfig.HypervisorConfig,
+					AgentType:        runtimeConfig.AgentType,
+					AgentConfig:      runtimeConfig.AgentConfig,
+					ProxyType:        runtimeConfig.ProxyType,
+					ProxyConfig:      runtimeConfig.ProxyConfig,
+				},
+			}
+			kataLog.WithField("factory", factoryConfig).Info("load vm factory")
+			_, err := vf.NewFactory(ctx, factoryConfig, true)
+			if err != nil {
+				fmt.Fprintln(defaultOutputFile, "vm factory is off")
+			} else {
+				fmt.Fprintln(defaultOutputFile, "vm factory is on")
+			}
+		} else {
+			fmt.Fprintln(defaultOutputFile, "vm factory not enabled")
+		}
+		return nil
+	},
+}

src/runtime/cli/factory_test.go

@@ -0,0 +1,152 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+
+	ktu "github.com/kata-containers/runtime/pkg/katatestutils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+)
+
+const testDisabledAsNonRoot = "Test disabled as requires root privileges"
+
+func TestFactoryCLIFunctionNoRuntimeConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+	ctx.App.Metadata["foo"] = "bar"
+
+	fn, ok := initFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err := fn(ctx)
+	// no runtime config in the Metadata
+	assert.Error(err)
+
+	fn, ok = destroyFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err = fn(ctx)
+	// no runtime config in the Metadata
+	assert.Error(err)
+}
+
+func TestFactoryCLIFunctionInit(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	set := flag.NewFlagSet("", 0)
+
+	set.String("console-socket", "", "")
+
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+
+	// No template
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	fn, ok := initFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err = fn(ctx)
+	assert.Nil(err)
+
+	// With template
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip(testDisabledAsNonRoot)
+	}
+
+	runtimeConfig.FactoryConfig.Template = true
+	runtimeConfig.FactoryConfig.TemplatePath = "/run/vc/vm/template"
+	runtimeConfig.HypervisorType = vc.MockHypervisor
+	runtimeConfig.AgentType = vc.NoopAgentType
+	runtimeConfig.ProxyType = vc.NoopProxyType
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+	fn, ok = initFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err = fn(ctx)
+	assert.Nil(err)
+}
+
+func TestFactoryCLIFunctionDestroy(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	set := flag.NewFlagSet("", 0)
+
+	set.String("console-socket", "", "")
+
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+
+	// No template
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+	fn, ok := destroyFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err = fn(ctx)
+	assert.Nil(err)
+
+	// With template
+	runtimeConfig.FactoryConfig.Template = true
+	runtimeConfig.HypervisorType = vc.MockHypervisor
+	runtimeConfig.AgentType = vc.NoopAgentType
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+	fn, ok = destroyFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err = fn(ctx)
+	assert.Nil(err)
+}
+
+func TestFactoryCLIFunctionStatus(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	set := flag.NewFlagSet("", 0)
+
+	set.String("console-socket", "", "")
+
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+
+	// No template
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	fn, ok := statusFactoryCommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+	err = fn(ctx)
+	assert.Nil(err)
+
+	// With template
+	runtimeConfig.FactoryConfig.Template = true
+	runtimeConfig.HypervisorType = vc.MockHypervisor
+	runtimeConfig.AgentType = vc.NoopAgentType
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+	err = fn(ctx)
+	assert.Nil(err)
+}

src/runtime/cli/kata-check.go

@@ -0,0 +1,503 @@
+// Copyright (c) 2017-2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Note: To add a new architecture, implement all identifiers beginning "arch".
+
+package main
+
+/*
+#include <linux/kvm.h>
+
+const int ioctl_KVM_CREATE_VM = KVM_CREATE_VM;
+const int ioctl_KVM_CHECK_EXTENSION = KVM_CHECK_EXTENSION;
+*/
+import "C"
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"syscall"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type kernelModule struct {
+	// description
+	desc string
+
+	// maps parameter names to values
+	parameters map[string]string
+
+	// if it is definitely required
+	required bool
+}
+
+// nolint: structcheck, unused, deadcode
+type kvmExtension struct {
+	// description
+	desc string
+
+	// extension identifier
+	id int
+}
+
+type vmContainerCapableDetails struct {
+	cpuInfoFile           string
+	requiredCPUFlags      map[string]string
+	requiredCPUAttribs    map[string]string
+	requiredKernelModules map[string]kernelModule
+}
+
+const (
+	moduleParamDir        = "parameters"
+	successMessageCapable = "System is capable of running " + project
+	successMessageCreate  = "System can currently create " + project
+	successMessageVersion = "Version consistency of " + project + " is verified"
+	failMessage           = "System is not capable of running " + project
+	kernelPropertyCorrect = "Kernel property value correct"
+
+	// these refer to fields in the procCPUINFO file
+	genericCPUFlagsTag    = "flags"      // nolint: varcheck, unused, deadcode
+	genericCPUVendorField = "vendor_id"  // nolint: varcheck, unused, deadcode
+	genericCPUModelField  = "model name" // nolint: varcheck, unused, deadcode
+)
+
+// variables rather than consts to allow tests to modify them
+var (
+	procCPUInfo  = "/proc/cpuinfo"
+	sysModuleDir = "/sys/module"
+	modProbeCmd  = "modprobe"
+)
+
+// variables rather than consts to allow tests to modify them
+var (
+	kvmDevice = "/dev/kvm"
+)
+
+// getCPUInfo returns details of the first CPU read from the specified cpuinfo file
+func getCPUInfo(cpuInfoFile string) (string, error) {
+	text, err := katautils.GetFileContents(cpuInfoFile)
+	if err != nil {
+		return "", err
+	}
+
+	cpus := strings.SplitAfter(text, "\n\n")
+
+	trimmed := strings.TrimSpace(cpus[0])
+	if trimmed == "" {
+		return "", fmt.Errorf("Cannot determine CPU details")
+	}
+
+	return trimmed, nil
+}
+
+// findAnchoredString searches haystack for needle and returns true if found
+func findAnchoredString(haystack, needle string) bool {
+	if haystack == "" || needle == "" {
+		return false
+	}
+
+	// Ensure the search string is anchored
+	pattern := regexp.MustCompile(`\b` + needle + `\b`)
+
+	return pattern.MatchString(haystack)
+}
+
+// getCPUFlags returns the CPU flags from the cpuinfo file specified
+func getCPUFlags(cpuinfo string) string {
+	for _, line := range strings.Split(cpuinfo, "\n") {
+		if strings.HasPrefix(line, cpuFlagsTag) {
+			fields := strings.Split(line, ":")
+			if len(fields) == 2 {
+				return strings.TrimSpace(fields[1])
+			}
+		}
+	}
+
+	return ""
+}
+
+// haveKernelModule returns true if the specified module exists
+// (either loaded or available to be loaded)
+func haveKernelModule(module string) bool {
+	// First, check to see if the module is already loaded
+	path := filepath.Join(sysModuleDir, module)
+	if katautils.FileExists(path) {
+		return true
+	}
+
+	// Now, check if the module is unloaded, but available.
+	// And modprobe it if so.
+	cmd := exec.Command(modProbeCmd, module)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		kataLog.WithField("module", module).WithError(err).Warnf("modprobe insert module failed: %s", string(output))
+		return false
+	}
+	return true
+}
+
+// checkCPU checks all required CPU attributes modules and returns a count of
+// the number of CPU attribute errors (all of which are logged by this
+// function). The specified tag is simply used for logging purposes.
+func checkCPU(tag, cpuinfo string, attribs map[string]string) (count uint32) {
+	if cpuinfo == "" {
+		return 0
+	}
+
+	for attrib, desc := range attribs {
+		fields := logrus.Fields{
+			"type":        tag,
+			"name":        attrib,
+			"description": desc,
+		}
+
+		found := findAnchoredString(cpuinfo, attrib)
+		if !found {
+			kataLog.WithFields(fields).Errorf("CPU property not found")
+			count++
+			continue
+
+		}
+
+		kataLog.WithFields(fields).Infof("CPU property found")
+	}
+
+	return count
+}
+
+func checkCPUFlags(cpuflags string, required map[string]string) uint32 {
+	return checkCPU("flag", cpuflags, required)
+}
+
+func checkCPUAttribs(cpuinfo string, attribs map[string]string) uint32 {
+	return checkCPU("attribute", cpuinfo, attribs)
+}
+
+// kernelParamHandler represents a function that allows kernel module
+// parameter errors to be ignored for special scenarios.
+//
+// The function is passed the following parameters:
+//
+// onVMM  - `true` if the host is running under a VMM environment
+// fields - A set of fields showing the expected and actual module parameter values.
+// msg    - The message that would be logged showing the incorrect kernel module
+//          parameter.
+//
+// The function must return `true` if the kernel module parameter error should
+// be ignored, or `false` if it is a real error.
+//
+// Note: it is up to the function to add an appropriate log call if the error
+// should be ignored.
+type kernelParamHandler func(onVMM bool, fields logrus.Fields, msg string) bool
+
+// checkKernelModules checks all required kernel modules modules and returns a count of
+// the number of module errors (all of which are logged by this
+// function). Only fatal errors result in an error return.
+func checkKernelModules(modules map[string]kernelModule, handler kernelParamHandler) (count uint32, err error) {
+	onVMM, err := vc.RunningOnVMM(procCPUInfo)
+	if err != nil {
+		return 0, err
+	}
+
+	for module, details := range modules {
+		fields := logrus.Fields{
+			"type":        "module",
+			"name":        module,
+			"description": details.desc,
+		}
+
+		if !haveKernelModule(module) {
+			kataLog.WithFields(fields).Error("kernel property not found")
+			if details.required {
+				count++
+			}
+			continue
+		}
+
+		kataLog.WithFields(fields).Infof("kernel property found")
+
+		for param, expected := range details.parameters {
+			path := filepath.Join(sysModuleDir, module, moduleParamDir, param)
+			value, err := katautils.GetFileContents(path)
+			if err != nil {
+				return 0, err
+			}
+
+			value = strings.TrimRight(value, "\n\r")
+
+			fields["parameter"] = param
+			fields["value"] = value
+
+			if value != expected {
+				fields["expected"] = expected
+
+				msg := "kernel module parameter has unexpected value"
+
+				if handler != nil {
+					ignoreError := handler(onVMM, fields, msg)
+					if ignoreError {
+						continue
+					}
+				}
+
+				kataLog.WithFields(fields).Error(msg)
+				count++
+			}
+
+			kataLog.WithFields(fields).Info(kernelPropertyCorrect)
+		}
+	}
+
+	return count, nil
+}
+
+// genericHostIsVMContainerCapable checks to see if the host is theoretically capable
+// of creating a VM container.
+//nolint: unused,deadcode
+func genericHostIsVMContainerCapable(details vmContainerCapableDetails) error {
+	cpuinfo, err := getCPUInfo(details.cpuInfoFile)
+	if err != nil {
+		return err
+	}
+
+	cpuFlags := getCPUFlags(cpuinfo)
+	if cpuFlags == "" {
+		return fmt.Errorf("Cannot find CPU flags")
+	}
+
+	// Keep a track of the error count, but don't error until all tests
+	// have been performed!
+	errorCount := uint32(0)
+
+	count := checkCPUAttribs(cpuinfo, details.requiredCPUAttribs)
+
+	errorCount += count
+
+	count = checkCPUFlags(cpuFlags, details.requiredCPUFlags)
+
+	errorCount += count
+
+	count, err = checkKernelModules(details.requiredKernelModules, archKernelParamHandler)
+	if err != nil {
+		return err
+	}
+
+	errorCount += count
+
+	if errorCount == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("ERROR: %s", failMessage)
+}
+
+var kataCheckCLICommand = cli.Command{
+	Name:  checkCmd,
+	Usage: "tests if system can run " + project,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "verbose, v",
+			Usage: "display the list of checks performed",
+		},
+		cli.BoolFlag{
+			Name:  "strict, s",
+			Usage: "perform strict checking",
+		},
+	},
+
+	Action: func(context *cli.Context) error {
+		verbose := context.Bool("verbose")
+		if verbose {
+			kataLog.Logger.SetLevel(logrus.InfoLevel)
+		}
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, _ := katautils.Trace(ctx, "kata-check")
+		defer span.Finish()
+
+		runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("kata-check: cannot determine runtime config")
+		}
+
+		err = setCPUtype(runtimeConfig.HypervisorType)
+		if err != nil {
+			return err
+		}
+
+		details := vmContainerCapableDetails{
+			cpuInfoFile:           procCPUInfo,
+			requiredCPUFlags:      archRequiredCPUFlags,
+			requiredCPUAttribs:    archRequiredCPUAttribs,
+			requiredKernelModules: archRequiredKernelModules,
+		}
+
+		err = hostIsVMContainerCapable(details)
+
+		if err != nil {
+			return err
+		}
+		fmt.Println(successMessageCapable)
+
+		if os.Geteuid() == 0 {
+			err = archHostCanCreateVMContainer(runtimeConfig.HypervisorType)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(successMessageCreate)
+		}
+
+		strict := context.Bool("strict")
+		if strict {
+			err = checkVersionConsistencyInComponents(runtimeConfig)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(successMessageVersion)
+		}
+
+		return nil
+	},
+}
+
+func genericArchKernelParamHandler(onVMM bool, fields logrus.Fields, msg string) bool {
+	param, ok := fields["parameter"].(string)
+	if !ok {
+		return false
+	}
+
+	// This option is not required when
+	// already running under a hypervisor.
+	if param == "unrestricted_guest" && onVMM {
+		kataLog.WithFields(fields).Warn(kernelPropertyCorrect)
+		return true
+	}
+
+	if param == "nested" {
+		kataLog.WithFields(fields).Warn(msg)
+		return true
+	}
+
+	// don't ignore the error
+	return false
+}
+
+// genericKvmIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func genericKvmIsUsable() error {
+	flags := syscall.O_RDWR | syscall.O_CLOEXEC
+
+	f, err := syscall.Open(kvmDevice, flags, 0)
+	if err != nil {
+		return err
+	}
+	defer syscall.Close(f)
+
+	fieldLogger := kataLog.WithField("check-type", "full")
+
+	fieldLogger.WithField("device", kvmDevice).Info("device available")
+
+	vm, _, errno := syscall.Syscall(syscall.SYS_IOCTL,
+		uintptr(f),
+		uintptr(C.ioctl_KVM_CREATE_VM),
+		0)
+	if errno != 0 {
+		if errno == syscall.EBUSY {
+			fieldLogger.WithField("reason", "another hypervisor running").Error("cannot create VM")
+		}
+
+		return errno
+	}
+	defer syscall.Close(int(vm))
+
+	fieldLogger.WithField("feature", "create-vm").Info("feature available")
+
+	return nil
+}
+
+// genericCheckKVMExtension allows to query about the specific kvm extensions
+// nolint: unused, deadcode
+func genericCheckKVMExtensions(extensions map[string]kvmExtension) (map[string]int, error) {
+	results := make(map[string]int)
+
+	flags := syscall.O_RDWR | syscall.O_CLOEXEC
+	kvm, err := syscall.Open(kvmDevice, flags, 0)
+	if err != nil {
+		return results, err
+	}
+	defer syscall.Close(kvm)
+
+	for name, extension := range extensions {
+		fields := logrus.Fields{
+			"type":        "kvm extension",
+			"name":        name,
+			"description": extension.desc,
+			"id":          extension.id,
+		}
+
+		ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL,
+			uintptr(kvm),
+			uintptr(C.ioctl_KVM_CHECK_EXTENSION),
+			uintptr(extension.id))
+
+		// Generally return value(ret) 0 means no and 1 means yes,
+		// but some extensions may report additional information in the integer return value.
+		if errno != 0 {
+			kataLog.WithFields(fields).Error("is not supported")
+			return results, errno
+		}
+
+		results[name] = int(ret)
+		kataLog.WithFields(fields).Info("kvm extension is supported")
+	}
+
+	return results, nil
+}
+
+// checkVersionConsistencyInComponents checks version consistency in Kata Components.
+func checkVersionConsistencyInComponents(config oci.RuntimeConfig) error {
+	proxyInfo := getProxyInfo(config)
+
+	shimInfo, err := getShimInfo(config)
+	if err != nil {
+		return err
+	}
+	shimVersionInfo := shimInfo.Version
+
+	runtimeVersionInfo := constructVersionInfo(version)
+
+	// kata-proxy exists
+	if proxyInfo.Type != string(vc.NoProxyType) {
+		proxyVersionInfo := proxyInfo.Version
+		if !versionEqual(proxyVersionInfo, runtimeVersionInfo) || !versionEqual(shimVersionInfo, runtimeVersionInfo) {
+			return fmt.Errorf("there exists version inconsistency in kata components. kata-proxy: v%d.%d.%d, kata-shim: v%d.%d.%d, kata-runtime: v%d.%d.%d",
+				proxyVersionInfo.Major, proxyVersionInfo.Minor, proxyVersionInfo.Patch,
+				shimVersionInfo.Major, shimVersionInfo.Minor, shimVersionInfo.Patch,
+				runtimeVersionInfo.Major, runtimeVersionInfo.Minor, runtimeVersionInfo.Patch)
+		}
+	} else {
+		if !versionEqual(shimVersionInfo, runtimeVersionInfo) {
+			return fmt.Errorf("there exists version inconsistency in kata components. kata-shim: v%d.%d.%d, kata-runtime: v%d.%d.%d",
+				shimVersionInfo.Major, shimVersionInfo.Minor, shimVersionInfo.Patch,
+				runtimeVersionInfo.Major, runtimeVersionInfo.Minor, runtimeVersionInfo.Patch)
+		}
+	}
+
+	return nil
+}

src/runtime/cli/kata-check_amd64.go

@@ -0,0 +1,312 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"syscall"
+	"unsafe"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	cpuFlagsTag               = genericCPUFlagsTag
+	archCPUVendorField        = genericCPUVendorField
+	archCPUModelField         = genericCPUModelField
+	archGenuineIntel          = "GenuineIntel"
+	archAuthenticAMD          = "AuthenticAMD"
+	msgKernelVM               = "Kernel-based Virtual Machine"
+	msgKernelVirtio           = "Host kernel accelerator for virtio"
+	msgKernelVirtioNet        = "Host kernel accelerator for virtio network"
+	msgKernelVirtioVhostVsock = "Host Support for Linux VM Sockets"
+	cpuFlagVMX                = "vmx"
+	cpuFlagLM                 = "lm"
+	cpuFlagSVM                = "svm"
+	cpuFlagSSE4_1             = "sse4_1"
+	kernelModvhm              = "vhm_dev"
+	kernelModvhost            = "vhost"
+	kernelModvhostnet         = "vhost_net"
+	kernelModvhostvsock       = "vhost_vsock"
+	kernelModkvm              = "kvm"
+	kernelModkvmintel         = "kvm_intel"
+	kernelModkvmamd           = "kvm_amd"
+)
+
+// CPU types
+const (
+	cpuTypeIntel   = 0
+	cpuTypeAMD     = 1
+	cpuTypeUnknown = -1
+)
+
+const acrnDevice = "/dev/acrn_vhm"
+
+// ioctl_ACRN_CREATE_VM is the IOCTL to create VM in ACRN.
+// Current Linux mainstream kernel doesn't have support for ACRN.
+// Due to this several macros are not defined in Linux headers.
+// Until the support is available, directly use the value instead
+// of macros.
+//https://github.com/kata-containers/runtime/issues/1784
+const ioctl_ACRN_CREATE_VM = 0x43000010  //nolint
+const ioctl_ACRN_DESTROY_VM = 0x43000011 //nolint
+
+type acrn_create_vm struct { //nolint
+	vmid      uint16 //nolint
+	reserved0 uint16 //nolint
+	vcpu_num  uint16 //nolint
+	reserved1 uint16 //nolint
+	uuid      [16]uint8
+	vm_flag   uint64    //nolint
+	req_buf   uint64    //nolint
+	reserved2 [16]uint8 //nolint
+}
+
+// cpuType save the CPU type
+var cpuType int
+
+// archRequiredCPUFlags maps a CPU flag value to search for and a
+// human-readable description of that value.
+var archRequiredCPUFlags map[string]string
+
+// archRequiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
+// and a human-readable description of that value.
+var archRequiredCPUAttribs map[string]string
+
+// archRequiredKernelModules maps a required module name to a human-readable
+// description of the modules functionality and an optional list of
+// required module parameters.
+var archRequiredKernelModules map[string]kernelModule
+
+func setCPUtype(hypervisorType vc.HypervisorType) error {
+	cpuType = getCPUtype()
+
+	if cpuType == cpuTypeUnknown {
+		return fmt.Errorf("Unknow CPU Type")
+	} else if cpuType == cpuTypeIntel {
+		var kvmIntelParams map[string]string
+		onVMM, err := vc.RunningOnVMM(procCPUInfo)
+		if err != nil && !onVMM {
+			kvmIntelParams = map[string]string{
+				// "VMX Unrestricted mode support". This is used
+				// as a heuristic to determine if the system is
+				// "new enough" to run a Kata Container
+				// (atleast a Westmere).
+				"unrestricted_guest": "Y",
+			}
+		}
+
+		switch hypervisorType {
+		case "firecracker":
+			fallthrough
+		case "clh":
+			fallthrough
+		case "qemu":
+			archRequiredCPUFlags = map[string]string{
+				cpuFlagVMX:    "Virtualization support",
+				cpuFlagLM:     "64Bit CPU",
+				cpuFlagSSE4_1: "SSE4.1",
+			}
+			archRequiredCPUAttribs = map[string]string{
+				archGenuineIntel: "Intel Architecture CPU",
+			}
+			archRequiredKernelModules = map[string]kernelModule{
+				kernelModkvm: {
+					desc:     msgKernelVM,
+					required: true,
+				},
+				kernelModkvmintel: {
+					desc:       "Intel KVM",
+					parameters: kvmIntelParams,
+					required:   true,
+				},
+				kernelModvhost: {
+					desc:     msgKernelVirtio,
+					required: true,
+				},
+				kernelModvhostnet: {
+					desc:     msgKernelVirtioNet,
+					required: true,
+				},
+				kernelModvhostvsock: {
+					desc:     msgKernelVirtioVhostVsock,
+					required: false,
+				},
+			}
+		case "acrn":
+			archRequiredCPUFlags = map[string]string{
+				cpuFlagLM:     "64Bit CPU",
+				cpuFlagSSE4_1: "SSE4.1",
+			}
+			archRequiredCPUAttribs = map[string]string{
+				archGenuineIntel: "Intel Architecture CPU",
+			}
+			archRequiredKernelModules = map[string]kernelModule{
+				kernelModvhm: {
+					desc:     "Intel ACRN",
+					required: false,
+				},
+				kernelModvhost: {
+					desc:     msgKernelVirtio,
+					required: false,
+				},
+				kernelModvhostnet: {
+					desc:     msgKernelVirtioNet,
+					required: false,
+				},
+			}
+		default:
+			return fmt.Errorf("setCPUtype: Unknown hypervisor type %s", hypervisorType)
+		}
+
+	} else if cpuType == cpuTypeAMD {
+		archRequiredCPUFlags = map[string]string{
+			cpuFlagSVM:    "Virtualization support",
+			cpuFlagLM:     "64Bit CPU",
+			cpuFlagSSE4_1: "SSE4.1",
+		}
+		archRequiredCPUAttribs = map[string]string{
+			archAuthenticAMD: "AMD Architecture CPU",
+		}
+		archRequiredKernelModules = map[string]kernelModule{
+			kernelModkvm: {
+				desc:     msgKernelVM,
+				required: true,
+			},
+			kernelModkvmamd: {
+				desc:     "AMD KVM",
+				required: true,
+			},
+			kernelModvhost: {
+				desc:     msgKernelVirtio,
+				required: true,
+			},
+			kernelModvhostnet: {
+				desc:     msgKernelVirtioNet,
+				required: true,
+			},
+			kernelModvhostvsock: {
+				desc:     msgKernelVirtioVhostVsock,
+				required: false,
+			},
+		}
+	}
+
+	return nil
+}
+
+func getCPUtype() int {
+	content, err := ioutil.ReadFile("/proc/cpuinfo")
+	if err != nil {
+		kataLog.WithError(err).Error("failed to read file")
+		return cpuTypeUnknown
+	}
+	str := string(content)
+	if strings.Contains(str, archGenuineIntel) {
+		return cpuTypeIntel
+	} else if strings.Contains(str, archAuthenticAMD) {
+		return cpuTypeAMD
+	} else {
+		return cpuTypeUnknown
+	}
+}
+
+// kvmIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func kvmIsUsable() error {
+	return genericKvmIsUsable()
+}
+
+// acrnIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func acrnIsUsable() error {
+	flags := syscall.O_RDWR | syscall.O_CLOEXEC
+
+	f, err := syscall.Open(acrnDevice, flags, 0)
+	if err != nil {
+		return err
+	}
+	defer syscall.Close(f)
+	kataLog.WithField("device", acrnDevice).Info("device available")
+
+	acrnInst := vc.Acrn{}
+	uuidStr, err := acrnInst.GetNextAvailableUUID()
+	if err != nil {
+		return err
+	}
+
+	uuid, err := acrnInst.GetACRNUUIDBytes(uuidStr)
+	if err != nil {
+		return fmt.Errorf("Converting UUID str to bytes failed, Err:%s", err)
+	}
+
+	var createVM acrn_create_vm
+	createVM.uuid = uuid
+
+	ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL,
+		uintptr(f),
+		uintptr(ioctl_ACRN_CREATE_VM),
+		uintptr(unsafe.Pointer(&createVM)))
+	if ret != 0 || errno != 0 {
+		if errno == syscall.EBUSY {
+			kataLog.WithField("reason", "another hypervisor running").Error("cannot create VM")
+		}
+		kataLog.WithFields(logrus.Fields{
+			"ret":   ret,
+			"errno": errno,
+		}).Info("Create VM Error")
+		return errno
+	}
+
+	ret, _, errno = syscall.Syscall(syscall.SYS_IOCTL,
+		uintptr(f),
+		uintptr(ioctl_ACRN_DESTROY_VM),
+		0)
+	if ret != 0 || errno != 0 {
+		kataLog.WithFields(logrus.Fields{
+			"ret":   ret,
+			"errno": errno,
+		}).Info("Destroy VM Error")
+		return errno
+	}
+
+	kataLog.WithField("feature", "create-vm").Info("feature available")
+
+	return nil
+}
+
+func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+
+	switch hypervisorType {
+	case "qemu":
+		fallthrough
+	case "clh":
+		fallthrough
+	case "firecracker":
+		return kvmIsUsable()
+	case "acrn":
+		return acrnIsUsable()
+	default:
+		return fmt.Errorf("archHostCanCreateVMContainer: Unknown hypervisor type %s", hypervisorType)
+	}
+}
+
+// hostIsVMContainerCapable checks to see if the host is theoretically capable
+// of creating a VM container.
+func hostIsVMContainerCapable(details vmContainerCapableDetails) error {
+	return genericHostIsVMContainerCapable(details)
+}
+
+func archKernelParamHandler(onVMM bool, fields logrus.Fields, msg string) bool {
+	return genericArchKernelParamHandler(onVMM, fields, msg)
+}
+
+func getCPUDetails() (vendor, model string, err error) {
+	return genericGetCPUDetails()
+}

src/runtime/cli/kata-check_amd64_test.go

@@ -0,0 +1,476 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func setupCheckHostIsVMContainerCapable(assert *assert.Assertions, cpuInfoFile string, cpuData []testCPUData, moduleData []testModuleData) {
+	createModules(assert, cpuInfoFile, moduleData)
+
+	// all the modules files have now been created, so deal with the
+	// cpuinfo data.
+	for _, d := range cpuData {
+		err := makeCPUInfoFile(cpuInfoFile, d.vendorID, d.flags)
+		assert.NoError(err)
+
+		details := vmContainerCapableDetails{
+			cpuInfoFile:           cpuInfoFile,
+			requiredCPUFlags:      archRequiredCPUFlags,
+			requiredCPUAttribs:    archRequiredCPUAttribs,
+			requiredKernelModules: archRequiredKernelModules,
+		}
+
+		err = hostIsVMContainerCapable(details)
+		if d.expectError {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+		}
+	}
+}
+
+func TestCCCheckCLIFunction(t *testing.T) {
+	var cpuData []testCPUData
+	var moduleData []testModuleData
+
+	if cpuType == cpuTypeIntel {
+		cpuData = []testCPUData{
+			{archGenuineIntel, "lm vmx sse4_1", false},
+		}
+
+		moduleData = []testModuleData{}
+	} else if cpuType == cpuTypeAMD {
+		cpuData = []testCPUData{
+			{archAuthenticAMD, "lm svm sse4_1", false},
+		}
+
+		moduleData = []testModuleData{}
+	}
+
+	genericCheckCLIFunction(t, cpuData, moduleData)
+}
+
+func TestCheckCheckKernelModulesNoNesting(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedSysModuleDir := sysModuleDir
+	savedProcCPUInfo := procCPUInfo
+
+	cpuInfoFile := filepath.Join(dir, "cpuinfo")
+
+	// XXX: override
+	sysModuleDir = filepath.Join(dir, "sys/module")
+	procCPUInfo = cpuInfoFile
+
+	defer func() {
+		sysModuleDir = savedSysModuleDir
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requiredModules := map[string]kernelModule{
+		"kvm_intel": {
+			desc: "Intel KVM",
+			parameters: map[string]string{
+				"nested":             "Y",
+				"unrestricted_guest": "Y",
+			},
+			required: true,
+		},
+	}
+
+	actualModuleData := []testModuleData{
+		{filepath.Join(sysModuleDir, "kvm"), true, ""},
+		{filepath.Join(sysModuleDir, "kvm_intel"), true, ""},
+		{filepath.Join(sysModuleDir, "kvm_intel/parameters/unrestricted_guest"), false, "Y"},
+
+		// XXX: force a warning
+		{filepath.Join(sysModuleDir, "kvm_intel/parameters/nested"), false, "N"},
+	}
+
+	vendor := archGenuineIntel
+	flags := "vmx lm sse4_1 hypervisor"
+
+	_, err = checkKernelModules(requiredModules, archKernelParamHandler)
+	// no cpuInfoFile yet
+	assert.Error(err)
+
+	createModules(assert, cpuInfoFile, actualModuleData)
+
+	err = makeCPUInfoFile(cpuInfoFile, vendor, flags)
+	assert.NoError(err)
+
+	count, err := checkKernelModules(requiredModules, archKernelParamHandler)
+	assert.NoError(err)
+	assert.Equal(count, uint32(0))
+
+	// create buffer to save logger output
+	buf := &bytes.Buffer{}
+
+	savedLogOutput := kataLog.Logger.Out
+
+	defer func() {
+		kataLog.Logger.Out = savedLogOutput
+	}()
+
+	kataLog.Logger.Out = buf
+
+	count, err = checkKernelModules(requiredModules, archKernelParamHandler)
+
+	assert.NoError(err)
+	assert.Equal(count, uint32(0))
+
+	re := regexp.MustCompile(`.*\bnested\b`)
+	matches := re.FindAllStringSubmatch(buf.String(), -1)
+	assert.NotEmpty(matches)
+}
+
+func TestCheckCheckKernelModulesNoUnrestrictedGuest(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedSysModuleDir := sysModuleDir
+	savedProcCPUInfo := procCPUInfo
+
+	cpuInfoFile := filepath.Join(dir, "cpuinfo")
+
+	// XXX: override
+	sysModuleDir = filepath.Join(dir, "sys/module")
+	procCPUInfo = cpuInfoFile
+
+	defer func() {
+		sysModuleDir = savedSysModuleDir
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requiredModules := map[string]kernelModule{
+		"kvm_intel": {
+			desc: "Intel KVM",
+			parameters: map[string]string{
+				"nested":             "Y",
+				"unrestricted_guest": "Y",
+			},
+			required: true,
+		},
+	}
+
+	actualModuleData := []testModuleData{
+		{filepath.Join(sysModuleDir, "kvm"), true, ""},
+		{filepath.Join(sysModuleDir, "kvm_intel"), true, ""},
+		{filepath.Join(sysModuleDir, "kvm_intel/parameters/nested"), false, "Y"},
+
+		// XXX: force a failure on non-VMM systems
+		{filepath.Join(sysModuleDir, "kvm_intel/parameters/unrestricted_guest"), false, "N"},
+	}
+
+	vendor := archGenuineIntel
+	flags := "vmx lm sse4_1"
+
+	_, err = checkKernelModules(requiredModules, archKernelParamHandler)
+	// no cpuInfoFile yet
+	assert.Error(err)
+
+	err = makeCPUInfoFile(cpuInfoFile, vendor, flags)
+	assert.NoError(err)
+
+	createModules(assert, cpuInfoFile, actualModuleData)
+
+	count, err := checkKernelModules(requiredModules, archKernelParamHandler)
+
+	assert.NoError(err)
+	// fails due to unrestricted_guest not being available
+	assert.Equal(count, uint32(1))
+
+	// pretend test is running under a hypervisor
+	flags += " hypervisor"
+
+	// recreate
+	err = makeCPUInfoFile(cpuInfoFile, vendor, flags)
+	assert.NoError(err)
+
+	// create buffer to save logger output
+	buf := &bytes.Buffer{}
+
+	savedLogOutput := kataLog.Logger.Out
+
+	defer func() {
+		kataLog.Logger.Out = savedLogOutput
+	}()
+
+	kataLog.Logger.Out = buf
+
+	count, err = checkKernelModules(requiredModules, archKernelParamHandler)
+
+	// no error now because running under a hypervisor
+	assert.NoError(err)
+	assert.Equal(count, uint32(0))
+
+	re := regexp.MustCompile(`.*\bunrestricted_guest\b`)
+	matches := re.FindAllStringSubmatch(buf.String(), -1)
+	assert.NotEmpty(matches)
+}
+
+func TestCheckHostIsVMContainerCapable(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedSysModuleDir := sysModuleDir
+	savedProcCPUInfo := procCPUInfo
+
+	cpuInfoFile := filepath.Join(dir, "cpuinfo")
+
+	// XXX: override
+	sysModuleDir = filepath.Join(dir, "sys/module")
+	procCPUInfo = cpuInfoFile
+
+	defer func() {
+		sysModuleDir = savedSysModuleDir
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var cpuData []testCPUData
+	var moduleData []testModuleData
+
+	if cpuType == cpuTypeIntel {
+		cpuData = []testCPUData{
+			{"", "", true},
+			{"Intel", "", true},
+			{archGenuineIntel, "", true},
+			{archGenuineIntel, "lm", true},
+			{archGenuineIntel, "lm vmx", true},
+			{archGenuineIntel, "lm vmx sse4_1", false},
+		}
+
+		moduleData = []testModuleData{
+			{filepath.Join(sysModuleDir, "kvm"), true, ""},
+			{filepath.Join(sysModuleDir, "kvm_intel"), true, ""},
+			{filepath.Join(sysModuleDir, "kvm_intel/parameters/nested"), false, "Y"},
+			{filepath.Join(sysModuleDir, "kvm_intel/parameters/unrestricted_guest"), false, "Y"},
+		}
+	} else if cpuType == cpuTypeAMD {
+		cpuData = []testCPUData{
+			{"", "", true},
+			{"AMD", "", true},
+			{archAuthenticAMD, "", true},
+			{archAuthenticAMD, "lm", true},
+			{archAuthenticAMD, "lm svm", true},
+			{archAuthenticAMD, "lm svm sse4_1", false},
+		}
+
+		moduleData = []testModuleData{
+			{filepath.Join(sysModuleDir, "kvm"), true, ""},
+			{filepath.Join(sysModuleDir, "kvm_amd"), true, ""},
+			{filepath.Join(sysModuleDir, "kvm_amd/parameters/nested"), false, "1"},
+		}
+	}
+
+	setupCheckHostIsVMContainerCapable(assert, cpuInfoFile, cpuData, moduleData)
+
+	// remove the modules to force a failure
+	err = os.RemoveAll(sysModuleDir)
+	assert.NoError(err)
+
+	details := vmContainerCapableDetails{
+		cpuInfoFile:           cpuInfoFile,
+		requiredCPUFlags:      archRequiredCPUFlags,
+		requiredCPUAttribs:    archRequiredCPUAttribs,
+		requiredKernelModules: archRequiredKernelModules,
+	}
+
+	err = hostIsVMContainerCapable(details)
+	assert.Nil(err)
+}
+
+func TestArchKernelParamHandler(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		onVMM        bool
+		expectIgnore bool
+		fields       logrus.Fields
+		msg          string
+	}
+
+	data := []testData{
+		{true, false, logrus.Fields{}, ""},
+		{false, false, logrus.Fields{}, ""},
+
+		{
+			false,
+			false,
+			logrus.Fields{
+				// wrong type
+				"parameter": 123,
+			},
+			"foo",
+		},
+
+		{
+			false,
+			false,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+		},
+
+		{
+			true,
+			true,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+		},
+
+		{
+			false,
+			true,
+			logrus.Fields{
+				"parameter": "nested",
+			},
+			"",
+		},
+	}
+
+	for i, d := range data {
+		result := archKernelParamHandler(d.onVMM, d.fields, d.msg)
+		if d.expectIgnore {
+			assert.True(result, "test %d (%+v)", i, d)
+		} else {
+			assert.False(result, "test %d (%+v)", i, d)
+		}
+	}
+}
+
+func TestKvmIsUsable(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedKvmDevice := kvmDevice
+	fakeKVMDevice := filepath.Join(dir, "kvm")
+	kvmDevice = fakeKVMDevice
+
+	defer func() {
+		kvmDevice = savedKvmDevice
+	}()
+
+	err = kvmIsUsable()
+	assert.Error(err)
+
+	err = createEmptyFile(fakeKVMDevice)
+	assert.NoError(err)
+
+	err = kvmIsUsable()
+	assert.Error(err)
+}
+
+func TestGetCPUDetails(t *testing.T) {
+	const validVendorName = "a vendor"
+	validVendor := fmt.Sprintf(`%s  : %s`, archCPUVendorField, validVendorName)
+
+	const validModelName = "some CPU model"
+	validModel := fmt.Sprintf(`%s   : %s`, archCPUModelField, validModelName)
+
+	validContents := fmt.Sprintf(`
+a       : b
+%s
+foo     : bar
+%s
+`, validVendor, validModel)
+
+	data := []testCPUDetail{
+		{"", "", "", true},
+		{"invalid", "", "", true},
+		{archCPUVendorField, "", "", true},
+		{validVendor, "", "", true},
+		{validModel, "", "", true},
+		{validContents, validVendorName, validModelName, false},
+	}
+	genericTestGetCPUDetails(t, validVendor, validModel, validContents, data)
+}
+
+func TestSetCPUtype(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	savedArchRequiredCPUFlags := archRequiredCPUFlags
+	savedArchRequiredCPUAttribs := archRequiredCPUAttribs
+	savedArchRequiredKernelModules := archRequiredKernelModules
+
+	defer func() {
+		archRequiredCPUFlags = savedArchRequiredCPUFlags
+		archRequiredCPUAttribs = savedArchRequiredCPUAttribs
+		archRequiredKernelModules = savedArchRequiredKernelModules
+	}()
+
+	archRequiredCPUFlags = map[string]string{}
+	archRequiredCPUAttribs = map[string]string{}
+	archRequiredKernelModules = map[string]kernelModule{}
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	setCPUtype(config.HypervisorType)
+
+	assert.NotEmpty(archRequiredCPUFlags)
+	assert.NotEmpty(archRequiredCPUAttribs)
+	assert.NotEmpty(archRequiredKernelModules)
+
+	assert.Equal(archRequiredCPUFlags["vmx"], "Virtualization support")
+
+	_, ok := archRequiredKernelModules["kvm"]
+	assert.True(ok)
+}

src/runtime/cli/kata-check_arm64.go

@@ -0,0 +1,171 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	cpuFlagsTag        = "Features"
+	archCPUVendorField = "CPU implementer"
+	archCPUModelField  = "CPU architecture"
+)
+
+// archRequiredCPUFlags maps a CPU flag value to search for and a
+// human-readable description of that value.
+var archRequiredCPUFlags = map[string]string{}
+
+// archRequiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
+// and a human-readable description of that value.
+var archRequiredCPUAttribs = map[string]string{}
+
+// archRequiredKernelModules maps a required module name to a human-readable
+// description of the modules functionality and an optional list of
+// required module parameters.
+var archRequiredKernelModules = map[string]kernelModule{
+	"kvm": {
+		desc:     "Kernel-based Virtual Machine",
+		required: true,
+	},
+	"vhost": {
+		desc:     "Host kernel accelerator for virtio",
+		required: true,
+	},
+	"vhost_net": {
+		desc:     "Host kernel accelerator for virtio network",
+		required: true,
+	},
+	"vhost_vsock": {
+		desc:     "Host Support for Linux VM Sockets",
+		required: false,
+	},
+}
+
+// archRequiredKVMExtensions maps a required kvm extension to a human-readable
+// description of what this extension intends to do and its unique identifier.
+var archRequiredKVMExtensions = map[string]kvmExtension{
+	"KVM_CAP_ARM_VM_IPA_SIZE": {
+		desc: "Maximum IPA shift supported by the host",
+		id:   165,
+	},
+}
+
+func setCPUtype(hypervisorType vc.HypervisorType) error {
+	return nil
+}
+
+// kvmIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func kvmIsUsable() error {
+	return genericKvmIsUsable()
+}
+
+func checkKVMExtensions() error {
+	results, err := genericCheckKVMExtensions(archRequiredKVMExtensions)
+	if err != nil {
+		return err
+	}
+
+	// different host supports different maximum IPA limit
+	ipa := results["KVM_CAP_ARM_VM_IPA_SIZE"]
+	fields := logrus.Fields{
+		"type": "kvm extension",
+		"name": "KVM_CAP_ARM_VM_IPA_SIZE",
+	}
+
+	kataLog.WithFields(fields).Infof("IPA limit size: %d bits.", ipa)
+
+	return nil
+}
+
+func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if err := kvmIsUsable(); err != nil {
+		return err
+	}
+
+	return checkKVMExtensions()
+}
+
+// hostIsVMContainerCapable checks to see if the host is theoretically capable
+// of creating a VM container.
+func hostIsVMContainerCapable(details vmContainerCapableDetails) error {
+
+	_, err := getCPUInfo(details.cpuInfoFile)
+	if err != nil {
+		return err
+	}
+
+	count, err := checkKernelModules(details.requiredKernelModules, archKernelParamHandler)
+	if err != nil {
+		return err
+	}
+
+	if count == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("ERROR: %s", failMessage)
+
+}
+
+func archKernelParamHandler(onVMM bool, fields logrus.Fields, msg string) bool {
+	return genericArchKernelParamHandler(onVMM, fields, msg)
+}
+
+// The CPU Vendor here for Arm means the CPU core
+// IP Implementer.
+// normalizeArmVendor maps 'CPU implementer' in /proc/cpuinfo
+// to human-readable description of that value.
+func normalizeArmVendor(vendor string) string {
+
+	switch vendor {
+	case "0x41":
+		vendor = "ARM Limited"
+	default:
+		vendor = "3rd Party Limited"
+	}
+
+	return vendor
+}
+
+// The CPU Model here for Arm means the Instruction set, that is
+// the variant number of Arm processor.
+// normalizeArmModel maps 'CPU architecture' in /proc/cpuinfo
+// to human-readable description of that value.
+func normalizeArmModel(model string) string {
+	switch model {
+	case "8":
+		model = "v8"
+	case "7", "7M", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
+		model = "v7"
+	case "6", "6TEJ":
+		model = "v6"
+	case "5", "5T", "5TE", "5TEJ":
+		model = "v5"
+	case "4", "4T":
+		model = "v4"
+	case "3":
+		model = "v3"
+	default:
+		model = "unknown"
+	}
+
+	return model
+}
+
+func getCPUDetails() (string, string, error) {
+	vendor, model, err := genericGetCPUDetails()
+	if err == nil {
+		vendor = normalizeArmVendor(vendor)
+		model = normalizeArmModel(model)
+	}
+
+	return vendor, model, err
+}

src/runtime/cli/kata-check_arm64_test.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2018 ARM Limited
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func setupCheckHostIsVMContainerCapable(assert *assert.Assertions, cpuInfoFile string, cpuData []testCPUData, moduleData []testModuleData) {
+	//For now, Arm64 only deal with module check
+	_ = cpuData
+
+	createModules(assert, cpuInfoFile, moduleData)
+
+	err := makeCPUInfoFile(cpuInfoFile, "", "")
+	assert.NoError(err)
+}
+
+func TestCCCheckCLIFunction(t *testing.T) {
+	var cpuData []testCPUData
+	moduleData := []testModuleData{
+		{filepath.Join(sysModuleDir, "kvm"), true, ""},
+		{filepath.Join(sysModuleDir, "vhost"), true, ""},
+		{filepath.Join(sysModuleDir, "vhost_net"), true, ""},
+	}
+
+	genericCheckCLIFunction(t, cpuData, moduleData)
+}
+
+func TestGetCPUDetails(t *testing.T) {
+	type testData struct {
+		contents                string
+		expectedNormalizeVendor string
+		expectedNormalizeModel  string
+		expectError             bool
+	}
+
+	validVendorName := "0x41"
+	validNormalizeVendorName := "ARM Limited"
+	validVendor := fmt.Sprintf(`%s  : %s`, archCPUVendorField, validVendorName)
+
+	validModelName := "8"
+	validNormalizeModelName := "v8"
+	validModel := fmt.Sprintf(`%s   : %s`, archCPUModelField, validModelName)
+
+	validContents := fmt.Sprintf(`
+a       : b
+%s
+foo     : bar
+%s
+`, validVendor, validModel)
+
+	data := []testData{
+		{"", "", "", true},
+		{"invalid", "", "", true},
+		{archCPUVendorField, "", "", true},
+		{validVendor, "", "", true},
+		{validModel, "", "", true},
+		{validContents, validNormalizeVendorName, validNormalizeModelName, false},
+	}
+
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	savedProcCPUInfo := procCPUInfo
+
+	testProcCPUInfo := filepath.Join(tmpdir, "cpuinfo")
+
+	// override
+	procCPUInfo = testProcCPUInfo
+
+	defer func() {
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	_, _, err = getCPUDetails()
+	// ENOENT
+	assert.Error(t, err)
+	assert.True(t, os.IsNotExist(err))
+
+	for _, d := range data {
+		err := createFile(procCPUInfo, d.contents)
+		assert.NoError(t, err)
+
+		vendor, model, err := getCPUDetails()
+
+		if d.expectError {
+			assert.Error(t, err, fmt.Sprintf("%+v", d))
+			continue
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("%+v", d))
+			assert.Equal(t, d.expectedNormalizeVendor, vendor)
+			assert.Equal(t, d.expectedNormalizeModel, model)
+		}
+	}
+}
+
+func TestSetCPUtype(t *testing.T) {
+	testSetCPUTypeGeneric(t)
+}

src/runtime/cli/kata-check_data_amd64_test.go

@@ -0,0 +1,58 @@
+package main
+
+const testCPUInfoTemplate = `
+processor	: 0
+vendor_id	: {{.VendorID}}
+cpu family	: 6
+model		: 61
+model name	: Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz
+stepping	: 4
+microcode	: 0x25
+cpu MHz		: 1999.987
+cache size	: 4096 KB
+physical id	: 0
+siblings	: 4
+core id		: 0
+cpu cores	: 2
+apicid		: 0
+initial apicid	: 0
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 20
+wp		: yes
+flags		: {{.Flags}}
+bugs		:
+bogomips	: 5188.36
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 39 bits physical, 48 bits virtual
+power management:
+
+processor	: 1
+vendor_id	: {{.VendorID}}
+cpu family	: 6
+model		: 61
+model name	: Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz
+stepping	: 4
+microcode	: 0x25
+cpu MHz		: 1999.987
+cache size	: 4096 KB
+physical id	: 0
+siblings	: 4
+core id		: 0
+cpu cores	: 2
+apicid		: 1
+initial apicid	: 1
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 20
+wp		: yes
+flags		: {{.Flags}}
+bugs		:
+bogomips	: 5194.90
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 39 bits physical, 48 bits virtual
+power management:
+
+`

src/runtime/cli/kata-check_data_arm64_test.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2018 ARM Limited
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+const testCPUInfoTemplate = `
+processor	: 0
+BogoMIPS	: 500.00
+Features	: fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid
+CPU implementer	: 0x41
+CPU architecture: 8
+CPU variant	: 0x1
+CPU part	: 0xd07
+CPU revision	: 2
+
+processor	: 1
+BogoMIPS	: 500.00
+Features	: fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid
+CPU implementer	: 0x41
+CPU architecture: 8
+CPU variant	: 0x1
+CPU part	: 0xd07
+CPU revision	: 2
+
+`

src/runtime/cli/kata-check_data_ppc64le_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+)
+
+var testCPUInfoTemplate = setTestCPUInfoTemplate()
+
+func setTestCPUInfoTemplate() string {
+
+	var kataLog *logrus.Entry
+	content, err := ioutil.ReadFile("/proc/cpuinfo")
+
+	if err != nil {
+		kataLog.WithError(err).Error("failed to read file /proc/cpuinfo")
+	}
+	return string(content)
+}

src/runtime/cli/kata-check_data_s390x_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+const testCPUInfoTemplate = `
+vendor_id       : IBM/S390
+# processors    : 4
+bogomips per cpu: 20325.00
+max thread id   : 0
+features	: esan3 zarch stfle msa ldisp eimm dfp edat etf3eh highgprs te vx sie 
+cache0          : level=1 type=Data scope=Private size=128K line_size=256 associativity=8
+cache1          : level=1 type=Instruction scope=Private size=96K line_size=256 associativity=6
+cache2          : level=2 type=Data scope=Private size=2048K line_size=256 associativity=8
+cache3          : level=2 type=Instruction scope=Private size=2048K line_size=256 associativity=8
+cache4          : level=3 type=Unified scope=Shared size=65536K line_size=256 associativity=16
+cache5          : level=4 type=Unified scope=Shared size=491520K line_size=256 associativity=30
+processor 0: version = FF,  identification = FFFFFF,  machine = 2964
+`

src/runtime/cli/kata-check_generic_test.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// +build arm64 ppc64le
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func testSetCPUTypeGeneric(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	savedArchRequiredCPUFlags := archRequiredCPUFlags
+	savedArchRequiredCPUAttribs := archRequiredCPUAttribs
+	savedArchRequiredKernelModules := archRequiredKernelModules
+
+	defer func() {
+		archRequiredCPUFlags = savedArchRequiredCPUFlags
+		archRequiredCPUAttribs = savedArchRequiredCPUAttribs
+		archRequiredKernelModules = savedArchRequiredKernelModules
+	}()
+
+	assert.Empty(archRequiredCPUFlags)
+	assert.Empty(archRequiredCPUAttribs)
+	assert.NotEmpty(archRequiredKernelModules)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	setCPUtype(config.HypervisorType)
+
+	assert.Equal(archRequiredCPUFlags, savedArchRequiredCPUFlags)
+	assert.Equal(archRequiredCPUAttribs, savedArchRequiredCPUAttribs)
+	assert.Equal(archRequiredKernelModules, savedArchRequiredKernelModules)
+}

src/runtime/cli/kata-check_ppc64le.go

@@ -0,0 +1,190 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	cpuFlagsTag        = genericCPUFlagsTag
+	archCPUVendorField = ""
+	archCPUModelField  = "model"
+)
+
+var (
+	ppc64CpuCmd     = "ppc64_cpu"
+	smtStatusOption = "--smt"
+	_               = genericCPUVendorField
+	_               = genericCPUModelField
+)
+
+// archRequiredCPUFlags maps a CPU flag value to search for and a
+// human-readable description of that value.
+var archRequiredCPUFlags = map[string]string{}
+
+// archRequiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
+// and a human-readable description of that value.
+var archRequiredCPUAttribs = map[string]string{}
+
+// archRequiredKernelModules maps a required module name to a human-readable
+// description of the modules functionality and an optional list of
+// required module parameters.
+var archRequiredKernelModules = map[string]kernelModule{
+	"kvm": {
+		desc:     "Kernel-based Virtual Machine",
+		required: true,
+	},
+	"kvm_hv": {
+		desc:     "Kernel-based Virtual Machine hardware virtualization",
+		required: true,
+	},
+	"vhost_vsock": {
+		desc:     "Host Support for Linux VM Sockets",
+		required: false,
+	},
+}
+
+func setCPUtype(hypervisorType vc.HypervisorType) error {
+	return nil
+}
+
+func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	return kvmIsUsable()
+}
+
+// hostIsVMContainerCapable checks to see if the host is theoretically capable
+// of creating a VM container.
+func hostIsVMContainerCapable(details vmContainerCapableDetails) error {
+
+	_, err := getCPUInfo(details.cpuInfoFile)
+	if err != nil {
+		return err
+	}
+
+	text, err := katautils.GetFileContents(details.cpuInfoFile)
+	if err != nil {
+		return err
+	}
+
+	ae := regexp.MustCompile("[0-9]+")
+	re := regexp.MustCompile("POWER[0-9]")
+	powerProcessor, err := strconv.Atoi(ae.FindString(re.FindString(text)))
+	if err != nil {
+		kataLog.WithError(err).Error("Failed to find Power Processor number from ", details.cpuInfoFile)
+	}
+
+	if powerProcessor <= 8 {
+		if !isSMTOff() {
+			return fmt.Errorf("SMT is not Off. %s", failMessage)
+		}
+	}
+
+	count, err := checkKernelModules(details.requiredKernelModules, archKernelParamHandler)
+	if err != nil {
+		return err
+	}
+
+	if count == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("ERROR: %s", failMessage)
+}
+
+// kvmIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func kvmIsUsable() error {
+	return genericKvmIsUsable()
+}
+
+func archKernelParamHandler(onVMM bool, fields logrus.Fields, msg string) bool {
+	return genericArchKernelParamHandler(onVMM, fields, msg)
+}
+
+func getPPC64leCPUInfo(cpuInfoFile string) (string, error) {
+	text, err := katautils.GetFileContents(cpuInfoFile)
+	if err != nil {
+		return "", err
+	}
+
+	if len(strings.TrimSpace(text)) == 0 {
+		return "", fmt.Errorf("Cannot determine CPU details")
+	}
+
+	return text, nil
+}
+
+func getCPUDetails() (vendor, model string, err error) {
+
+	if vendor, model, err := genericGetCPUDetails(); err == nil {
+		return vendor, model, nil
+	}
+
+	cpuinfo, err := getPPC64leCPUInfo(procCPUInfo)
+	if err != nil {
+		return "", "", err
+	}
+
+	lines := strings.Split(cpuinfo, "\n")
+
+	for _, line := range lines {
+		if archCPUVendorField != "" {
+			if strings.HasPrefix(line, archCPUVendorField) {
+				fields := strings.Split(line, ":")
+				if len(fields) > 1 {
+					vendor = strings.TrimSpace(fields[1])
+				}
+			}
+		}
+
+		if archCPUModelField != "" {
+			if strings.HasPrefix(line, archCPUModelField) {
+				fields := strings.Split(line, ":")
+				if len(fields) > 1 {
+					model = strings.TrimSpace(fields[1])
+				}
+			}
+		}
+	}
+
+	if archCPUVendorField != "" && vendor == "" {
+		return "", "", fmt.Errorf("cannot find vendor field in file %v", procCPUInfo)
+	}
+
+	if archCPUModelField != "" && model == "" {
+		return "", "", fmt.Errorf("cannot find model field in file %v", procCPUInfo)
+	}
+
+	return vendor, model, nil
+}
+
+func isSMTOff() bool {
+
+	// Check if the SMT is available and off
+
+	cmd := exec.Command(ppc64CpuCmd, smtStatusOption)
+	additionalEnv := "LANG=C"
+	cmd.Env = append(cmd.Env, additionalEnv)
+	out, err := cmd.Output()
+
+	if err == nil && strings.TrimRight(string(out), "\n") == "SMT is off" {
+		return true
+	} else if err != nil {
+		kataLog.Warn("ppc64_cpu isn't installed, can't detect SMT")
+		return true
+	}
+
+	return false
+}

src/runtime/cli/kata-check_ppc64le_test.go

@@ -0,0 +1,175 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func setupCheckHostIsVMContainerCapable(assert *assert.Assertions, cpuInfoFile string, cpuData []testCPUData, moduleData []testModuleData) {
+	createModules(assert, cpuInfoFile, moduleData)
+
+	// all the modules files have now been created, so deal with the
+	// cpuinfo data.
+	for _, d := range cpuData {
+		err := makeCPUInfoFile(cpuInfoFile, d.vendorID, d.flags)
+		assert.NoError(err)
+
+		details := vmContainerCapableDetails{
+			cpuInfoFile:           cpuInfoFile,
+			requiredCPUFlags:      archRequiredCPUFlags,
+			requiredCPUAttribs:    archRequiredCPUAttribs,
+			requiredKernelModules: archRequiredKernelModules,
+		}
+
+		err = hostIsVMContainerCapable(details)
+		if d.expectError {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+		}
+	}
+}
+
+func TestCCCheckCLIFunction(t *testing.T) {
+	cpuData := []testCPUData{
+		fakeCPUData,
+	}
+
+	moduleData := []testModuleData{
+		{filepath.Join(sysModuleDir, "kvm"), false, "Y"},
+		{filepath.Join(sysModuleDir, "kvm_hv"), false, "Y"},
+	}
+
+	genericCheckCLIFunction(t, cpuData, moduleData)
+}
+
+func TestArchKernelParamHandler(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		onVMM        bool
+		expectIgnore bool
+		fields       logrus.Fields
+		msg          string
+	}
+
+	data := []testData{
+		{true, false, logrus.Fields{}, ""},
+		{false, false, logrus.Fields{}, ""},
+
+		{
+			false,
+			false,
+			logrus.Fields{
+				// wrong type
+				"parameter": 123,
+			},
+			"foo",
+		},
+
+		{
+			false,
+			false,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+		},
+
+		{
+			true,
+			true,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+		},
+
+		{
+			false,
+			true,
+			logrus.Fields{
+				"parameter": "nested",
+			},
+			"",
+		},
+	}
+
+	for i, d := range data {
+		result := archKernelParamHandler(d.onVMM, d.fields, d.msg)
+		if d.expectIgnore {
+			assert.True(result, "test %d (%+v)", i, d)
+		} else {
+			assert.False(result, "test %d (%+v)", i, d)
+		}
+	}
+}
+
+func TestKvmIsUsable(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedKvmDevice := kvmDevice
+	fakeKVMDevice := filepath.Join(dir, "kvm")
+	kvmDevice = fakeKVMDevice
+
+	defer func() {
+		kvmDevice = savedKvmDevice
+	}()
+
+	err = kvmIsUsable()
+	assert.Error(err)
+
+	err = createEmptyFile(fakeKVMDevice)
+	assert.NoError(err)
+
+	err = kvmIsUsable()
+	assert.Error(err)
+}
+
+func TestGetCPUDetails(t *testing.T) {
+
+	const validVendorName = ""
+	validVendor := fmt.Sprintf(`%s  : %s`, archCPUVendorField, validVendorName)
+
+	const validModelName = "8247-22L"
+	validModel := fmt.Sprintf(`%s   : %s`, archCPUModelField, validModelName)
+
+	validContents := fmt.Sprintf(`
+a       : b
+%s
+foo     : bar
+%s
+`, validVendor, validModel)
+
+	data := []testCPUDetail{
+		{"", "", "", true},
+		{"invalid", "", "", true},
+		{archCPUVendorField, "", "", true},
+		{validVendor, "", "", true},
+		{validModel, "", validModelName, false},
+		{validContents, validVendorName, validModelName, false},
+	}
+
+	genericTestGetCPUDetails(t, validVendor, validModel, validContents, data)
+}
+
+func TestSetCPUtype(t *testing.T) {
+	testSetCPUTypeGeneric(t)
+}

src/runtime/cli/kata-check_s390x.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	cpuFlagsTag        = genericCPUFlagsTag
+	archCPUVendorField = genericCPUVendorField
+	// On s390x the cpu model is indicated by the field machine.
+	// Example:
+	// processor 0: version = FF,  identification = 3FEC87,  machine = 2964
+	archCPUModelField = "machine"
+)
+
+// archRequiredCPUFlags maps a CPU flag value to search for and a
+// human-readable description of that value.
+var archRequiredCPUFlags = map[string]string{}
+
+// archRequiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
+// and a human-readable description of that value.
+var archRequiredCPUAttribs = map[string]string{}
+
+// archRequiredKernelModules maps a required module name to a human-readable
+// description of the modules functionality and an optional list of
+// required module parameters.
+var archRequiredKernelModules = map[string]kernelModule{
+	"kvm": {
+		desc:     "Kernel-based Virtual Machine",
+		required: true,
+	},
+	"vhost_vsock": {
+		desc:     "Host Support for Linux VM Sockets",
+		required: false,
+	},
+}
+
+func setCPUtype(hypervisorType vc.HypervisorType) error {
+	return nil
+}
+
+// kvmIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func kvmIsUsable() error {
+	return genericKvmIsUsable()
+}
+
+func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	return kvmIsUsable()
+}
+
+// hostIsVMContainerCapable checks to see if the host is theoretically capable
+// of creating a VM container.
+func hostIsVMContainerCapable(details vmContainerCapableDetails) error {
+
+	_, err := getCPUInfo(details.cpuInfoFile)
+	if err != nil {
+		return err
+	}
+
+	count, err := checkKernelModules(details.requiredKernelModules, archKernelParamHandler)
+	if err != nil {
+		return err
+	}
+
+	if count == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("ERROR: %s", failMessage)
+
+}
+
+func archKernelParamHandler(onVMM bool, fields logrus.Fields, msg string) bool {
+	return genericArchKernelParamHandler(onVMM, fields, msg)
+}
+
+// getS390xCPUDetails returns the cpu information
+func getS390xCPUDetails() (vendor, model string, err error) {
+	prefixModel := "processor"
+	cpuinfo, err := getCPUInfo(procCPUInfo)
+	if err != nil {
+		return "", "", err
+	}
+
+	lines := strings.Split(cpuinfo, "\n")
+
+	for _, line := range lines {
+		if archCPUVendorField != "" {
+			if strings.HasPrefix(line, archCPUVendorField) {
+				fields := strings.Split(line, ":")
+				if len(fields) > 1 {
+					vendor = strings.TrimSpace(fields[1])
+				}
+			}
+		}
+		if archCPUModelField != "" {
+			if strings.HasPrefix(line, prefixModel) {
+				fields := strings.Split(strings.TrimSpace(line), ",")
+				cpuModel := strings.Split(fields[2], "=")
+				model = strings.TrimSpace(cpuModel[1])
+			}
+		}
+	}
+
+	if vendor == "" {
+		return "", "", fmt.Errorf("cannot find vendor field in file %v", procCPUInfo)
+	}
+
+	if model == "" {
+		return "", "", fmt.Errorf("Error in parsing cpu model from %v", procCPUInfo)
+	}
+
+	return vendor, model, nil
+}
+
+func getCPUDetails() (vendor, model string, err error) {
+	if vendor, model, err := genericGetCPUDetails(); err == nil {
+		return vendor, model, nil
+	}
+	return getS390xCPUDetails()
+}

src/runtime/cli/kata-check_s390x_test.go

@@ -0,0 +1,168 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func setupCheckHostIsVMContainerCapable(assert *assert.Assertions, cpuInfoFile string, cpuData []testCPUData, moduleData []testModuleData) {
+	createModules(assert, cpuInfoFile, moduleData)
+
+	// all the modules files have now been created, so deal with the
+	// cpuinfo data.
+	for _, d := range cpuData {
+		err := makeCPUInfoFile(cpuInfoFile, d.vendorID, d.flags)
+		assert.NoError(err)
+
+		details := vmContainerCapableDetails{
+			cpuInfoFile:           cpuInfoFile,
+			requiredCPUFlags:      archRequiredCPUFlags,
+			requiredCPUAttribs:    archRequiredCPUAttribs,
+			requiredKernelModules: archRequiredKernelModules,
+		}
+
+		err = hostIsVMContainerCapable(details)
+		if d.expectError {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+		}
+	}
+}
+
+func TestCCCheckCLIFunction(t *testing.T) {
+	cpuData := []testCPUData{
+		fakeCPUData,
+	}
+
+	moduleData := []testModuleData{
+		{filepath.Join(sysModuleDir, "kvm"), false, "Y"},
+	}
+
+	genericCheckCLIFunction(t, cpuData, moduleData)
+}
+
+func TestArchKernelParamHandler(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		onVMM        bool
+		expectIgnore bool
+		fields       logrus.Fields
+		msg          string
+	}
+
+	data := []testData{
+		{true, false, logrus.Fields{}, ""},
+		{false, false, logrus.Fields{}, ""},
+
+		{
+			false,
+			false,
+			logrus.Fields{
+				// wrong type
+				"parameter": 123,
+			},
+			"foo",
+		},
+
+		{
+			false,
+			false,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+		},
+
+		{
+			true,
+			true,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+		},
+
+		{
+			false,
+			true,
+			logrus.Fields{
+				"parameter": "nested",
+			},
+			"",
+		},
+	}
+
+	for i, d := range data {
+		result := archKernelParamHandler(d.onVMM, d.fields, d.msg)
+		if d.expectIgnore {
+			assert.True(result, "test %d (%+v)", i, d)
+		} else {
+			assert.False(result, "test %d (%+v)", i, d)
+		}
+	}
+}
+
+func TestKvmIsUsable(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedKvmDevice := kvmDevice
+	fakeKVMDevice := filepath.Join(dir, "kvm")
+	kvmDevice = fakeKVMDevice
+
+	defer func() {
+		kvmDevice = savedKvmDevice
+	}()
+
+	err = kvmIsUsable()
+	assert.Error(err)
+
+	err = createEmptyFile(fakeKVMDevice)
+	assert.NoError(err)
+
+	err = kvmIsUsable()
+	assert.Error(err)
+}
+
+func TestGetCPUDetails(t *testing.T) {
+	const validVendorName = "a vendor"
+	validVendor := fmt.Sprintf(`%s  : %s`, archCPUVendorField, validVendorName)
+
+	const validModelName = "some CPU model"
+	validModel := fmt.Sprintf(`processor 0: version = 00,  identification = XXXXX, %s = %s`, archCPUModelField, validModelName)
+
+	validContents := fmt.Sprintf(`
+a       : b
+%s
+foo     : bar
+%s
+`, validVendor, validModel)
+
+	data := []testCPUDetail{
+		{"", "", "", true},
+		{"invalid", "", "", true},
+		{archCPUVendorField, "", "", true},
+		{validVendor, "", "", true},
+		{validModel, "", "", true},
+		{validContents, validVendorName, validModelName, false},
+	}
+	genericTestGetCPUDetails(t, validVendor, validModel, validContents, data)
+}

src/runtime/cli/kata-check_test.go

@@ -0,0 +1,992 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	ktu "github.com/kata-containers/runtime/pkg/katatestutils"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+type testModuleData struct {
+	path     string
+	isDir    bool
+	contents string
+}
+
+// nolint: structcheck, unused, deadcode
+type testCPUData struct {
+	vendorID    string
+	flags       string
+	expectError bool
+}
+
+// nolint: structcheck, unused, deadcode
+type testCPUDetail struct {
+	contents       string
+	expectedVendor string
+	expectedModel  string
+	expectError    bool
+}
+
+var fakeCPUData = testCPUData{"", "", false}
+
+func createFile(file, contents string) error {
+	return ioutil.WriteFile(file, []byte(contents), testFileMode)
+}
+
+func createModules(assert *assert.Assertions, cpuInfoFile string, moduleData []testModuleData) {
+	for _, d := range moduleData {
+		var dir string
+
+		if d.isDir {
+			dir = d.path
+		} else {
+			dir = path.Dir(d.path)
+		}
+
+		err := os.MkdirAll(dir, testDirMode)
+		assert.NoError(err)
+
+		if !d.isDir {
+			err = createFile(d.path, d.contents)
+			assert.NoError(err)
+		}
+
+		details := vmContainerCapableDetails{
+			cpuInfoFile: cpuInfoFile,
+		}
+
+		err = hostIsVMContainerCapable(details)
+		if katautils.FileExists(cpuInfoFile) {
+			assert.NoError(err)
+		} else {
+			assert.Error(err)
+		}
+	}
+}
+
+func checkKernelParamHandler(assert *assert.Assertions, kernelModulesToCreate, expectedKernelModules map[string]kernelModule, handler kernelParamHandler, expectHandlerError bool, expectedErrorCount uint32) {
+	err := os.RemoveAll(sysModuleDir)
+	assert.NoError(err)
+
+	count, err := checkKernelModules(map[string]kernelModule{}, handler)
+
+	// No required modules means no error
+	assert.NoError(err)
+	assert.Equal(count, uint32(0))
+
+	count, err = checkKernelModules(expectedKernelModules, handler)
+	assert.NoError(err)
+
+	// No modules exist
+	expectedCount := len(expectedKernelModules)
+	assert.Equal(count, uint32(expectedCount))
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	assert.NoError(err)
+
+	for module, details := range kernelModulesToCreate {
+		path := filepath.Join(sysModuleDir, module)
+		err = os.MkdirAll(path, testDirMode)
+		assert.NoError(err)
+
+		paramDir := filepath.Join(path, "parameters")
+		err = os.MkdirAll(paramDir, testDirMode)
+		assert.NoError(err)
+
+		for param, value := range details.parameters {
+			paramPath := filepath.Join(paramDir, param)
+			err = createFile(paramPath, value)
+			assert.NoError(err)
+		}
+	}
+
+	count, err = checkKernelModules(expectedKernelModules, handler)
+
+	if expectHandlerError {
+		assert.Error(err)
+		return
+	}
+
+	assert.NoError(err)
+	assert.Equal(count, expectedErrorCount)
+}
+
+func makeCPUInfoFile(path, vendorID, flags string) error {
+	t := template.New("cpuinfo")
+
+	t, err := t.Parse(testCPUInfoTemplate)
+	if err != nil {
+		return err
+	}
+
+	args := map[string]string{
+		"Flags":    flags,
+		"VendorID": vendorID,
+	}
+
+	contents := &bytes.Buffer{}
+
+	err = t.Execute(contents, args)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(path, contents.Bytes(), testFileMode)
+}
+
+// nolint: unused, deadcode
+func genericTestGetCPUDetails(t *testing.T, validVendor string, validModel string, validContents string, data []testCPUDetail) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	savedProcCPUInfo := procCPUInfo
+
+	testProcCPUInfo := filepath.Join(tmpdir, "cpuinfo")
+
+	// override
+	procCPUInfo = testProcCPUInfo
+
+	defer func() {
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	_, _, err = getCPUDetails()
+	// ENOENT
+	assert.Error(t, err)
+	assert.True(t, os.IsNotExist(err))
+
+	for _, d := range data {
+		err := createFile(procCPUInfo, d.contents)
+		assert.NoError(t, err)
+
+		vendor, model, err := getCPUDetails()
+
+		if d.expectError {
+			assert.Error(t, err, fmt.Sprintf("%+v", d))
+			continue
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("%+v", d))
+			assert.Equal(t, d.expectedVendor, vendor)
+			assert.Equal(t, d.expectedModel, model)
+		}
+	}
+}
+
+func genericCheckCLIFunction(t *testing.T, cpuData []testCPUData, moduleData []testModuleData) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	_, config, err := makeRuntimeConfig(dir)
+	assert.NoError(err)
+
+	savedSysModuleDir := sysModuleDir
+	savedProcCPUInfo := procCPUInfo
+
+	cpuInfoFile := filepath.Join(dir, "cpuinfo")
+
+	// XXX: override
+	sysModuleDir = filepath.Join(dir, "sys/module")
+	procCPUInfo = cpuInfoFile
+
+	defer func() {
+		sysModuleDir = savedSysModuleDir
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	// Replace sysModuleDir in moduleData with the test temp path
+	for i := range moduleData {
+		moduleData[i].path = strings.Replace(moduleData[i].path, savedSysModuleDir, sysModuleDir, 1)
+	}
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0666)
+	assert.NoError(err)
+	defer devNull.Close()
+
+	savedLogOutput := kataLog.Logger.Out
+
+	// discard normal output
+	kataLog.Logger.Out = devNull
+
+	defer func() {
+		kataLog.Logger.Out = savedLogOutput
+	}()
+
+	setupCheckHostIsVMContainerCapable(assert, cpuInfoFile, cpuData, moduleData)
+
+	flagSet := &flag.FlagSet{}
+	ctx := createCLIContext(flagSet)
+	ctx.App.Name = "foo"
+	ctx.App.Metadata["runtimeConfig"] = config
+
+	// create buffer to save logger output
+	buf := &bytes.Buffer{}
+
+	// capture output this time
+	kataLog.Logger.Out = buf
+
+	fn, ok := kataCheckCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.NoError(err)
+
+	output := buf.String()
+
+	for _, c := range cpuData {
+		if c == fakeCPUData {
+			continue
+		}
+
+		assert.True(findAnchoredString(output, c.vendorID))
+		for _, flag := range strings.Fields(c.flags) {
+			assert.True(findAnchoredString(output, flag))
+		}
+	}
+
+	for _, m := range moduleData {
+		name := path.Base(m.path)
+		assert.True(findAnchoredString(output, name))
+	}
+}
+func TestCheckGetCPUInfo(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		contents       string
+		expectedResult string
+		expectError    bool
+	}
+
+	data := []testData{
+		{"", "", true},
+		{" ", "", true},
+		{"\n", "", true},
+		{"\n\n", "", true},
+		{"hello\n", "hello", false},
+		{"foo\n\n", "foo", false},
+		{"foo\n\nbar\n\n", "foo", false},
+		{"foo\n\nbar\nbaz\n\n", "foo", false},
+	}
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "cpuinfo")
+	// file doesn't exist
+	_, err = getCPUInfo(file)
+	assert.Error(err)
+
+	for _, d := range data {
+		err = ioutil.WriteFile(file, []byte(d.contents), testFileMode)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.Remove(file)
+
+		contents, err := getCPUInfo(file)
+		if d.expectError {
+			assert.Error(err, fmt.Sprintf("got %q, test data: %+v", contents, d))
+		} else {
+			assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", contents, d))
+		}
+
+		assert.Equal(d.expectedResult, contents)
+	}
+}
+
+func TestCheckFindAnchoredString(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		haystack      string
+		needle        string
+		expectSuccess bool
+	}
+
+	data := []testData{
+		{"", "", false},
+		{"", "foo", false},
+		{"foo", "", false},
+		{"food", "foo", false},
+		{"foo", "foo", true},
+		{"foo bar", "foo", true},
+		{"foo bar baz", "bar", true},
+	}
+
+	for _, d := range data {
+		result := findAnchoredString(d.haystack, d.needle)
+
+		if d.expectSuccess {
+			assert.True(result)
+		} else {
+			assert.False(result)
+		}
+	}
+}
+
+func TestCheckGetCPUFlags(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		cpuinfo       string
+		expectedFlags string
+	}
+
+	data := []testData{
+		{"", ""},
+		{"foo", ""},
+		{"foo bar", ""},
+		{":", ""},
+
+		{
+			cpuFlagsTag,
+			"",
+		},
+		{
+			cpuFlagsTag + ":",
+			"",
+		},
+		{
+			fmt.Sprintf("%s: a b c", cpuFlagsTag),
+			"a b c",
+		},
+		{
+			fmt.Sprintf("%s: a b c foo bar d", cpuFlagsTag),
+			"a b c foo bar d",
+		},
+	}
+
+	for _, d := range data {
+		result := getCPUFlags(d.cpuinfo)
+		assert.Equal(d.expectedFlags, result)
+	}
+}
+
+func TestCheckCheckCPUFlags(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		cpuflags    string
+		required    map[string]string
+		expectCount uint32
+	}
+
+	data := []testData{
+		{
+			"",
+			map[string]string{},
+			0,
+		},
+		{
+			"",
+			map[string]string{
+				"a": "A flag",
+			},
+			0,
+		},
+		{
+			"",
+			map[string]string{
+				"a": "A flag",
+				"b": "B flag",
+			},
+			0,
+		},
+		{
+			"a b c",
+			map[string]string{
+				"b": "B flag",
+			},
+			0,
+		},
+		{
+			"a b c",
+			map[string]string{
+				"x": "X flag",
+				"y": "Y flag",
+				"z": "Z flag",
+			},
+			3,
+		},
+	}
+
+	for _, d := range data {
+		count := checkCPUFlags(d.cpuflags, d.required)
+		assert.Equal(d.expectCount, count, "%+v", d)
+	}
+}
+
+func TestCheckCheckCPUAttribs(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		cpuinfo     string
+		required    map[string]string
+		expectCount uint32
+	}
+
+	data := []testData{
+		{
+			"",
+			map[string]string{},
+			0,
+		},
+		{
+			"",
+			map[string]string{
+				"a": "",
+			},
+			0,
+		},
+		{
+			"a: b",
+			map[string]string{
+				"b": "B attribute",
+			},
+			0,
+		},
+		{
+			"a: b\nc: d\ne: f",
+			map[string]string{
+				"b": "B attribute",
+			},
+			0,
+		},
+		{
+			"a: b\n",
+			map[string]string{
+				"b": "B attribute",
+				"c": "C attribute",
+				"d": "D attribute",
+			},
+			2,
+		},
+		{
+			"a: b\nc: d\ne: f",
+			map[string]string{
+				"b": "B attribute",
+				"d": "D attribute",
+				"f": "F attribute",
+			},
+			0,
+		},
+	}
+
+	for _, d := range data {
+		count := checkCPUAttribs(d.cpuinfo, d.required)
+		assert.Equal(d.expectCount, count, "%+v", d)
+	}
+}
+
+func TestCheckHaveKernelModule(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedModProbeCmd := modProbeCmd
+	savedSysModuleDir := sysModuleDir
+
+	// XXX: override (fake the modprobe command failing)
+	modProbeCmd = "false"
+	sysModuleDir = filepath.Join(dir, "sys/module")
+
+	defer func() {
+		modProbeCmd = savedModProbeCmd
+		sysModuleDir = savedSysModuleDir
+	}()
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	module := "foo"
+
+	result := haveKernelModule(module)
+	assert.False(result)
+
+	// XXX: override - make our fake "modprobe" succeed
+	modProbeCmd = "true"
+
+	result = haveKernelModule(module)
+	assert.True(result)
+
+	// disable "modprobe" again
+	modProbeCmd = "false"
+
+	fooDir := filepath.Join(sysModuleDir, module)
+	err = os.MkdirAll(fooDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result = haveKernelModule(module)
+	assert.True(result)
+}
+
+func TestCheckCheckKernelModules(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedModProbeCmd := modProbeCmd
+	savedSysModuleDir := sysModuleDir
+
+	// XXX: override (fake the modprobe command failing)
+	modProbeCmd = "false"
+	sysModuleDir = filepath.Join(dir, "sys/module")
+
+	defer func() {
+		modProbeCmd = savedModProbeCmd
+		sysModuleDir = savedSysModuleDir
+	}()
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testData := map[string]kernelModule{
+		"foo": {
+			desc:       "desc",
+			parameters: map[string]string{},
+			required:   true,
+		},
+		"bar": {
+			desc: "desc",
+			parameters: map[string]string{
+				"param1": "hello",
+				"param2": "world",
+				"param3": "a",
+				"param4": ".",
+			},
+			required: true,
+		},
+	}
+
+	count, err := checkKernelModules(map[string]kernelModule{}, nil)
+	// No required modules means no error
+	assert.NoError(err)
+	assert.Equal(count, uint32(0))
+
+	count, err = checkKernelModules(testData, nil)
+	assert.NoError(err)
+	// No modules exist
+	assert.Equal(count, uint32(2))
+
+	for module, details := range testData {
+		path := filepath.Join(sysModuleDir, module)
+		err = os.MkdirAll(path, testDirMode)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		paramDir := filepath.Join(path, "parameters")
+		err = os.MkdirAll(paramDir, testDirMode)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for param, value := range details.parameters {
+			paramPath := filepath.Join(paramDir, param)
+			err = createFile(paramPath, value)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	count, err = checkKernelModules(testData, nil)
+	assert.NoError(err)
+	assert.Equal(count, uint32(0))
+}
+
+func TestCheckCheckKernelModulesUnreadableFile(t *testing.T) {
+	assert := assert.New(t)
+
+	if tc.NotValid(ktu.NeedNonRoot()) {
+		t.Skip(ktu.TestDisabledNeedNonRoot)
+	}
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	testData := map[string]kernelModule{
+		"foo": {
+			desc: "desc",
+			parameters: map[string]string{
+				"param1": "wibble",
+			},
+			required: true,
+		},
+	}
+
+	savedModProbeCmd := modProbeCmd
+	savedSysModuleDir := sysModuleDir
+
+	// XXX: override (fake the modprobe command failing)
+	modProbeCmd = "false"
+	sysModuleDir = filepath.Join(dir, "sys/module")
+
+	defer func() {
+		modProbeCmd = savedModProbeCmd
+		sysModuleDir = savedSysModuleDir
+	}()
+
+	modPath := filepath.Join(sysModuleDir, "foo/parameters")
+	err = os.MkdirAll(modPath, testDirMode)
+	assert.NoError(err)
+
+	modParamFile := filepath.Join(modPath, "param1")
+
+	err = createEmptyFile(modParamFile)
+	assert.NoError(err)
+
+	// make file unreadable by non-root user
+	err = os.Chmod(modParamFile, 0000)
+	assert.NoError(err)
+
+	_, err = checkKernelModules(testData, nil)
+	assert.Error(err)
+}
+
+func TestCheckCheckKernelModulesInvalidFileContents(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	testData := map[string]kernelModule{
+		"foo": {
+			desc: "desc",
+			parameters: map[string]string{
+				"param1": "wibble",
+			},
+			required: true,
+		},
+	}
+
+	savedModProbeCmd := modProbeCmd
+	savedSysModuleDir := sysModuleDir
+
+	// XXX: override (fake the modprobe command failing)
+	modProbeCmd = "false"
+	sysModuleDir = filepath.Join(dir, "sys/module")
+
+	defer func() {
+		modProbeCmd = savedModProbeCmd
+		sysModuleDir = savedSysModuleDir
+	}()
+
+	modPath := filepath.Join(sysModuleDir, "foo/parameters")
+	err = os.MkdirAll(modPath, testDirMode)
+	assert.NoError(err)
+
+	modParamFile := filepath.Join(modPath, "param1")
+
+	err = createFile(modParamFile, "burp")
+	assert.NoError(err)
+
+	count, err := checkKernelModules(testData, nil)
+	assert.NoError(err)
+	assert.Equal(count, uint32(1))
+}
+
+func TestCheckCLIFunctionFail(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	_, config, err := makeRuntimeConfig(dir)
+	assert.NoError(err)
+
+	oldProcCPUInfo := procCPUInfo
+
+	// doesn't exist
+	procCPUInfo = filepath.Join(dir, "cpuinfo")
+
+	defer func() {
+		procCPUInfo = oldProcCPUInfo
+	}()
+
+	flagSet := &flag.FlagSet{}
+	ctx := createCLIContext(flagSet)
+	ctx.App.Name = "foo"
+	ctx.App.Metadata["runtimeConfig"] = config
+
+	fn, ok := kataCheckCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.Error(err)
+}
+
+func TestCheckKernelParamHandler(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedModProbeCmd := modProbeCmd
+	savedSysModuleDir := sysModuleDir
+
+	// XXX: override (fake the modprobe command failing)
+	modProbeCmd = "false"
+	sysModuleDir = filepath.Join(dir, "sys/module")
+
+	defer func() {
+		modProbeCmd = savedModProbeCmd
+		sysModuleDir = savedSysModuleDir
+	}()
+
+	handler := func(onVMM bool, fields logrus.Fields, msg string) bool {
+		param, ok := fields["parameter"].(string)
+		if !ok {
+			return false
+		}
+
+		if param == "param1" {
+			return true
+		}
+
+		// don't ignore the error
+		return false
+	}
+
+	testData1 := map[string]kernelModule{
+		"foo": {
+			desc:       "desc",
+			parameters: map[string]string{},
+			required:   true,
+		},
+		"bar": {
+			desc: "desc",
+			parameters: map[string]string{
+				"param1": "hello",
+				"param2": "world",
+			},
+			required: true,
+		},
+	}
+
+	checkKernelParamHandler(assert, testData1, testData1, handler, false, uint32(0))
+
+	testDataToCreate := map[string]kernelModule{
+		"foo": {
+			desc: "desc",
+			parameters: map[string]string{
+				"param1": "moo",
+			},
+			required: true,
+		},
+	}
+
+	testDataToExpect := map[string]kernelModule{
+		"foo": {
+			desc: "desc",
+			parameters: map[string]string{
+				"param1": "bar",
+			},
+			required: true,
+		},
+	}
+
+	// Expected and actual are different, but the handler should deal with
+	// the problem.
+	checkKernelParamHandler(assert, testDataToCreate, testDataToExpect, handler, false, uint32(0))
+
+	// Expected and actual are different, so with no handler we expect a
+	// single error (due to "param1"'s value being different)
+	checkKernelParamHandler(assert, testDataToCreate, testDataToExpect, nil, false, uint32(1))
+}
+
+func TestArchRequiredKernelModules(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	err = setCPUtype(config.HypervisorType)
+	assert.NoError(err)
+
+	if len(archRequiredKernelModules) == 0 {
+		// No modules to check
+		return
+	}
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedModProbeCmd := modProbeCmd
+	savedSysModuleDir := sysModuleDir
+
+	// XXX: override
+	sysModuleDir = filepath.Join(dir, "sys/module")
+	modProbeCmd = "false"
+
+	defer func() {
+		sysModuleDir = savedSysModuleDir
+		modProbeCmd = savedModProbeCmd
+	}()
+
+	// Running check with no modules
+	count, err := checkKernelModules(archRequiredKernelModules, nil)
+	assert.NoError(err)
+
+	// Test that count returned matches the # of modules with required set.
+	expectedCount := 0
+	for _, module := range archRequiredKernelModules {
+		if module.required {
+			expectedCount++
+		}
+	}
+
+	assert.EqualValues(count, expectedCount)
+}
+
+func TestCheckVersionConsistencyInComponents(t *testing.T) {
+	type testData struct {
+		proxyExist     bool
+		expectError    bool
+		shimVersion    string
+		proxyVersion   string
+		runtimeVersion string
+	}
+
+	data := []testData{
+		{
+			true,
+			true,
+			"kata-shim version 0.2.0-rc0-xxxxxxxxxxxxx",
+			"kata-proxy version 0.1.0-rc0-xxxxxxxxxxxxx",
+			"0.2.0-rc0",
+		},
+		{
+			true,
+			true,
+			"kata-shim version 0.1.0-rc0-xxxxxxxxxxxxx",
+			"kata-proxy version 0.2.0-rc0-xxxxxxxxxxxxx",
+			"0.2.0-rc0",
+		},
+		{
+			true,
+			true,
+			"kata-shim version 0.1.0-rc0-xxxxxxxxxxxxx",
+			"kata-proxy version 0.1.0-rc0-xxxxxxxxxxxxx",
+			"0.2.0-rc0",
+		},
+		{
+			true,
+			false,
+			"kata-shim version 0.2.0-rc0-xxxxxxxxxxxxx",
+			"kata-proxy version 0.2.0-rc0-xxxxxxxxxxxxx",
+			"0.2.0-rc0",
+		},
+		{
+			false,
+			true,
+			"kata-shim version 0.1.0-rc0-xxxxxxxxxxxxx",
+			"",
+			"0.2.0-rc0",
+		},
+		{
+			false,
+			false,
+			"kata-shim version 0.2.0-rc0-xxxxxxxxxxxxx",
+			"",
+			"0.2.0-rc0",
+		},
+	}
+
+	origVersion := version
+	for _, d := range data {
+		tmpdir, err := ioutil.TempDir("", "")
+		assert.NoError(t, err)
+		defer os.RemoveAll(tmpdir)
+
+		testShimVersion = d.shimVersion
+		if d.proxyExist {
+			testProxyVersion = d.proxyVersion
+		}
+		_, config, err := makeRuntimeConfig(tmpdir)
+		assert.NoError(t, err)
+		if !d.proxyExist {
+			config.ProxyType = vc.NoProxyType
+		}
+		version = d.runtimeVersion
+		defer func() {
+			version = origVersion
+		}()
+
+		err = checkVersionConsistencyInComponents(config)
+		if d.expectError {
+			assert.Error(t, err, fmt.Sprintf("%+v", d))
+			continue
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("%+v", d))
+		}
+	}
+}

src/runtime/cli/kata-env.go

@@ -0,0 +1,517 @@
+// Copyright (c) 2017-2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	runtim "runtime"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	vcUtils "github.com/kata-containers/runtime/virtcontainers/utils"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/urfave/cli"
+)
+
+// Semantic version for the output of the command.
+//
+// XXX: Increment for every change to the output format
+// (meaning any change to the EnvInfo type).
+const formatVersion = "1.0.24"
+
+// MetaInfo stores information on the format of the output itself
+type MetaInfo struct {
+	// output format version
+	Version string
+}
+
+// KernelInfo stores kernel details
+type KernelInfo struct {
+	Path       string
+	Parameters string
+}
+
+// InitrdInfo stores initrd image details
+type InitrdInfo struct {
+	Path string
+}
+
+// ImageInfo stores root filesystem image details
+type ImageInfo struct {
+	Path string
+}
+
+// CPUInfo stores host CPU details
+type CPUInfo struct {
+	Vendor string
+	Model  string
+}
+
+// RuntimeConfigInfo stores runtime config details.
+type RuntimeConfigInfo struct {
+	Path string
+}
+
+// RuntimeInfo stores runtime details.
+type RuntimeInfo struct {
+	Version             RuntimeVersionInfo
+	Config              RuntimeConfigInfo
+	Debug               bool
+	Trace               bool
+	DisableGuestSeccomp bool
+	DisableNewNetNs     bool
+	SandboxCgroupOnly   bool
+	Experimental        []exp.Feature
+	Path                string
+}
+
+type VersionInfo struct {
+	Semver string
+	Major  uint64
+	Minor  uint64
+	Patch  uint64
+	Commit string
+}
+
+// RuntimeVersionInfo stores details of the runtime version
+type RuntimeVersionInfo struct {
+	Version VersionInfo
+	OCI     string
+}
+
+// HypervisorInfo stores hypervisor details
+type HypervisorInfo struct {
+	MachineType          string
+	Version              string
+	Path                 string
+	BlockDeviceDriver    string
+	EntropySource        string
+	SharedFS             string
+	VirtioFSDaemon       string
+	Msize9p              uint32
+	MemorySlots          uint32
+	PCIeRootPort         uint32
+	HotplugVFIOOnRootBus bool
+	Debug                bool
+	UseVSock             bool
+}
+
+// ProxyInfo stores proxy details
+type ProxyInfo struct {
+	Type    string
+	Version VersionInfo
+	Path    string
+	Debug   bool
+}
+
+// ShimInfo stores shim details
+type ShimInfo struct {
+	Type    string
+	Version VersionInfo
+	Path    string
+	Debug   bool
+}
+
+// AgentInfo stores agent details
+type AgentInfo struct {
+	Type      string
+	Debug     bool
+	Trace     bool
+	TraceMode string
+	TraceType string
+}
+
+// DistroInfo stores host operating system distribution details.
+type DistroInfo struct {
+	Name    string
+	Version string
+}
+
+// HostInfo stores host details
+type HostInfo struct {
+	Kernel             string
+	Architecture       string
+	Distro             DistroInfo
+	CPU                CPUInfo
+	VMContainerCapable bool
+	SupportVSocks      bool
+}
+
+// NetmonInfo stores netmon details
+type NetmonInfo struct {
+	Version VersionInfo
+	Path    string
+	Debug   bool
+	Enable  bool
+}
+
+// EnvInfo collects all information that will be displayed by the
+// env command.
+//
+// XXX: Any changes must be coupled with a change to formatVersion.
+type EnvInfo struct {
+	Meta       MetaInfo
+	Runtime    RuntimeInfo
+	Hypervisor HypervisorInfo
+	Image      ImageInfo
+	Kernel     KernelInfo
+	Initrd     InitrdInfo
+	Proxy      ProxyInfo
+	Shim       ShimInfo
+	Agent      AgentInfo
+	Host       HostInfo
+	Netmon     NetmonInfo
+}
+
+func getMetaInfo() MetaInfo {
+	return MetaInfo{
+		Version: formatVersion,
+	}
+}
+
+func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo {
+	runtimeVersionInfo := constructVersionInfo(version)
+	runtimeVersionInfo.Commit = commit
+
+	runtimeVersion := RuntimeVersionInfo{
+		Version: runtimeVersionInfo,
+		OCI:     specs.Version,
+	}
+
+	runtimeConfig := RuntimeConfigInfo{
+		Path: configFile,
+	}
+
+	runtimePath, _ := os.Executable()
+
+	return RuntimeInfo{
+		Debug:               config.Debug,
+		Trace:               config.Trace,
+		Version:             runtimeVersion,
+		Config:              runtimeConfig,
+		Path:                runtimePath,
+		DisableNewNetNs:     config.DisableNewNetNs,
+		SandboxCgroupOnly:   config.SandboxCgroupOnly,
+		Experimental:        config.Experimental,
+		DisableGuestSeccomp: config.DisableGuestSeccomp,
+	}
+}
+
+func getHostInfo() (HostInfo, error) {
+	hostKernelVersion, err := getKernelVersion()
+	if err != nil {
+		return HostInfo{}, err
+	}
+
+	hostDistroName, hostDistroVersion, err := getDistroDetails()
+	if err != nil {
+		return HostInfo{}, err
+	}
+
+	cpuVendor, cpuModel, err := getCPUDetails()
+	if err != nil {
+		return HostInfo{}, err
+	}
+
+	hostVMContainerCapable := true
+	if runtim.GOARCH == "ppc64le" {
+		hostVMContainerCapable = false
+	}
+
+	details := vmContainerCapableDetails{
+		cpuInfoFile:           procCPUInfo,
+		requiredCPUFlags:      archRequiredCPUFlags,
+		requiredCPUAttribs:    archRequiredCPUAttribs,
+		requiredKernelModules: archRequiredKernelModules,
+	}
+
+	if err = hostIsVMContainerCapable(details); err != nil {
+		hostVMContainerCapable = false
+	}
+
+	hostDistro := DistroInfo{
+		Name:    hostDistroName,
+		Version: hostDistroVersion,
+	}
+
+	hostCPU := CPUInfo{
+		Vendor: cpuVendor,
+		Model:  cpuModel,
+	}
+
+	host := HostInfo{
+		Kernel:             hostKernelVersion,
+		Architecture:       arch,
+		Distro:             hostDistro,
+		CPU:                hostCPU,
+		VMContainerCapable: hostVMContainerCapable,
+		SupportVSocks:      vcUtils.SupportsVsocks(),
+	}
+
+	return host, nil
+}
+
+func getProxyInfo(config oci.RuntimeConfig) ProxyInfo {
+	if config.ProxyType == vc.NoProxyType {
+		return ProxyInfo{Type: string(config.ProxyType)}
+	}
+
+	proxyConfig := config.ProxyConfig
+
+	var proxyVersionInfo VersionInfo
+	if version, err := getCommandVersion(proxyConfig.Path); err != nil {
+		proxyVersionInfo = unknownVersionInfo
+	} else {
+		proxyVersionInfo = constructVersionInfo(version)
+	}
+
+	proxy := ProxyInfo{
+		Type:    string(config.ProxyType),
+		Version: proxyVersionInfo,
+		Path:    proxyConfig.Path,
+		Debug:   proxyConfig.Debug,
+	}
+
+	return proxy
+}
+
+func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo {
+	netmonConfig := config.NetmonConfig
+
+	var netmonVersionInfo VersionInfo
+	if version, err := getCommandVersion(netmonConfig.Path); err != nil {
+		netmonVersionInfo = unknownVersionInfo
+	} else {
+		netmonVersionInfo = constructVersionInfo(version)
+	}
+
+	netmon := NetmonInfo{
+		Version: netmonVersionInfo,
+		Path:    netmonConfig.Path,
+		Debug:   netmonConfig.Debug,
+		Enable:  netmonConfig.Enable,
+	}
+
+	return netmon
+}
+
+func getCommandVersion(cmd string) (string, error) {
+	return katautils.RunCommand([]string{cmd, "--version"})
+}
+
+func getShimInfo(config oci.RuntimeConfig) (ShimInfo, error) {
+	shimConfig, ok := config.ShimConfig.(vc.ShimConfig)
+	if !ok {
+		return ShimInfo{}, errors.New("cannot determine shim config")
+	}
+
+	shimPath := shimConfig.Path
+
+	var shimVersionInfo VersionInfo
+	if version, err := getCommandVersion(shimConfig.Path); err != nil {
+		shimVersionInfo = unknownVersionInfo
+	} else {
+		shimVersionInfo = constructVersionInfo(version)
+	}
+
+	shim := ShimInfo{
+		Type:    string(config.ShimType),
+		Version: shimVersionInfo,
+		Path:    shimPath,
+		Debug:   shimConfig.Debug,
+	}
+
+	return shim, nil
+}
+
+func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
+	agent := AgentInfo{
+		Type: string(config.AgentType),
+	}
+
+	switch config.AgentType {
+	case vc.KataContainersAgent:
+		agentConfig, ok := config.AgentConfig.(vc.KataAgentConfig)
+		if !ok {
+			return AgentInfo{}, errors.New("cannot determine Kata agent config")
+		}
+		agent.Debug = agentConfig.Debug
+		agent.Trace = agentConfig.Trace
+		agent.TraceMode = agentConfig.TraceMode
+		agent.TraceType = agentConfig.TraceType
+	default:
+		// Nothing useful to report for the other agent types
+	}
+
+	return agent, nil
+}
+
+func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
+	hypervisorPath := config.HypervisorConfig.HypervisorPath
+
+	version, err := getCommandVersion(hypervisorPath)
+	if err != nil {
+		version = unknown
+	}
+
+	return HypervisorInfo{
+		Debug:             config.HypervisorConfig.Debug,
+		MachineType:       config.HypervisorConfig.HypervisorMachineType,
+		Version:           version,
+		Path:              hypervisorPath,
+		BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver,
+		Msize9p:           config.HypervisorConfig.Msize9p,
+		UseVSock:          config.HypervisorConfig.UseVSock,
+		MemorySlots:       config.HypervisorConfig.MemSlots,
+		EntropySource:     config.HypervisorConfig.EntropySource,
+		SharedFS:          config.HypervisorConfig.SharedFS,
+		VirtioFSDaemon:    config.HypervisorConfig.VirtioFSDaemon,
+
+		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
+		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
+	}
+}
+
+func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) {
+	err = setCPUtype(config.HypervisorType)
+	if err != nil {
+		return EnvInfo{}, err
+	}
+
+	meta := getMetaInfo()
+
+	runtime := getRuntimeInfo(configFile, config)
+
+	host, err := getHostInfo()
+	if err != nil {
+		return EnvInfo{}, err
+	}
+
+	proxy := getProxyInfo(config)
+
+	netmon := getNetmonInfo(config)
+
+	shim, err := getShimInfo(config)
+	if err != nil {
+		return EnvInfo{}, err
+	}
+
+	agent, err := getAgentInfo(config)
+	if err != nil {
+		return EnvInfo{}, err
+	}
+
+	hypervisor := getHypervisorInfo(config)
+
+	image := ImageInfo{
+		Path: config.HypervisorConfig.ImagePath,
+	}
+
+	kernel := KernelInfo{
+		Path:       config.HypervisorConfig.KernelPath,
+		Parameters: strings.Join(vc.SerializeParams(config.HypervisorConfig.KernelParams, "="), " "),
+	}
+
+	initrd := InitrdInfo{
+		Path: config.HypervisorConfig.InitrdPath,
+	}
+
+	env = EnvInfo{
+		Meta:       meta,
+		Runtime:    runtime,
+		Hypervisor: hypervisor,
+		Image:      image,
+		Kernel:     kernel,
+		Initrd:     initrd,
+		Proxy:      proxy,
+		Shim:       shim,
+		Agent:      agent,
+		Host:       host,
+		Netmon:     netmon,
+	}
+
+	return env, nil
+}
+
+func handleSettings(file *os.File, c *cli.Context) error {
+	if file == nil {
+		return errors.New("Invalid output file specified")
+	}
+
+	configFile, ok := c.App.Metadata["configFile"].(string)
+	if !ok {
+		return errors.New("cannot determine config file")
+	}
+
+	runtimeConfig, ok := c.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+	if !ok {
+		return errors.New("cannot determine runtime config")
+	}
+
+	env, err := getEnvInfo(configFile, runtimeConfig)
+	if err != nil {
+		return err
+	}
+
+	if c.Bool("json") {
+		return writeJSONSettings(env, file)
+	}
+
+	return writeTOMLSettings(env, file)
+}
+
+func writeTOMLSettings(env EnvInfo, file *os.File) error {
+	encoder := toml.NewEncoder(file)
+
+	err := encoder.Encode(env)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func writeJSONSettings(env EnvInfo, file *os.File) error {
+	encoder := json.NewEncoder(file)
+
+	// Make it more human readable
+	encoder.SetIndent("", "  ")
+
+	err := encoder.Encode(env)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+var kataEnvCLICommand = cli.Command{
+	Name:  envCmd,
+	Usage: "display settings. Default to TOML",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "json",
+			Usage: "Format output as JSON",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, _ := katautils.Trace(ctx, "kata-env")
+		defer span.Finish()
+
+		return handleSettings(defaultOutputFile, context)
+	},
+}

src/runtime/cli/kata-env_amd64_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func getExpectedHostDetails(tmpdir string) (HostInfo, error) {
+	expectedVendor := "moi"
+	expectedModel := "awesome XI"
+	expectedVMContainerCapable := false
+	return genericGetExpectedHostDetails(tmpdir, expectedVendor, expectedModel, expectedVMContainerCapable)
+}
+
+func TestEnvGetEnvInfoSetsCPUType(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	savedArchRequiredCPUFlags := archRequiredCPUFlags
+	savedArchRequiredCPUAttribs := archRequiredCPUAttribs
+	savedArchRequiredKernelModules := archRequiredKernelModules
+
+	defer func() {
+		archRequiredCPUFlags = savedArchRequiredCPUFlags
+		archRequiredCPUAttribs = savedArchRequiredCPUAttribs
+		archRequiredKernelModules = savedArchRequiredKernelModules
+	}()
+
+	archRequiredCPUFlags = map[string]string{}
+	archRequiredCPUAttribs = map[string]string{}
+	archRequiredKernelModules = map[string]kernelModule{}
+
+	configFile, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	expectedEnv, err := getExpectedSettings(config, tmpdir, configFile)
+	assert.NoError(err)
+
+	env, err := getEnvInfo(configFile, config)
+	assert.NoError(err)
+
+	assert.Equal(expectedEnv, env)
+
+	assert.NotEmpty(archRequiredCPUFlags)
+	assert.NotEmpty(archRequiredCPUAttribs)
+	assert.NotEmpty(archRequiredKernelModules)
+
+	assert.Equal(archRequiredCPUFlags["vmx"], "Virtualization support")
+
+	_, ok := archRequiredKernelModules["kvm"]
+	assert.True(ok)
+}

src/runtime/cli/kata-env_arm64_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2018 ARM Limited
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"testing"
+)
+
+func getExpectedHostDetails(tmpdir string) (HostInfo, error) {
+	expectedVendor := "0x41"
+	expectedModel := "8"
+	expectedVMContainerCapable := true
+	return genericGetExpectedHostDetails(tmpdir, expectedVendor, expectedModel, expectedVMContainerCapable)
+}
+
+func TestEnvGetEnvInfoSetsCPUType(t *testing.T) {
+	testEnvGetEnvInfoSetsCPUTypeGeneric(t)
+}

src/runtime/cli/kata-env_generic_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// +build arm64 ppc64le
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func testEnvGetEnvInfoSetsCPUTypeGeneric(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	savedArchRequiredCPUFlags := archRequiredCPUFlags
+	savedArchRequiredCPUAttribs := archRequiredCPUAttribs
+	savedArchRequiredKernelModules := archRequiredKernelModules
+
+	defer func() {
+		archRequiredCPUFlags = savedArchRequiredCPUFlags
+		archRequiredCPUAttribs = savedArchRequiredCPUAttribs
+		archRequiredKernelModules = savedArchRequiredKernelModules
+	}()
+
+	assert.Empty(archRequiredCPUFlags)
+	assert.Empty(archRequiredCPUAttribs)
+	assert.NotEmpty(archRequiredKernelModules)
+
+	configFile, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(err)
+
+	expectedEnv, err := getExpectedSettings(config, tmpdir, configFile)
+	assert.NoError(err)
+
+	env, err := getEnvInfo(configFile, config)
+	assert.NoError(err)
+
+	assert.Equal(expectedEnv, env)
+
+	assert.Equal(archRequiredCPUFlags, savedArchRequiredCPUFlags)
+	assert.Equal(archRequiredCPUAttribs, savedArchRequiredCPUAttribs)
+	assert.Equal(archRequiredKernelModules, savedArchRequiredKernelModules)
+}

src/runtime/cli/kata-env_ppc64le_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import "testing"
+
+func getExpectedHostDetails(tmpdir string) (HostInfo, error) {
+	expectedVendor := ""
+	expectedModel := "POWER8"
+	expectedVMContainerCapable := false
+	return genericGetExpectedHostDetails(tmpdir, expectedVendor, expectedModel, expectedVMContainerCapable)
+}
+
+func TestEnvGetEnvInfoSetsCPUType(t *testing.T) {
+	testEnvGetEnvInfoSetsCPUTypeGeneric(t)
+}

src/runtime/cli/kata-env_s390x_test.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	vcUtils "github.com/kata-containers/runtime/virtcontainers/utils"
+	"path/filepath"
+	goruntime "runtime"
+)
+
+func getExpectedHostDetails(tmpdir string) (HostInfo, error) {
+	type filesToCreate struct {
+		file     string
+		contents string
+	}
+
+	const expectedKernelVersion = "99.1"
+	const expectedArch = goruntime.GOARCH
+
+	expectedDistro := DistroInfo{
+		Name:    "Foo",
+		Version: "42",
+	}
+
+	expectedCPU := CPUInfo{
+		Vendor: "moi",
+		Model:  "awesome XI",
+	}
+
+	expectedHostDetails := HostInfo{
+		Kernel:             expectedKernelVersion,
+		Architecture:       expectedArch,
+		Distro:             expectedDistro,
+		CPU:                expectedCPU,
+		VMContainerCapable: true,
+		SupportVSocks:      vcUtils.SupportsVsocks(),
+	}
+
+	testProcCPUInfo := filepath.Join(tmpdir, "cpuinfo")
+	testOSRelease := filepath.Join(tmpdir, "os-release")
+
+	// XXX: This file is *NOT* created by this function on purpose
+	// (to ensure the only file checked by the tests is
+	// testOSRelease). osReleaseClr handling is tested in
+	// utils_test.go.
+	testOSReleaseClr := filepath.Join(tmpdir, "os-release-clr")
+
+	testProcVersion := filepath.Join(tmpdir, "proc-version")
+
+	// override
+	procVersion = testProcVersion
+	osRelease = testOSRelease
+	osReleaseClr = testOSReleaseClr
+	procCPUInfo = testProcCPUInfo
+
+	procVersionContents := fmt.Sprintf("Linux version %s a b c",
+		expectedKernelVersion)
+
+	osReleaseContents := fmt.Sprintf(`
+NAME="%s"
+VERSION_ID="%s"
+`, expectedDistro.Name, expectedDistro.Version)
+
+	procCPUInfoContents := fmt.Sprintf(`
+%s	: %s
+processor 0: version = 00,  identification = 3929E7,  %s = %s
+`,
+		archCPUVendorField,
+		expectedCPU.Vendor,
+		archCPUModelField,
+		expectedCPU.Model)
+
+	data := []filesToCreate{
+		{procVersion, procVersionContents},
+		{osRelease, osReleaseContents},
+		{procCPUInfo, procCPUInfoContents},
+	}
+
+	for _, d := range data {
+		err := createFile(d.file, d.contents)
+		if err != nil {
+			return HostInfo{}, err
+		}
+	}
+
+	return expectedHostDetails, nil
+}

src/runtime/cli/kata-overhead.go

@@ -0,0 +1,136 @@
+// +build cgo,linux
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var kataOverheadCLICommand = cli.Command{
+	Name:  "kata-overhead",
+	Usage: "provides kata overhead at sandbox level",
+	ArgsUsage: `<sandbox-id> [sandbox-id...]
+
+   <sandbox-id> is your name for the instance of the sandbox.`,
+
+	Description: `The kata-overhead command shows the overhead of a running Kata sandbox. Overhead 
+       is calculated as the sum of pod resource utilization as measured on host cgroup minus the total
+       container usage measured inside the Kata guest for each container's cgroup.`,
+
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		args := context.Args()
+		if !args.Present() {
+			return fmt.Errorf("Missing container ID, should at least provide one")
+		}
+
+		for _, cID := range []string(args) {
+			if err := overhead(ctx, cID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
+func overhead(ctx context.Context, containerID string) error {
+	span, _ := katautils.Trace(ctx, "overhead")
+	defer span.Finish()
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+	span.SetTag("sandbox", sandboxID)
+
+	if status.State.State == types.StateStopped {
+		return fmt.Errorf("container with id %s is not running", status.ID)
+	}
+
+	initTime := time.Now().UnixNano()
+
+	initialSandboxStats, initialContainerStats, err := vci.StatsSandbox(ctx, sandboxID)
+	if err != nil {
+		return err
+	}
+
+	hostInitCPU := initialSandboxStats.CgroupStats.CPUStats.CPUUsage.TotalUsage
+	guestInitCPU := uint64(0)
+	for _, cs := range initialContainerStats {
+		guestInitCPU += cs.CgroupStats.CPUStats.CPUUsage.TotalUsage
+	}
+
+	// Wait for 1 second to calculate CPU usage
+	time.Sleep(time.Second * 1)
+	finishtTime := time.Now().UnixNano()
+
+	finishSandboxStats, finishContainersStats, err := vci.StatsSandbox(ctx, sandboxID)
+	if err != nil {
+		return err
+	}
+
+	hostFinalCPU := finishSandboxStats.CgroupStats.CPUStats.CPUUsage.TotalUsage
+	guestFinalCPU := uint64(0)
+	for _, cs := range finishContainersStats {
+		guestFinalCPU += cs.CgroupStats.CPUStats.CPUUsage.TotalUsage
+	}
+
+	var guestMemoryUsage uint64
+	for _, cs := range finishContainersStats {
+		guestMemoryUsage += cs.CgroupStats.MemoryStats.Usage.Usage
+	}
+
+	hostMemoryUsage := finishSandboxStats.CgroupStats.MemoryStats.Usage.Usage
+	deltaTime := finishtTime - initTime
+
+	cpuUsageGuest := float64(guestFinalCPU-guestInitCPU) / float64(deltaTime) * 100
+	cpuUsageHost := float64(hostFinalCPU-hostInitCPU) / float64(deltaTime) * 100
+
+	fmt.Printf("Sandbox overhead for container: %s\n", containerID)
+	fmt.Printf("cpu_overhead=%f\n", cpuUsageHost-cpuUsageGuest)
+	fmt.Printf("memory_overhead_bytes=%d\n\n", hostMemoryUsage-guestMemoryUsage)
+	fmt.Printf(" --CPU details--\n")
+	fmt.Printf("cpu_host=%f\n", cpuUsageHost)
+	fmt.Printf("\tcpu_host_init=%d\n", hostInitCPU)
+	fmt.Printf("\tcpu_host_final=%d\n", hostFinalCPU)
+	fmt.Printf("cpu_guest=%f\n", cpuUsageGuest)
+	fmt.Printf("\tcpu_guest_init=%d\n", guestInitCPU)
+	fmt.Printf("\tcpu_guest_final=%d\n", guestFinalCPU)
+	fmt.Printf("Number of available vCPUs=%d\n", finishSandboxStats.Cpus)
+	fmt.Printf(" --Memory details--\n")
+	fmt.Printf("memory_host_bytes=%d\n", hostMemoryUsage)
+	fmt.Printf("memory_guest_bytes=%d\n\n", guestMemoryUsage)
+
+	return nil
+}

src/runtime/cli/kill.go

@@ -0,0 +1,193 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"syscall"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var killCLICommand = cli.Command{
+	Name:  "kill",
+	Usage: "Kill sends signals to the container's init process",
+	ArgsUsage: `<container-id> [signal]
+
+   <container-id> is the name for the instance of the container
+   [signal] is the signal to be sent to the init process (default: SIGTERM)
+
+EXAMPLE:
+   If the container id is "ubuntu01" the following will send a "KILL" signal
+   to the init process of the "ubuntu01" container:
+	 
+       # ` + name + ` kill ubuntu01 KILL`,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "all, a",
+			Usage: "send the specified signal to all processes inside the container",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		args := context.Args()
+		if !args.Present() {
+			return fmt.Errorf("Missing container ID")
+		}
+
+		// If signal is provided, it has to be the second argument.
+		signal := args.Get(1)
+		if signal == "" {
+			signal = "SIGTERM"
+		}
+
+		return kill(ctx, args.First(), signal, context.Bool("all"))
+	},
+}
+
+var signalList = map[string]syscall.Signal{
+	"SIGABRT":   syscall.SIGABRT,
+	"SIGALRM":   syscall.SIGALRM,
+	"SIGBUS":    syscall.SIGBUS,
+	"SIGCHLD":   syscall.SIGCHLD,
+	"SIGCLD":    syscall.SIGCLD,
+	"SIGCONT":   syscall.SIGCONT,
+	"SIGFPE":    syscall.SIGFPE,
+	"SIGHUP":    syscall.SIGHUP,
+	"SIGILL":    syscall.SIGILL,
+	"SIGINT":    syscall.SIGINT,
+	"SIGIO":     syscall.SIGIO,
+	"SIGIOT":    syscall.SIGIOT,
+	"SIGKILL":   syscall.SIGKILL,
+	"SIGPIPE":   syscall.SIGPIPE,
+	"SIGPOLL":   syscall.SIGPOLL,
+	"SIGPROF":   syscall.SIGPROF,
+	"SIGPWR":    syscall.SIGPWR,
+	"SIGQUIT":   syscall.SIGQUIT,
+	"SIGSEGV":   syscall.SIGSEGV,
+	"SIGSTKFLT": syscall.SIGSTKFLT,
+	"SIGSTOP":   syscall.SIGSTOP,
+	"SIGSYS":    syscall.SIGSYS,
+	"SIGTERM":   syscall.SIGTERM,
+	"SIGTRAP":   syscall.SIGTRAP,
+	"SIGTSTP":   syscall.SIGTSTP,
+	"SIGTTIN":   syscall.SIGTTIN,
+	"SIGTTOU":   syscall.SIGTTOU,
+	"SIGUNUSED": syscall.SIGUNUSED,
+	"SIGURG":    syscall.SIGURG,
+	"SIGUSR1":   syscall.SIGUSR1,
+	"SIGUSR2":   syscall.SIGUSR2,
+	"SIGVTALRM": syscall.SIGVTALRM,
+	"SIGWINCH":  syscall.SIGWINCH,
+	"SIGXCPU":   syscall.SIGXCPU,
+	"SIGXFSZ":   syscall.SIGXFSZ,
+}
+
+func kill(ctx context.Context, containerID, signal string, all bool) error {
+	span, _ := katautils.Trace(ctx, "kill")
+	defer span.Finish()
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+
+	if err != nil {
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	span.SetTag("container", containerID)
+	span.SetTag("sandbox", sandboxID)
+
+	setExternalLoggers(ctx, kataLog)
+
+	signum, err := processSignal(signal)
+	if err != nil {
+		return err
+	}
+
+	kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill")
+
+	// container MUST be created, running or paused
+	if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused {
+		if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil {
+			return err
+		}
+	} else if !all {
+		return fmt.Errorf("container not running")
+	}
+
+	if signum != syscall.SIGKILL && signum != syscall.SIGTERM {
+		return nil
+	}
+
+	containerType, err := oci.GetContainerType(status.Annotations)
+	if err != nil {
+		return err
+	}
+
+	switch containerType {
+	case vc.PodSandbox:
+		_, err = vci.StopSandbox(ctx, sandboxID, signum == syscall.SIGKILL)
+	case vc.PodContainer:
+		_, err = vci.StopContainer(ctx, sandboxID, containerID)
+	default:
+		return fmt.Errorf("Invalid container type found")
+	}
+
+	return err
+}
+
+func processSignal(signal string) (syscall.Signal, error) {
+	signum, signalOk := signalList[signal]
+	if signalOk {
+		return signum, nil
+	}
+
+	// Support for short name signals (INT)
+	signum, signalOk = signalList["SIG"+signal]
+	if signalOk {
+		return signum, nil
+	}
+
+	// Support for numeric signals
+	s, err := strconv.Atoi(signal)
+	if err != nil {
+		return 0, fmt.Errorf("Failed to convert signal %s to int", signal)
+	}
+
+	signum = syscall.Signal(s)
+	// Check whether signal is valid or not
+	for _, sig := range signalList {
+		if sig == signum {
+			// signal is a valid signal
+			return signum, nil
+		}
+	}
+
+	return 0, fmt.Errorf("Signal %s is not supported", signal)
+}

src/runtime/cli/kill_test.go

@@ -0,0 +1,431 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"syscall"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+)
+
+var (
+	testKillContainerFuncReturnNil = func(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error {
+		return nil
+	}
+
+	testStopContainerFuncReturnNil = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return &vcmock.Container{}, nil
+	}
+
+	testStopSandboxFuncReturnNil = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
+		return &vcmock.Sandbox{}, nil
+	}
+)
+
+func TestProcessSignal(t *testing.T) {
+	tests := []struct {
+		signal string
+		valid  bool
+		signum syscall.Signal
+	}{
+		{"SIGDCKBY", false, 0}, //invalid signal
+		{"DCKBY", false, 0},    //invalid signal
+		{"99999", false, 0},    //invalid signal
+		{"SIGTERM", true, syscall.SIGTERM},
+		{"TERM", true, syscall.SIGTERM},
+		{"15", true, syscall.SIGTERM},
+	}
+
+	for _, test := range tests {
+		signum, err := processSignal(test.signal)
+		if signum != test.signum {
+			t.Fatalf("signal received: %d expected signal: %d\n", signum, test.signum)
+		}
+		if test.valid && err != nil {
+			t.Fatalf("signal %s is a valid but a error was received: %s\n", test.signal, err)
+		}
+		if !test.valid && err == nil {
+			t.Fatalf("signal %s is not a valid signal and no error was reported\n", test.signal)
+		}
+	}
+}
+
+func testKillCLIFunctionTerminationSignalSuccessful(t *testing.T, sig string) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+	testingImpl.StopContainerFunc = testStopContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StopContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID, sig})
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+
+	annotations = map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &specs.Spec{}), nil
+	}
+
+	testingImpl.StopContainerFunc = nil
+	testingImpl.StopSandboxFunc = testStopSandboxFuncReturnNil
+	defer func() {
+		testingImpl.StopSandboxFunc = nil
+	}()
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+}
+
+func TestKillCLIFunctionSigkillSuccessful(t *testing.T) {
+	testKillCLIFunctionTerminationSignalSuccessful(t, "SIGKILL")
+}
+
+func TestKillCLIFunctionSigtermSuccessful(t *testing.T) {
+	testKillCLIFunctionTerminationSignalSuccessful(t, "SIGTERM")
+}
+
+func TestKillCLIFunctionNotTerminationSignalSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID, "SIGUSR1"})
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+}
+
+func TestKillCLIFunctionNoSignalSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+	testingImpl.StopContainerFunc = testStopContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StopContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+
+	annotations = map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &specs.Spec{}), nil
+	}
+
+	testingImpl.StopContainerFunc = nil
+	testingImpl.StopSandboxFunc = testStopSandboxFuncReturnNil
+	defer func() {
+		testingImpl.StopSandboxFunc = nil
+	}()
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+}
+
+func TestKillCLIFunctionEnableAllSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	annotations := map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+	}
+
+	testingImpl.KillContainerFunc = func(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error {
+		if !all {
+			return fmt.Errorf("Expecting -all flag = true, Got false")
+		}
+
+		return nil
+	}
+	testingImpl.StopContainerFunc = testStopContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StopContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Bool("all", true, "")
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+
+	annotations = map[string]string{
+		vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, annotations, &specs.Spec{}), nil
+	}
+
+	testingImpl.StopContainerFunc = nil
+	testingImpl.StopSandboxFunc = testStopSandboxFuncReturnNil
+	defer func() {
+		testingImpl.StopSandboxFunc = nil
+	}()
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+}
+
+func TestKillCLIFunctionNoContainerIDFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	set := flag.NewFlagSet("", 0)
+
+	execCLICommandFunc(assert, killCLICommand, set, true)
+}
+
+func TestKillCLIFunctionContainerNotExistFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{}, nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, killCLICommand, set, true)
+}
+
+func TestKillCLIFunctionInvalidSignalFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID, "SIGINVALID"})
+
+	execCLICommandFunc(assert, killCLICommand, set, true)
+}
+
+func TestKillCLIFunctionStatePausedSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StatePaused,
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+	testingImpl.StopContainerFunc = testStopContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state,
+			map[string]string{string(vcAnnotations.ContainerTypeKey): string(vc.PodContainer)}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StopContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+}
+
+func TestKillCLIFunctionInvalidStateStoppedFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateStopped,
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, killCLICommand, set, true)
+}
+
+func TestKillCLIFunctionKillContainerFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, killCLICommand, set, true)
+}
+
+func TestKillCLIFunctionInvalidStateStoppedAllSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateStopped,
+	}
+
+	testingImpl.KillContainerFunc = testKillContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.KillContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	var all bool
+	set.BoolVar(&all, "all", false, "")
+	set.Parse([]string{"-all", testContainerID, "10"})
+
+	execCLICommandFunc(assert, killCLICommand, set, false)
+}

src/runtime/cli/list.go

@@ -0,0 +1,394 @@
+// Copyright (c) 2014,2015,2016,2017 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+	"time"
+
+	"github.com/urfave/cli"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	oci "github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+)
+
+const formatOptions = `table or json`
+
+// containerState represents the platform agnostic pieces relating to a
+// running container's status and state
+type containerState struct {
+	// Version is the OCI version for the container
+	Version string `json:"ociVersion"`
+	// ID is the container ID
+	ID string `json:"id"`
+	// InitProcessPid is the init process id in the parent namespace
+	InitProcessPid int `json:"pid"`
+	// Status is the current status of the container, running, paused, ...
+	Status string `json:"status"`
+	// Bundle is the path on the filesystem to the bundle
+	Bundle string `json:"bundle"`
+	// Rootfs is a path to a directory containing the container's root filesystem.
+	Rootfs string `json:"rootfs"`
+	// Created is the unix timestamp for the creation time of the container in UTC
+	Created time.Time `json:"created"`
+	// Annotations is the user defined annotations added to the config.
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// The owner of the state directory (the owner of the container).
+	Owner string `json:"owner"`
+}
+
+type asset struct {
+	Path   string `json:"path"`
+	Custom bool   `json:"bool"`
+}
+
+// hypervisorDetails stores details of the hypervisor used to host
+// the container
+type hypervisorDetails struct {
+	HypervisorAsset asset `json:"hypervisorAsset"`
+	ImageAsset      asset `json:"imageAsset"`
+	KernelAsset     asset `json:"kernelAsset"`
+}
+
+// fullContainerState specifies the core state plus the hypervisor
+// details
+type fullContainerState struct {
+	containerState
+	CurrentHypervisorDetails hypervisorDetails `json:"currentHypervisor"`
+	LatestHypervisorDetails  hypervisorDetails `json:"latestHypervisor"`
+	StaleAssets              []string
+}
+
+type formatState interface {
+	Write(state []fullContainerState, showAll bool, file *os.File) error
+}
+
+type formatJSON struct{}
+type formatIDList struct{}
+type formatTabular struct{}
+
+var listCLICommand = cli.Command{
+	Name:  "list",
+	Usage: "lists containers started by " + name + " with the given root",
+	ArgsUsage: `
+
+Where the given root is specified via the global option "--root"
+(default: "` + defaultRootDirectory + `").
+
+EXAMPLE 1:
+To list containers created via the default "--root":
+       # ` + name + ` list
+
+EXAMPLE 2:
+To list containers created using a non-default value for "--root":
+       # ` + name + ` --root value list`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "format, f",
+			Value: "table",
+			Usage: `select one of: ` + formatOptions,
+		},
+		cli.BoolFlag{
+			Name:  "quiet, q",
+			Usage: "display only container IDs",
+		},
+		cli.BoolFlag{
+			Name:  "kata-all",
+			Usage: "display all available " + project + " information",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, ctx := katautils.Trace(ctx, "list")
+		defer span.Finish()
+
+		s, err := getContainers(ctx, context)
+		if err != nil {
+			return err
+		}
+
+		file := defaultOutputFile
+		showAll := context.Bool("kata-all")
+
+		var fs formatState = formatIDList{}
+
+		if context.Bool("quiet") {
+			fs = formatIDList{}
+		} else {
+
+			switch context.String("format") {
+			case "table":
+				fs = formatTabular{}
+
+			case "json":
+				fs = formatJSON{}
+
+			default:
+				return fmt.Errorf("invalid format option")
+			}
+		}
+
+		return fs.Write(s, showAll, file)
+	},
+}
+
+// getStaleAssetsreturns compares the two specified hypervisorDetails objects
+// and returns a list of strings representing which assets in "old" are not
+// current compared to "new". If old and new are identical, the empty string
+// will be returned.
+//
+// Notes:
+//
+// - This function is trivial because it relies upon the fact that new
+//   containers are always created with the latest versions of all assets.
+//
+// - WARNING: Since this function only compares local values, it is unable to
+//   determine if newer (remote) assets are available.
+func getStaleAssets(old, new hypervisorDetails) []string {
+	var stale []string
+
+	if old.KernelAsset.Path != new.KernelAsset.Path {
+		if old.KernelAsset.Custom {
+			// The workload kernel asset is a custom one, i.e. it's not coming
+			// from the runtime configuration file. Thus it does not make sense
+			// to compare it against the configured kernel asset.
+			// We assume a custom kernel asset has been updated if the
+			// corresponding path no longer exists, i.e. it's been replaced by
+			// a new kernel, e.g. with a new version name.
+			// Replacing a custom kernel asset binary with exactly the same
+			// binary name won't allow us to detect if it's staled or not.
+			if _, err := os.Stat(old.KernelAsset.Path); os.IsNotExist(err) {
+				stale = append(stale, "kernel")
+			}
+		} else {
+			stale = append(stale, "kernel")
+		}
+	}
+
+	if old.ImageAsset.Path != new.ImageAsset.Path {
+		if old.ImageAsset.Custom {
+			// The workload image asset is a custom one, i.e. it's not coming
+			// from the runtime configuration file. Thus it does not make sense
+			// to compare it against the configured image asset.
+			// We assume a custom image asset has been updated if the
+			// corresponding path no longer exists, i.e. it's been replaced by
+			// a new image, e.g. with a new version name.
+			// Replacing a custom image asset binary with exactly the same
+			// binary name won't allow us to detect if it's staled or not.
+			if _, err := os.Stat(old.ImageAsset.Path); os.IsNotExist(err) {
+				stale = append(stale, "image")
+			}
+		} else {
+			stale = append(stale, "image")
+		}
+	}
+
+	return stale
+}
+
+func (f formatIDList) Write(state []fullContainerState, showAll bool, file *os.File) error {
+	for _, item := range state {
+		_, err := fmt.Fprintln(file, item.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (f formatTabular) Write(state []fullContainerState, showAll bool, file *os.File) error {
+	// values used by runc
+	flags := uint(0)
+	minWidth := 12
+	tabWidth := 1
+	padding := 3
+
+	w := tabwriter.NewWriter(file, minWidth, tabWidth, padding, ' ', flags)
+
+	fmt.Fprint(w, "ID\tPID\tSTATUS\tBUNDLE\tCREATED\tOWNER")
+
+	if showAll {
+		fmt.Fprint(w, "\tHYPERVISOR\tKERNEL\tIMAGE\tLATEST-KERNEL\tLATEST-IMAGE\tSTALE\n")
+	} else {
+		fmt.Fprintf(w, "\n")
+	}
+
+	for _, item := range state {
+		fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s",
+			item.ID,
+			item.InitProcessPid,
+			item.Status,
+			item.Bundle,
+			item.Created.Format(time.RFC3339Nano),
+			item.Owner)
+
+		if showAll {
+			stale := strings.Join(item.StaleAssets, ",")
+			if stale == "" {
+				stale = "-"
+			}
+
+			current := item.CurrentHypervisorDetails
+			latest := item.LatestHypervisorDetails
+
+			all := fmt.Sprintf("\t%s\t%s\t%s",
+				current.HypervisorAsset.Path,
+				current.KernelAsset.Path,
+				current.ImageAsset.Path)
+
+			if !current.KernelAsset.Custom {
+				all += fmt.Sprintf("\t%s", latest.KernelAsset.Path)
+			} else {
+				all += fmt.Sprintf("\t%s", current.KernelAsset.Path)
+			}
+
+			if !current.ImageAsset.Custom {
+				all += fmt.Sprintf("\t%s", latest.ImageAsset.Path)
+			} else {
+				all += fmt.Sprintf("\t%s", current.ImageAsset.Path)
+			}
+
+			all += fmt.Sprintf("\t%s\n", stale)
+
+			fmt.Fprint(w, all)
+		} else {
+			fmt.Fprint(w, "\n")
+		}
+	}
+
+	return w.Flush()
+}
+
+func (f formatJSON) Write(state []fullContainerState, showAll bool, file *os.File) error {
+	return json.NewEncoder(file).Encode(state)
+}
+
+// getDirOwner returns the UID of the specified directory
+func getDirOwner(dir string) (uint32, error) {
+	if dir == "" {
+		return 0, errors.New("BUG: need directory")
+	}
+	st, err := os.Stat(dir)
+	if err != nil {
+		return 0, err
+	}
+
+	if !st.IsDir() {
+		return 0, fmt.Errorf("%q is not a directory", dir)
+	}
+
+	statType, ok := st.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("cannot convert %+v to stat type for directory %q", st, dir)
+	}
+
+	return statType.Uid, nil
+}
+
+func getContainers(ctx context.Context, context *cli.Context) ([]fullContainerState, error) {
+	runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+	if !ok {
+		return nil, errors.New("invalid runtime config")
+	}
+
+	latestHypervisorDetails := getHypervisorDetails(&runtimeConfig.HypervisorConfig)
+
+	sandboxList, err := vci.ListSandbox(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var s []fullContainerState
+
+	for _, sandbox := range sandboxList {
+		if len(sandbox.ContainersStatus) == 0 {
+			// ignore empty sandboxes
+			continue
+		}
+
+		currentHypervisorDetails := getHypervisorDetails(&sandbox.HypervisorConfig)
+
+		for _, container := range sandbox.ContainersStatus {
+			ociState := oci.StatusToOCIState(container)
+			staleAssets := getStaleAssets(currentHypervisorDetails, latestHypervisorDetails)
+
+			uid, err := getDirOwner(container.RootFs)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "WARNING: failed to get container %s rootfs: %s\n", ociState.ID, err)
+				continue
+			}
+
+			owner := fmt.Sprintf("#%v", uid)
+
+			s = append(s, fullContainerState{
+				containerState: containerState{
+					Version:        ociState.Version,
+					ID:             ociState.ID,
+					InitProcessPid: ociState.Pid,
+					Status:         ociState.Status,
+					Bundle:         ociState.Bundle,
+					Rootfs:         container.RootFs,
+					Created:        container.StartTime,
+					Annotations:    ociState.Annotations,
+					Owner:          owner,
+				},
+				CurrentHypervisorDetails: currentHypervisorDetails,
+				LatestHypervisorDetails:  latestHypervisorDetails,
+				StaleAssets:              staleAssets,
+			})
+		}
+	}
+
+	return s, nil
+}
+
+// getHypervisorDetails returns details of the latest version of the
+// hypervisor and the associated assets.
+func getHypervisorDetails(hypervisorConfig *vc.HypervisorConfig) hypervisorDetails {
+	hypervisorPath, err := hypervisorConfig.HypervisorAssetPath()
+	if err != nil {
+		hypervisorPath = hypervisorConfig.HypervisorPath
+	}
+
+	kernelPath, err := hypervisorConfig.KernelAssetPath()
+	if err != nil {
+		kernelPath = hypervisorConfig.KernelPath
+	}
+
+	imagePath, err := hypervisorConfig.ImageAssetPath()
+	if err != nil {
+		imagePath = hypervisorConfig.ImagePath
+	}
+
+	return hypervisorDetails{
+		HypervisorAsset: asset{
+			Path:   hypervisorPath,
+			Custom: hypervisorConfig.CustomHypervisorAsset(),
+		},
+		KernelAsset: asset{
+			Path:   kernelPath,
+			Custom: hypervisorConfig.CustomKernelAsset(),
+		},
+		ImageAsset: asset{
+			Path:   imagePath,
+			Custom: hypervisorConfig.CustomImageAsset(),
+		},
+	}
+}

src/runtime/cli/list_test.go

@@ -0,0 +1,785 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+type TestFileWriter struct {
+	Name string
+	File *os.File
+}
+
+var hypervisorDetails1 = hypervisorDetails{
+	HypervisorAsset: asset{
+		Path: "/hypervisor/path",
+	},
+	ImageAsset: asset{
+		Path: "/image/path",
+	},
+	KernelAsset: asset{
+		Path: "/kernel/path",
+	},
+}
+
+var hypervisorDetails2 = hypervisorDetails{
+	HypervisorAsset: asset{
+		Path: "/hypervisor/path2",
+	},
+	ImageAsset: asset{
+		Path: "/image/path2",
+	},
+	KernelAsset: asset{
+		Path: "/kernel/path2",
+	},
+}
+
+var hypervisorDetails3 = hypervisorDetails{
+	HypervisorAsset: asset{
+		Path: "/hypervisor/path3",
+	},
+	ImageAsset: asset{
+		Path: "/image/path3",
+	},
+	KernelAsset: asset{
+		Path: "/kernel/path3",
+	},
+}
+
+var testStatuses = []fullContainerState{
+	{
+		containerState: containerState{
+			Version:        "",
+			ID:             "1",
+			InitProcessPid: 1234,
+			Status:         "running",
+			Bundle:         "/somewhere/over/the/rainbow",
+			Created:        time.Now().UTC(),
+			Annotations:    map[string]string(nil),
+			Owner:          "#0",
+		},
+
+		CurrentHypervisorDetails: hypervisorDetails1,
+		LatestHypervisorDetails:  hypervisorDetails1,
+		StaleAssets:              []string{},
+	},
+	{
+		containerState: containerState{
+			Version:        "",
+			ID:             "2",
+			InitProcessPid: 2345,
+			Status:         "stopped",
+			Bundle:         "/this/path/is/invalid",
+			Created:        time.Now().UTC(),
+			Annotations:    map[string]string(nil),
+			Owner:          "#0",
+		},
+
+		CurrentHypervisorDetails: hypervisorDetails2,
+		LatestHypervisorDetails:  hypervisorDetails2,
+		StaleAssets:              []string{},
+	},
+	{
+		containerState: containerState{
+			Version:        "",
+			ID:             "3",
+			InitProcessPid: 9999,
+			Status:         "ready",
+			Bundle:         "/foo/bar/baz",
+			Created:        time.Now().UTC(),
+			Annotations:    map[string]string(nil),
+			Owner:          "#0",
+		},
+
+		CurrentHypervisorDetails: hypervisorDetails3,
+		LatestHypervisorDetails:  hypervisorDetails3,
+		StaleAssets:              []string{},
+	},
+}
+
+// Implement the io.Writer interface
+func (w *TestFileWriter) Write(bytes []byte) (n int, err error) {
+	return w.File.Write(bytes)
+}
+
+func formatListDataAsBytes(formatter formatState, state []fullContainerState, showAll bool) (bytes []byte, err error) {
+	tmpfile, err := ioutil.TempFile("", "formatListData-")
+	if err != nil {
+		return nil, err
+	}
+
+	defer os.Remove(tmpfile.Name())
+
+	err = formatter.Write(state, showAll, tmpfile)
+	if err != nil {
+		return nil, err
+	}
+
+	tmpfile.Close()
+
+	return ioutil.ReadFile(tmpfile.Name())
+}
+
+func formatListDataAsString(formatter formatState, state []fullContainerState, showAll bool) (lines []string, err error) {
+	bytes, err := formatListDataAsBytes(formatter, state, showAll)
+	if err != nil {
+		return nil, err
+	}
+
+	lines = strings.Split(string(bytes), "\n")
+
+	// Remove last line if empty
+	length := len(lines)
+	last := lines[length-1]
+	if last == "" {
+		lines = lines[:length-1]
+	}
+
+	return lines, nil
+}
+
+func TestStateToIDList(t *testing.T) {
+
+	// no header
+	expectedLength := len(testStatuses)
+
+	// showAll should not affect the output
+	for _, showAll := range []bool{true, false} {
+		lines, err := formatListDataAsString(&formatIDList{}, testStatuses, showAll)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var expected []string
+		for _, s := range testStatuses {
+			expected = append(expected, s.ID)
+		}
+
+		length := len(lines)
+
+		if length != expectedLength {
+			t.Fatalf("Expected %d lines, got %d: %v", expectedLength, length, lines)
+		}
+
+		assert.Equal(t, lines, expected, "lines + expected")
+	}
+}
+
+func TestStateToTabular(t *testing.T) {
+	// +1 for header line
+	expectedLength := len(testStatuses) + 1
+
+	expectedDefaultHeaderPattern := `\AID\s+PID\s+STATUS\s+BUNDLE\s+CREATED\s+OWNER`
+	expectedExtendedHeaderPattern := `HYPERVISOR\s+KERNEL\s+IMAGE\s+LATEST-KERNEL\s+LATEST-IMAGE\s+STALE`
+	endingPattern := `\s*\z`
+
+	lines, err := formatListDataAsString(&formatTabular{}, testStatuses, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := len(lines)
+
+	expectedHeaderPattern := expectedDefaultHeaderPattern + endingPattern
+	expectedHeaderRE := regexp.MustCompile(expectedHeaderPattern)
+
+	if length != expectedLength {
+		t.Fatalf("Expected %d lines, got %d", expectedLength, length)
+	}
+
+	header := lines[0]
+
+	matches := expectedHeaderRE.FindAllStringSubmatch(header, -1)
+	if matches == nil {
+		t.Fatalf("Header line failed to match:\n"+
+			"pattern : %v\n"+
+			"line    : %v\n",
+			expectedDefaultHeaderPattern,
+			header)
+	}
+
+	for i, status := range testStatuses {
+		lineIndex := i + 1
+		line := lines[lineIndex]
+
+		expectedLinePattern := fmt.Sprintf(`\A%s\s+%d\s+%s\s+%s\s+%s\s+%s\s*\z`,
+			regexp.QuoteMeta(status.ID),
+			status.InitProcessPid,
+			regexp.QuoteMeta(status.Status),
+			regexp.QuoteMeta(status.Bundle),
+			regexp.QuoteMeta(status.Created.Format(time.RFC3339Nano)),
+			regexp.QuoteMeta(status.Owner))
+
+		expectedLineRE := regexp.MustCompile(expectedLinePattern)
+
+		matches := expectedLineRE.FindAllStringSubmatch(line, -1)
+		if matches == nil {
+			t.Fatalf("Data line failed to match:\n"+
+				"pattern : %v\n"+
+				"line    : %v\n",
+				expectedLinePattern,
+				line)
+		}
+	}
+
+	// Try again with full details this time
+	lines, err = formatListDataAsString(&formatTabular{}, testStatuses, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length = len(lines)
+
+	expectedHeaderPattern = expectedDefaultHeaderPattern + `\s+` + expectedExtendedHeaderPattern + endingPattern
+	expectedHeaderRE = regexp.MustCompile(expectedHeaderPattern)
+
+	if length != expectedLength {
+		t.Fatalf("Expected %d lines, got %d", expectedLength, length)
+	}
+
+	header = lines[0]
+
+	matches = expectedHeaderRE.FindAllStringSubmatch(header, -1)
+	if matches == nil {
+		t.Fatalf("Header line failed to match:\n"+
+			"pattern : %v\n"+
+			"line    : %v\n",
+			expectedDefaultHeaderPattern,
+			header)
+	}
+
+	for i, status := range testStatuses {
+		lineIndex := i + 1
+		line := lines[lineIndex]
+
+		expectedLinePattern := fmt.Sprintf(`\A%s\s+%d\s+%s\s+%s\s+%s\s+%s\s+%s\s+%s\s+%s\s+%s\s+%s\s+%s\s*\z`,
+			regexp.QuoteMeta(status.ID),
+			status.InitProcessPid,
+			regexp.QuoteMeta(status.Status),
+			regexp.QuoteMeta(status.Bundle),
+			regexp.QuoteMeta(status.Created.Format(time.RFC3339Nano)),
+			regexp.QuoteMeta(status.Owner),
+			regexp.QuoteMeta(status.CurrentHypervisorDetails.HypervisorAsset.Path),
+			regexp.QuoteMeta(status.CurrentHypervisorDetails.KernelAsset.Path),
+			regexp.QuoteMeta(status.CurrentHypervisorDetails.ImageAsset.Path),
+			regexp.QuoteMeta(status.LatestHypervisorDetails.KernelAsset.Path),
+			regexp.QuoteMeta(status.LatestHypervisorDetails.ImageAsset.Path),
+			regexp.QuoteMeta("-"))
+
+		expectedLineRE := regexp.MustCompile(expectedLinePattern)
+
+		matches := expectedLineRE.FindAllStringSubmatch(line, -1)
+		if matches == nil {
+			t.Fatalf("Data line failed to match:\n"+
+				"pattern : %v\n"+
+				"line    : %v\n",
+				expectedLinePattern,
+				line)
+		}
+	}
+}
+
+func TestStateToJSON(t *testing.T) {
+	expectedLength := len(testStatuses)
+
+	// showAll should not affect the output
+	for _, showAll := range []bool{true, false} {
+		bytes, err := formatListDataAsBytes(&formatJSON{}, testStatuses, showAll)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Force capacity to match the original otherwise assert.Equal() complains.
+		states := make([]fullContainerState, 0, len(testStatuses))
+
+		err = json.Unmarshal(bytes, &states)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		length := len(states)
+
+		if length != expectedLength {
+			t.Fatalf("Expected %d lines, got %d", expectedLength, length)
+		}
+
+		// golang tip (what will presumably become v1.9) now
+		// stores a monotonic clock value as part of time.Time's
+		// internal representation (this is shown by a suffix in
+		// the form "m=±ddd.nnnnnnnnn" when calling String() on
+		// the time.Time object). However, this monotonic value
+		// is stripped out when marshaling.
+		//
+		// This behaviour change makes comparing the original
+		// object and the marshaled-and-then-unmarshaled copy of
+		// the object doomed to failure.
+		//
+		// The solution? Manually strip the monotonic time out
+		// of the original before comparison (yuck!)
+		//
+		// See:
+		//
+		// - https://go-review.googlesource.com/c/36255/7/src/time/time.go#54
+		//
+		for i := 0; i < expectedLength; i++ {
+			// remove monotonic time part
+			testStatuses[i].Created = testStatuses[i].Created.Truncate(0)
+		}
+
+		assert.Equal(t, states, testStatuses, "states + testStatuses")
+	}
+}
+
+func TestListCLIFunctionNoContainers(t *testing.T) {
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+	ctx.App.Metadata["foo"] = "bar"
+
+	fn, ok := listCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(t, ok)
+
+	err := fn(ctx)
+
+	// no config in the Metadata
+	assert.Error(t, err)
+}
+
+func TestListGetContainersListSandboxFail(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	ctx.App.Metadata = map[string]interface{}{
+		"runtimeConfig": runtimeConfig,
+	}
+
+	_, err = getContainers(context.Background(), ctx)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestListGetContainers(t *testing.T) {
+	assert := assert.New(t)
+
+	testingImpl.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) {
+		// No pre-existing sandboxes
+		return []vc.SandboxStatus{}, nil
+	}
+
+	defer func() {
+		testingImpl.ListSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	ctx.App.Metadata = map[string]interface{}{
+		"runtimeConfig": runtimeConfig,
+	}
+
+	state, err := getContainers(context.Background(), ctx)
+	assert.NoError(err)
+	assert.Equal(state, []fullContainerState(nil))
+}
+
+func TestListGetContainersSandboxWithoutContainers(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) {
+		return []vc.SandboxStatus{
+			{
+				ID:               sandbox.ID(),
+				ContainersStatus: []vc.ContainerStatus(nil),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.ListSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	ctx.App.Metadata = map[string]interface{}{
+		"runtimeConfig": runtimeConfig,
+	}
+
+	state, err := getContainers(context.Background(), ctx)
+	assert.NoError(err)
+	assert.Equal(state, []fullContainerState(nil))
+}
+
+func TestListGetContainersSandboxWithContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	err = os.MkdirAll(rootfs, testDirMode)
+	assert.NoError(err)
+
+	testingImpl.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) {
+		return []vc.SandboxStatus{
+			{
+				ID: sandbox.ID(),
+				ContainersStatus: []vc.ContainerStatus{
+					{
+						ID:          sandbox.ID(),
+						Annotations: map[string]string{},
+						RootFs:      rootfs,
+					},
+				},
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.ListSandboxFunc = nil
+	}()
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	_, err = getContainers(context.Background(), ctx)
+	assert.NoError(err)
+}
+
+func TestListCLIFunctionFormatFail(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	quietFlags := flag.NewFlagSet("test", 0)
+	quietFlags.Bool("quiet", true, "")
+
+	tableFlags := flag.NewFlagSet("test", 0)
+	tableFlags.String("format", "table", "")
+
+	jsonFlags := flag.NewFlagSet("test", 0)
+	jsonFlags.String("format", "json", "")
+
+	invalidFlags := flag.NewFlagSet("test", 0)
+	invalidFlags.String("format", "not-a-valid-format", "")
+
+	type testData struct {
+		format string
+		flags  *flag.FlagSet
+	}
+
+	data := []testData{
+		{"quiet", quietFlags},
+		{"table", tableFlags},
+		{"json", jsonFlags},
+		{"invalid", invalidFlags},
+	}
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+
+	testingImpl.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) {
+		return []vc.SandboxStatus{
+			{
+				ID: sandbox.ID(),
+				ContainersStatus: []vc.ContainerStatus{
+					{
+						ID: sandbox.ID(),
+						Annotations: map[string]string{
+							vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+						},
+						RootFs: rootfs,
+					},
+				},
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.ListSandboxFunc = nil
+	}()
+
+	savedOutputFile := defaultOutputFile
+	defer func() {
+		defaultOutputFile = savedOutputFile
+	}()
+
+	// purposely invalid
+	var invalidFile *os.File
+
+	for _, d := range data {
+		// start off with an invalid output file
+		defaultOutputFile = invalidFile
+
+		ctx := createCLIContext(d.flags)
+		ctx.App.Name = "foo"
+		ctx.App.Metadata["foo"] = "bar"
+
+		fn, ok := listCLICommand.Action.(func(context *cli.Context) error)
+		assert.True(ok, d)
+
+		err = fn(ctx)
+
+		// no config in the Metadata
+		assert.Error(err, d)
+
+		runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+		assert.NoError(err, d)
+
+		ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+		err = os.MkdirAll(rootfs, testDirMode)
+		assert.NoError(err)
+
+		err = fn(ctx)
+
+		// invalid output file
+		assert.Error(err, d)
+		assert.False(vcmock.IsMockError(err), d)
+
+		output := filepath.Join(tmpdir, "output")
+		f, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE, testFileMode)
+		assert.NoError(err)
+		defer f.Close()
+
+		// output file is now valid
+		defaultOutputFile = f
+
+		err = fn(ctx)
+		if d.format == "invalid" {
+			assert.Error(err)
+			assert.False(vcmock.IsMockError(err), d)
+		} else {
+			assert.NoError(err)
+		}
+	}
+}
+
+func TestListCLIFunctionQuiet(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	err = os.MkdirAll(rootfs, testDirMode)
+	assert.NoError(err)
+
+	testingImpl.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) {
+		return []vc.SandboxStatus{
+			{
+				ID: sandbox.ID(),
+				ContainersStatus: []vc.ContainerStatus{
+					{
+						ID: sandbox.ID(),
+						Annotations: map[string]string{
+							vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+						},
+						RootFs: rootfs,
+					},
+				},
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.ListSandboxFunc = nil
+	}()
+
+	set := flag.NewFlagSet("test", 0)
+	set.Bool("quiet", true, "")
+
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	savedOutputFile := defaultOutputFile
+	defer func() {
+		defaultOutputFile = savedOutputFile
+	}()
+
+	output := filepath.Join(tmpdir, "output")
+	f, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY|os.O_SYNC, testFileMode)
+	assert.NoError(err)
+	defer f.Close()
+
+	defaultOutputFile = f
+
+	fn, ok := listCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.NoError(err)
+	f.Close()
+
+	text, err := katautils.GetFileContents(output)
+	assert.NoError(err)
+
+	trimmed := strings.TrimSpace(text)
+	assert.Equal(testSandboxID, trimmed)
+}
+
+func TestListGetDirOwner(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	_, err = getDirOwner("")
+	// invalid parameter
+	assert.Error(err)
+
+	dir := filepath.Join(tmpdir, "dir")
+
+	_, err = getDirOwner(dir)
+	// ENOENT
+	assert.Error(err)
+
+	err = createEmptyFile(dir)
+	assert.NoError(err)
+
+	_, err = getDirOwner(dir)
+	// wrong file type
+	assert.Error(err)
+
+	err = os.Remove(dir)
+	assert.NoError(err)
+
+	err = os.MkdirAll(dir, testDirMode)
+	assert.NoError(err)
+
+	uid := uint32(os.Getuid())
+
+	dirUID, err := getDirOwner(dir)
+	assert.NoError(err)
+	assert.Equal(dirUID, uid)
+}
+
+func TestListWithRootfsMissShouldSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir(testDir, "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	err = os.MkdirAll(rootfs, testDirMode)
+	assert.NoError(err)
+
+	testingImpl.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) {
+		return []vc.SandboxStatus{
+			{
+				ID: sandbox.ID(),
+				ContainersStatus: []vc.ContainerStatus{
+					{
+						ID: sandbox.ID(),
+						Annotations: map[string]string{
+							vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+						},
+						RootFs: rootfs,
+					},
+				},
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.ListSandboxFunc = nil
+	}()
+
+	set := flag.NewFlagSet("test", 0)
+	set.String("format", "table", "")
+	ctx := createCLIContext(set)
+	ctx.App.Name = "foo"
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	ctx.App.Metadata["runtimeConfig"] = runtimeConfig
+
+	fn, ok := listCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.NoError(err)
+
+	// remove container rootfs, check list command should also work
+	assert.NoError(os.RemoveAll(rootfs))
+	err = fn(ctx)
+	assert.NoError(err)
+}

src/runtime/cli/main.go

@@ -0,0 +1,606 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017-2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	goruntime "runtime"
+	"strings"
+	"syscall"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/kata-containers/runtime/pkg/signals"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
+	vf "github.com/kata-containers/runtime/virtcontainers/factory"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// specConfig is the name of the file holding the containers configuration
+const specConfig = "config.json"
+
+// arch is the architecture for the running program
+const arch = goruntime.GOARCH
+
+var usage = fmt.Sprintf(`%s runtime
+
+%s is a command line program for running applications packaged
+according to the Open Container Initiative (OCI).`, name, name)
+
+var notes = fmt.Sprintf(`
+NOTES:
+
+- Commands starting "%s-" and options starting "--%s-" are `+project+` extensions.
+
+URL:
+
+  The canonical URL for this project is: %s
+
+`, projectPrefix, projectPrefix, projectURL)
+
+// kataLog is the logger used to record all messages
+var kataLog *logrus.Entry
+
+// originalLoggerLevel is the default log level. It is used to revert the
+// current log level back to its original value if debug output is not
+// required. We set the default to 'Warn' for the runtime.
+var originalLoggerLevel = logrus.WarnLevel
+
+var debug = false
+
+// if true, coredump when an internal error occurs or a fatal signal is received
+var crashOnError = false
+
+// concrete virtcontainer implementation
+var virtcontainersImpl = &vc.VCImpl{}
+
+// vci is used to access a particular virtcontainers implementation.
+// Normally, it refers to the official package, but is re-assigned in
+// the tests to allow virtcontainers to be mocked.
+var vci vc.VC = virtcontainersImpl
+
+// defaultOutputFile is the default output file to write the gathered
+// information to.
+var defaultOutputFile = os.Stdout
+
+// defaultErrorFile is the default output file to write error
+// messages to.
+var defaultErrorFile = os.Stderr
+
+// runtimeFlags is the list of supported global command-line flags
+var runtimeFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:  configFilePathOption,
+		Usage: project + " config file path",
+	},
+	cli.StringFlag{
+		Name:  "log",
+		Value: "/dev/null",
+		Usage: "set the log file path where internal debug information is written",
+	},
+	cli.StringFlag{
+		Name:  "log-format",
+		Value: "text",
+		Usage: "set the format used by logs ('text' (default), or 'json')",
+	},
+	cli.StringFlag{
+		Name:  "root",
+		Value: defaultRootDirectory,
+		Usage: "root directory for storage of container state (this should be located in tmpfs)",
+	},
+	cli.StringFlag{
+		Name:  "rootless",
+		Value: "auto",
+		Usage: "ignore cgroup permission errors ('true', 'false', or 'auto')",
+	},
+	cli.BoolFlag{
+		Name:  showConfigPathsOption,
+		Usage: "show config file paths that will be checked for (in order)",
+	},
+	cli.BoolFlag{
+		Name:  "systemd-cgroup",
+		Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
+	},
+}
+
+// runtimeCommands is the list of supported command-line (sub-)
+// commands.
+var runtimeCommands = []cli.Command{
+	createCLICommand,
+	deleteCLICommand,
+	execCLICommand,
+	killCLICommand,
+	listCLICommand,
+	pauseCLICommand,
+	psCLICommand,
+	resumeCLICommand,
+	runCLICommand,
+	specCLICommand,
+	startCLICommand,
+	stateCLICommand,
+	updateCLICommand,
+	eventsCLICommand,
+	versionCLICommand,
+
+	// Kata Containers specific extensions
+	kataCheckCLICommand,
+	kataEnvCLICommand,
+	kataNetworkCLICommand,
+	kataOverheadCLICommand,
+	factoryCLICommand,
+}
+
+// runtimeBeforeSubcommands is the function to run before command-line
+// parsing occurs.
+var runtimeBeforeSubcommands = beforeSubcommands
+
+// runtimeAfterSubcommands is the function to run after the command-line
+// has been parsed.
+var runtimeAfterSubcommands = afterSubcommands
+
+// runtimeCommandNotFound is the function to handle an invalid sub-command.
+var runtimeCommandNotFound = commandNotFound
+
+// runtimeVersion is the function that returns the full version
+// string describing the runtime.
+var runtimeVersion = makeVersionString
+
+// saved default cli package values (for testing).
+var savedCLIAppHelpTemplate = cli.AppHelpTemplate
+var savedCLIVersionPrinter = cli.VersionPrinter
+var savedCLIErrWriter = cli.ErrWriter
+
+func init() {
+	kataLog = logrus.WithFields(logrus.Fields{
+		"name":   name,
+		"source": "runtime",
+		"arch":   arch,
+		"pid":    os.Getpid(),
+	})
+
+	// Save the original log level and then set to debug level to ensure
+	// that any problems detected before the config file is parsed are
+	// logged. This is required since the config file determines the true
+	// log level for the runtime: once parsed the log level is set
+	// appropriately but for issues between now and completion of the
+	// config file parsing, it is prudent to operate in verbose mode.
+	originalLoggerLevel = kataLog.Logger.Level
+	kataLog.Logger.Level = logrus.DebugLevel
+}
+
+// setupSignalHandler sets up signal handling, starting a go routine to deal
+// with signals as they arrive.
+//
+// Note that the specified context is NOT used to create a trace span (since the
+// first (root) span must be created in beforeSubcommands()): it is simply
+// used to pass to the crash handling functions to finalise tracing.
+func setupSignalHandler(ctx context.Context) {
+	signals.SetLogger(kataLog)
+
+	sigCh := make(chan os.Signal, 8)
+
+	for _, sig := range signals.HandledSignals() {
+		signal.Notify(sigCh, sig)
+	}
+
+	dieCb := func() {
+		katautils.StopTracing(ctx)
+	}
+
+	go func() {
+		for {
+			sig := <-sigCh
+
+			nativeSignal, ok := sig.(syscall.Signal)
+			if !ok {
+				err := errors.New("unknown signal")
+				kataLog.WithError(err).WithField("signal", sig.String()).Error()
+				continue
+			}
+
+			if signals.FatalSignal(nativeSignal) {
+				kataLog.WithField("signal", sig).Error("received fatal signal")
+				signals.Die(dieCb)
+			} else if debug && signals.NonFatalSignal(nativeSignal) {
+				kataLog.WithField("signal", sig).Debug("handling signal")
+				signals.Backtrace()
+			}
+		}
+	}()
+}
+
+// setExternalLoggers registers the specified logger with the external
+// packages which accept a logger to handle their own logging.
+func setExternalLoggers(ctx context.Context, logger *logrus.Entry) {
+	var span opentracing.Span
+
+	// Only create a new span if a root span already exists. This is
+	// required to ensure that this function will not disrupt the root
+	// span logic by creating a span before the proper root span has been
+	// created.
+
+	if opentracing.SpanFromContext(ctx) != nil {
+		span, ctx = katautils.Trace(ctx, "setExternalLoggers")
+		defer span.Finish()
+	}
+
+	// Set virtcontainers logger.
+	vci.SetLogger(ctx, logger)
+
+	// Set vm factory logger.
+	vf.SetLogger(ctx, logger)
+
+	// Set the OCI package logger.
+	oci.SetLogger(ctx, logger)
+
+	// Set the katautils package logger
+	katautils.SetLogger(ctx, logger, originalLoggerLevel)
+
+	// Set the rootless package logger
+	rootless.SetLogger(ctx, logger)
+}
+
+// beforeSubcommands is the function to perform preliminary checks
+// before command-line parsing occurs.
+func beforeSubcommands(c *cli.Context) error {
+	var configFile string
+	var runtimeConfig oci.RuntimeConfig
+	var err error
+
+	katautils.SetConfigOptions(name, defaultRuntimeConfiguration, defaultSysConfRuntimeConfiguration)
+
+	handleShowConfig(c)
+
+	if userWantsUsage(c) {
+		// No setup required if the user just
+		// wants to see the usage statement.
+		return nil
+	}
+
+	r, err := parseBoolOrAuto(c.GlobalString("rootless"))
+	if err != nil {
+		return err
+	}
+	// If flag is true/false, assign the rootless flag.
+	// vc will not perform any auto-detection in that case.
+	// In case flag is nil or auto, vc detects if the runtime is running as rootless.
+	if r != nil {
+		rootless.SetRootless(*r)
+	}
+	// Support --systed-cgroup
+	// Issue: https://github.com/kata-containers/runtime/issues/2428
+
+	ignoreConfigLogs := false
+	var traceRootSpan string
+
+	subCmdIsCheckCmd := (c.NArg() >= 1 && (c.Args()[0] == checkCmd))
+	if subCmdIsCheckCmd {
+		// checkCmd will use the default logrus logger to stderr
+		// raise the logger default level to warn
+		kataLog.Logger.SetLevel(logrus.WarnLevel)
+		// do not print configuration logs for checkCmd
+		ignoreConfigLogs = true
+	} else {
+		if path := c.GlobalString("log"); path != "" {
+			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
+			if err != nil {
+				return err
+			}
+			kataLog.Logger.Out = f
+		}
+
+		switch c.GlobalString("log-format") {
+		case "text":
+			// retain logrus's default.
+		case "json":
+			kataLog.Logger.Formatter = new(logrus.JSONFormatter)
+		default:
+			return fmt.Errorf("unknown log-format %q", c.GlobalString("log-format"))
+		}
+
+		// Add the name of the sub-command to each log entry for easier
+		// debugging.
+		cmdName := c.Args().First()
+		if c.App.Command(cmdName) != nil {
+			kataLog = kataLog.WithField("command", cmdName)
+
+			// Name for the root span (used for tracing) now the
+			// sub-command name is known.
+			traceRootSpan = name + " " + cmdName
+		}
+
+		// Since a context is required, pass a new (throw-away) one - we
+		// cannot use the main context as tracing hasn't been enabled yet
+		// (meaning any spans created at this point will be silently ignored).
+		setExternalLoggers(context.Background(), kataLog)
+
+		if c.NArg() == 1 && c.Args()[0] == envCmd {
+			// simply report the logging setup
+			ignoreConfigLogs = true
+		}
+	}
+
+	configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false)
+	if err != nil {
+		fatal(err)
+	}
+	if !subCmdIsCheckCmd {
+		debug = runtimeConfig.Debug
+		crashOnError = runtimeConfig.Debug
+
+		if traceRootSpan != "" {
+			// Create the tracer.
+			//
+			// Note: no spans are created until the command-line has been parsed.
+			// This delays collection of trace data slightly but benefits the user by
+			// ensuring the first span is the name of the sub-command being
+			// invoked from the command-line.
+			err = setupTracing(c, traceRootSpan)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	args := strings.Join(c.Args(), " ")
+
+	fields := logrus.Fields{
+		"version":   version,
+		"commit":    commit,
+		"arguments": `"` + args + `"`,
+	}
+
+	err = addExpFeatures(c, runtimeConfig)
+	if err != nil {
+		return err
+	}
+
+	kataLog.WithFields(fields).Info()
+
+	// make the data accessible to the sub-commands.
+	c.App.Metadata["runtimeConfig"] = runtimeConfig
+	c.App.Metadata["configFile"] = configFile
+
+	return nil
+}
+
+// handleShowConfig determines if the user wishes to see the configuration
+// paths. If so, it will display them and then exit.
+func handleShowConfig(context *cli.Context) {
+	if context.GlobalBool(showConfigPathsOption) {
+		files := katautils.GetDefaultConfigFilePaths()
+
+		for _, file := range files {
+			fmt.Fprintf(defaultOutputFile, "%s\n", file)
+		}
+
+		exit(0)
+	}
+}
+
+func setupTracing(context *cli.Context, rootSpanName string) error {
+	tracer, err := katautils.CreateTracer(name)
+	if err != nil {
+		fatal(err)
+	}
+
+	// Create the root span now that the sub-command name is
+	// known.
+	//
+	// Note that this "Before" function is called (and returns)
+	// before the subcommand handler is called. As such, we cannot
+	// "Finish()" the span here - that is handled in the .After
+	// function.
+	span := tracer.StartSpan(rootSpanName)
+
+	ctx, err := cliContextToContext(context)
+	if err != nil {
+		return err
+	}
+
+	span.SetTag("subsystem", "runtime")
+
+	// Associate the root span with the context
+	ctx = opentracing.ContextWithSpan(ctx, span)
+
+	// Add tracer to metadata and update the context
+	context.App.Metadata["tracer"] = tracer
+	context.App.Metadata["context"] = ctx
+
+	return nil
+}
+
+// add supported experimental features in context
+func addExpFeatures(clictx *cli.Context, runtimeConfig oci.RuntimeConfig) error {
+	ctx, err := cliContextToContext(clictx)
+	if err != nil {
+		return err
+	}
+
+	var exps []string
+	for _, e := range runtimeConfig.Experimental {
+		exps = append(exps, e.Name)
+	}
+
+	ctx = exp.ContextWithExp(ctx, exps)
+	// Add tracer to metadata and update the context
+	clictx.App.Metadata["context"] = ctx
+	return nil
+}
+
+func afterSubcommands(c *cli.Context) error {
+	ctx, err := cliContextToContext(c)
+	if err != nil {
+		return err
+	}
+
+	katautils.StopTracing(ctx)
+
+	return nil
+}
+
+// function called when an invalid command is specified which causes the
+// runtime to error.
+func commandNotFound(c *cli.Context, command string) {
+	err := fmt.Errorf("Invalid command %q", command)
+	fatal(err)
+}
+
+// makeVersionString returns a multi-line string describing the runtime
+// version along with the version of the OCI specification it supports.
+func makeVersionString() string {
+	v := make([]string, 0, 3)
+
+	versionStr := version
+	if versionStr == "" {
+		versionStr = unknown
+	}
+
+	v = append(v, name+"  : "+versionStr)
+
+	commitStr := commit
+	if commitStr == "" {
+		commitStr = unknown
+	}
+
+	v = append(v, "   commit   : "+commitStr)
+
+	specVersionStr := specs.Version
+	if specVersionStr == "" {
+		specVersionStr = unknown
+	}
+
+	v = append(v, "   OCI specs: "+specVersionStr)
+
+	return strings.Join(v, "\n")
+}
+
+// setCLIGlobals modifies various cli package global variables
+func setCLIGlobals() {
+	cli.AppHelpTemplate = fmt.Sprintf(`%s%s`, cli.AppHelpTemplate, notes)
+
+	// Override the default function to display version details to
+	// ensure the "--version" option and "version" command are identical.
+	cli.VersionPrinter = func(c *cli.Context) {
+		fmt.Fprintln(defaultOutputFile, c.App.Version)
+	}
+
+	// If the command returns an error, cli takes upon itself to print
+	// the error on cli.ErrWriter and exit.
+	// Use our own writer here to ensure the log gets sent to the right
+	// location.
+	cli.ErrWriter = &fatalWriter{cli.ErrWriter}
+}
+
+// createRuntimeApp creates an application to process the command-line
+// arguments and invoke the requested runtime command.
+func createRuntimeApp(ctx context.Context, args []string) error {
+	app := cli.NewApp()
+
+	app.Name = name
+	app.Writer = defaultOutputFile
+	app.Usage = usage
+	app.CommandNotFound = runtimeCommandNotFound
+	app.Version = runtimeVersion()
+	app.Flags = runtimeFlags
+	app.Commands = runtimeCommands
+	app.Before = runtimeBeforeSubcommands
+	app.After = runtimeAfterSubcommands
+	app.EnableBashCompletion = true
+
+	// allow sub-commands to access context
+	app.Metadata = map[string]interface{}{
+		"context": ctx,
+	}
+
+	return app.Run(args)
+}
+
+// userWantsUsage determines if the user only wishes to see the usage
+// statement.
+func userWantsUsage(context *cli.Context) bool {
+	if context.NArg() == 0 {
+		return true
+	}
+
+	if context.NArg() == 1 && (context.Args()[0] == "help" || context.Args()[0] == "version") {
+		return true
+	}
+
+	if context.NArg() >= 2 && (context.Args()[1] == "-h" || context.Args()[1] == "--help") {
+		return true
+	}
+
+	return false
+}
+
+// fatal prints the error's details exits the program.
+func fatal(err error) {
+	kataLog.Error(err)
+	fmt.Fprintln(defaultErrorFile, err)
+	exit(1)
+}
+
+type fatalWriter struct {
+	cliErrWriter io.Writer
+}
+
+func (f *fatalWriter) Write(p []byte) (n int, err error) {
+	// Ensure error is logged before displaying to the user
+	kataLog.Error(string(p))
+	return f.cliErrWriter.Write(p)
+}
+
+func createRuntime(ctx context.Context) {
+	setupSignalHandler(ctx)
+
+	setCLIGlobals()
+
+	err := createRuntimeApp(ctx, os.Args)
+	if err != nil {
+		fatal(err)
+	}
+}
+
+// cliContextToContext extracts the generic context from the specified
+// cli context.
+func cliContextToContext(c *cli.Context) (context.Context, error) {
+	if c == nil {
+		return nil, errors.New("need cli.Context")
+	}
+
+	// extract the main context
+	ctx, ok := c.App.Metadata["context"].(context.Context)
+	if !ok {
+		return nil, errors.New("invalid or missing context in metadata")
+	}
+
+	return ctx, nil
+}
+
+func main() {
+	// create a new empty context
+	ctx := context.Background()
+
+	dieCb := func() {
+		katautils.StopTracing(ctx)
+	}
+
+	defer signals.HandlePanic(dieCb)
+
+	createRuntime(ctx)
+}

src/runtime/cli/network.go

@@ -0,0 +1,229 @@
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type networkType int
+
+const (
+	// interfaceType for interface operation
+	interfaceType networkType = iota
+
+	routeType
+)
+
+var kataNetworkCLICommand = cli.Command{
+	Name:  "kata-network",
+	Usage: "manage interfaces and routes for container",
+	Subcommands: []cli.Command{
+		addIfaceCommand,
+		delIfaceCommand,
+		listIfacesCommand,
+		updateRoutesCommand,
+		listRoutesCommand,
+	},
+	Action: func(context *cli.Context) error {
+		return cli.ShowSubcommandHelp(context)
+	},
+}
+
+var addIfaceCommand = cli.Command{
+	Name:      "add-iface",
+	Usage:     "add an interface to a container",
+	ArgsUsage: `add-iface <container-id> file or - for stdin`,
+	Flags:     []cli.Flag{},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, true)
+	},
+}
+
+var delIfaceCommand = cli.Command{
+	Name:      "del-iface",
+	Usage:     "delete an interface from a container",
+	ArgsUsage: `del-iface <container-id> file or - for stdin`,
+	Flags:     []cli.Flag{},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, false)
+	},
+}
+
+var listIfacesCommand = cli.Command{
+	Name:      "list-ifaces",
+	Usage:     "list network interfaces in a container",
+	ArgsUsage: `list-ifaces <container-id>`,
+	Flags:     []cli.Flag{},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		return networkListCommand(ctx, context.Args().First(), interfaceType)
+	},
+}
+
+var updateRoutesCommand = cli.Command{
+	Name:      "update-routes",
+	Usage:     "update routes of a container",
+	ArgsUsage: `update-routes <container-id> file or - for stdin`,
+	Flags:     []cli.Flag{},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, true)
+	},
+}
+
+var listRoutesCommand = cli.Command{
+	Name:      "list-routes",
+	Usage:     "list network routes in a container",
+	ArgsUsage: `list-routes <container-id>`,
+	Flags:     []cli.Flag{},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		return networkListCommand(ctx, context.Args().First(), routeType)
+	},
+}
+
+func networkModifyCommand(ctx context.Context, containerID, input string, opType networkType, add bool) (err error) {
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+
+	// container MUST be running
+	if status.State.State != types.StateRunning {
+		return fmt.Errorf("container %s is not running", containerID)
+	}
+
+	var (
+		f      *os.File
+		output = defaultOutputFile
+	)
+
+	if input == "-" {
+		f = os.Stdin
+	} else {
+		f, err = os.Open(input)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+	}
+	switch opType {
+	case interfaceType:
+		var inf, resultingInf *vcTypes.Interface
+		if err = json.NewDecoder(f).Decode(&inf); err != nil {
+			return err
+		}
+		if add {
+			resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
+			if err != nil {
+				kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
+					WithError(err).Error("add interface failed")
+			}
+		} else {
+			resultingInf, err = vci.RemoveInterface(ctx, sandboxID, inf)
+			if err != nil {
+				kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)).
+					WithError(err).Error("delete interface failed")
+			}
+		}
+		json.NewEncoder(output).Encode(resultingInf)
+	case routeType:
+		var routes, resultingRoutes []*vcTypes.Route
+		if err = json.NewDecoder(f).Decode(&routes); err != nil {
+			return err
+		}
+		resultingRoutes, err = vci.UpdateRoutes(ctx, sandboxID, routes)
+		json.NewEncoder(output).Encode(resultingRoutes)
+		if err != nil {
+			kataLog.WithField("resulting-routes", fmt.Sprintf("%+v", resultingRoutes)).
+				WithError(err).Error("update routes failed")
+		}
+	}
+	return err
+}
+
+func networkListCommand(ctx context.Context, containerID string, opType networkType) (err error) {
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+
+	// container MUST be running
+	if status.State.State != types.StateRunning {
+		return fmt.Errorf("container %s is not running", containerID)
+	}
+
+	var file = defaultOutputFile
+
+	switch opType {
+	case interfaceType:
+		var interfaces []*vcTypes.Interface
+		interfaces, err = vci.ListInterfaces(ctx, sandboxID)
+		if err != nil {
+			kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
+				WithError(err).Error("list interfaces failed")
+		}
+		json.NewEncoder(file).Encode(interfaces)
+	case routeType:
+		var routes []*vcTypes.Route
+		routes, err = vci.ListRoutes(ctx, sandboxID)
+		if err != nil {
+			kataLog.WithField("resulting-routes", fmt.Sprintf("%+v", routes)).
+				WithError(err).Error("update routes failed")
+		}
+		json.NewEncoder(file).Encode(routes)
+	}
+	return err
+}

src/runtime/cli/network_test.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+)
+
+var (
+	testAddInterfaceFuncReturnNil = func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+		return nil, nil
+	}
+	testRemoveInterfaceFuncReturnNil = func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+		return nil, nil
+	}
+	testListInterfacesFuncReturnNil = func(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) {
+		return nil, nil
+	}
+	testUpdateRoutsFuncReturnNil = func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) {
+		return nil, nil
+	}
+	testListRoutesFuncReturnNil = func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) {
+		return nil, nil
+	}
+)
+
+func TestNetworkCliFunction(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	testingImpl.AddInterfaceFunc = testAddInterfaceFuncReturnNil
+	testingImpl.RemoveInterfaceFunc = testRemoveInterfaceFuncReturnNil
+	testingImpl.ListInterfacesFunc = testListInterfacesFuncReturnNil
+	testingImpl.UpdateRoutesFunc = testUpdateRoutsFuncReturnNil
+	testingImpl.ListRoutesFunc = testListRoutesFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.AddInterfaceFunc = nil
+		testingImpl.RemoveInterfaceFunc = nil
+		testingImpl.ListInterfacesFunc = nil
+		testingImpl.UpdateRoutesFunc = nil
+		testingImpl.ListRoutesFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	execCLICommandFunc(assert, addIfaceCommand, set, true)
+
+	set.Parse([]string{testContainerID})
+	execCLICommandFunc(assert, listIfacesCommand, set, false)
+	execCLICommandFunc(assert, listRoutesCommand, set, false)
+
+	f, err := ioutil.TempFile("", "interface")
+	defer os.Remove(f.Name())
+	assert.NoError(err)
+	assert.NotNil(f)
+	f.WriteString("{}")
+
+	set.Parse([]string{testContainerID, f.Name()})
+	execCLICommandFunc(assert, addIfaceCommand, set, false)
+	execCLICommandFunc(assert, delIfaceCommand, set, false)
+
+	f.Seek(0, 0)
+	f.WriteString("[{}]")
+	f.Close()
+	execCLICommandFunc(assert, updateRoutesCommand, set, false)
+}

src/runtime/cli/oci.go

@@ -0,0 +1,213 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/opencontainers/runc/libcontainer/utils"
+)
+
+// Contants related to cgroup memory directory
+const (
+	// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
+	// here: http://man7.org/linux/man-pages/man2/statfs.2.html
+	cgroupFsType = 0x27e0eb
+)
+
+var cgroupsDirPath string
+
+var procMountInfo = "/proc/self/mountinfo"
+
+// getContainerInfo returns the container status and its sandbox ID.
+func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStatus, string, error) {
+	// container ID MUST be provided.
+	if containerID == "" {
+		return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
+	}
+
+	sandboxID, err := katautils.FetchContainerIDMapping(containerID)
+	if err != nil {
+		return vc.ContainerStatus{}, "", err
+	}
+	if sandboxID == "" {
+		// Not finding a container should not trigger an error as
+		// getContainerInfo is used for checking the existence and
+		// the absence of a container ID.
+		return vc.ContainerStatus{}, "", nil
+	}
+
+	ctrStatus, err := vci.StatusContainer(ctx, sandboxID, containerID)
+	if err != nil {
+		return vc.ContainerStatus{}, "", err
+	}
+
+	return ctrStatus, sandboxID, nil
+}
+
+func getExistingContainerInfo(ctx context.Context, containerID string) (vc.ContainerStatus, string, error) {
+	cStatus, sandboxID, err := getContainerInfo(ctx, containerID)
+	if err != nil {
+		return vc.ContainerStatus{}, "", err
+	}
+
+	// container ID MUST exist.
+	if cStatus.ID == "" {
+		return vc.ContainerStatus{}, "", fmt.Errorf("Container ID (%v) does not exist", containerID)
+	}
+
+	return cStatus, sandboxID, nil
+}
+
+func validCreateParams(ctx context.Context, containerID, bundlePath string) (string, error) {
+	// container ID MUST be provided.
+	if containerID == "" {
+		return "", fmt.Errorf("Missing container ID")
+	}
+
+	// container ID MUST be unique.
+	cStatus, _, err := getContainerInfo(ctx, containerID)
+	if err != nil {
+		return "", err
+	}
+
+	if cStatus.ID != "" {
+		return "", fmt.Errorf("ID already in use, unique ID should be provided")
+	}
+
+	// bundle path MUST be provided.
+	if bundlePath == "" {
+		return "", fmt.Errorf("Missing bundle path")
+	}
+
+	// bundle path MUST be valid.
+	fileInfo, err := os.Stat(bundlePath)
+	if err != nil {
+		return "", fmt.Errorf("Invalid bundle path '%s': %s", bundlePath, err)
+	}
+	if !fileInfo.IsDir() {
+		return "", fmt.Errorf("Invalid bundle path '%s', it should be a directory", bundlePath)
+	}
+
+	resolved, err := katautils.ResolvePath(bundlePath)
+	if err != nil {
+		return "", err
+	}
+
+	return resolved, nil
+}
+
+func isCgroupMounted(cgroupPath string) bool {
+	var statFs syscall.Statfs_t
+
+	if err := syscall.Statfs(cgroupPath, &statFs); err != nil {
+		return false
+	}
+
+	if statFs.Type != archConvertStatFs(cgroupFsType) {
+		return false
+	}
+
+	return true
+}
+
+func setupConsole(consolePath, consoleSockPath string) (string, error) {
+	if consolePath != "" {
+		return consolePath, nil
+	}
+
+	if consoleSockPath == "" {
+		return "", nil
+	}
+
+	console, err := newConsole()
+	if err != nil {
+		return "", err
+	}
+	defer console.master.Close()
+
+	// Open the socket path provided by the caller
+	conn, err := net.Dial("unix", consoleSockPath)
+	if err != nil {
+		return "", err
+	}
+
+	uConn, ok := conn.(*net.UnixConn)
+	if !ok {
+		return "", fmt.Errorf("casting to *net.UnixConn failed")
+	}
+
+	socket, err := uConn.File()
+	if err != nil {
+		return "", err
+	}
+
+	// Send the parent fd through the provided socket
+	if err := utils.SendFd(socket, console.master.Name(), console.master.Fd()); err != nil {
+		return "", err
+	}
+
+	return console.slavePath, nil
+}
+
+func noNeedForOutput(detach bool, tty bool) bool {
+	if !detach {
+		return false
+	}
+
+	if !tty {
+		return false
+	}
+
+	return true
+}
+
+func getCgroupsDirPath(mountInfoFile string) (string, error) {
+	if cgroupsDirPath != "" {
+		return cgroupsDirPath, nil
+	}
+
+	f, err := os.Open(mountInfoFile)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	var cgroupRootPath string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		text := scanner.Text()
+		index := strings.Index(text, " - ")
+		if index < 0 {
+			continue
+		}
+		fields := strings.Split(text, " ")
+		postSeparatorFields := strings.Fields(text[index+3:])
+		numPostFields := len(postSeparatorFields)
+
+		if len(fields) < 5 || postSeparatorFields[0] != "cgroup" || numPostFields < 3 {
+			continue
+		}
+
+		cgroupRootPath = filepath.Dir(fields[4])
+		break
+	}
+
+	if _, err = os.Stat(cgroupRootPath); err != nil {
+		return "", err
+	}
+
+	return cgroupRootPath, nil
+}

src/runtime/cli/oci_test.go

@@ -0,0 +1,319 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/opencontainers/runc/libcontainer/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	consolePathTest       = "console-test"
+	consoleSocketPathTest = "console-socket-test"
+)
+
+func TestGetContainerInfoContainerIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+	status, _, err := getContainerInfo(context.Background(), "")
+
+	assert.Error(err, "This test should fail because containerID is empty")
+	assert.Empty(status.ID, "Expected blank fullID, but got %v", status.ID)
+}
+
+func TestGetContainerInfo(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	containerID := testContainerID
+
+	containerStatus := vc.ContainerStatus{
+		ID: containerID,
+		Annotations: map[string]string{
+			vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+		},
+	}
+
+	path, err := createTempContainerIDMapping(containerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return containerStatus, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	status, sandboxID, err := getContainerInfo(context.Background(), testContainerID)
+	assert.NoError(err)
+	assert.Equal(sandboxID, sandbox.ID())
+	assert.Equal(status, containerStatus)
+}
+
+func TestValidCreateParamsContainerIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+	_, err := validCreateParams(context.Background(), "", "")
+
+	assert.Error(err, "This test should fail because containerID is empty")
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestGetExistingContainerInfoContainerIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+	status, _, err := getExistingContainerInfo(context.Background(), "")
+
+	assert.Error(err, "This test should fail because containerID is empty")
+	assert.Empty(status.ID, "Expected blank fullID, but got %v", status.ID)
+}
+
+func TestValidCreateParamsContainerIDNotUnique(t *testing.T) {
+	assert := assert.New(t)
+
+	testSandboxID2 := testSandboxID + "2"
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	err = os.MkdirAll(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID2), 0750)
+	assert.NoError(err)
+
+	_, err = validCreateParams(context.Background(), testContainerID, "")
+
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestValidCreateParamsInvalidBundle(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	_, err = validCreateParams(context.Background(), testContainerID, bundlePath)
+	// bundle is ENOENT
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestValidCreateParamsBundleIsAFile(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+	err = createEmptyFile(bundlePath)
+	assert.NoError(err)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	_, err = validCreateParams(context.Background(), testContainerID, bundlePath)
+	// bundle exists as a file, not a directory
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestSetupConsoleExistingConsolePathSuccessful(t *testing.T) {
+	assert := assert.New(t)
+	console, err := setupConsole(consolePathTest, "")
+
+	assert.NoError(err)
+	assert.Equal(console, consolePathTest, "Got %q, Expecting %q", console, consolePathTest)
+}
+
+func TestSetupConsoleExistingConsolePathAndConsoleSocketPathSuccessful(t *testing.T) {
+	assert := assert.New(t)
+	console, err := setupConsole(consolePathTest, consoleSocketPathTest)
+
+	assert.NoError(err)
+	assert.Equal(console, consolePathTest, "Got %q, Expecting %q", console, consolePathTest)
+}
+
+func TestSetupConsoleEmptyPathsSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	console, err := setupConsole("", "")
+	assert.NoError(err)
+	assert.Empty(console, "Console path should be empty, got %q instead", console)
+}
+
+func TestSetupConsoleExistingConsoleSocketPath(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "test-socket")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	sockName := filepath.Join(dir, "console.sock")
+
+	l, err := net.Listen("unix", sockName)
+	assert.NoError(err)
+
+	console, err := setupConsole("", sockName)
+	assert.NoError(err)
+
+	waitCh := make(chan error)
+	go func() {
+		conn, err1 := l.Accept()
+		if err != nil {
+			waitCh <- err1
+		}
+
+		uConn, ok := conn.(*net.UnixConn)
+		if !ok {
+			waitCh <- fmt.Errorf("casting to *net.UnixConn failed")
+		}
+
+		f, err1 := uConn.File()
+		if err != nil {
+			waitCh <- err1
+		}
+
+		_, err1 = utils.RecvFd(f)
+		waitCh <- err1
+	}()
+
+	assert.NotEmpty(console, "Console socket path should not be empty")
+
+	err = <-waitCh
+	assert.NoError(err)
+}
+
+func TestSetupConsoleNotExistingSocketPathFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	console, err := setupConsole("", "unknown-sock-path")
+	assert.Error(err, "This test should fail because the console socket path does not exist")
+	assert.Empty(console, "This test should fail because the console socket path does not exist")
+}
+
+func testNoNeedForOutput(t *testing.T, detach bool, tty bool, expected bool) {
+	assert := assert.New(t)
+	result := noNeedForOutput(detach, tty)
+
+	assert.Equal(result, expected)
+}
+
+func TestNoNeedForOutputDetachTrueTtyTrue(t *testing.T) {
+	testNoNeedForOutput(t, true, true, true)
+}
+
+func TestNoNeedForOutputDetachFalseTtyTrue(t *testing.T) {
+	testNoNeedForOutput(t, false, true, false)
+}
+
+func TestNoNeedForOutputDetachFalseTtyFalse(t *testing.T) {
+	testNoNeedForOutput(t, false, false, false)
+}
+
+func TestNoNeedForOutputDetachTrueTtyFalse(t *testing.T) {
+	testNoNeedForOutput(t, true, false, false)
+}
+
+func TestIsCgroupMounted(t *testing.T) {
+	assert := assert.New(t)
+
+	r := rand.New(rand.NewSource(time.Now().Unix()))
+	randPath := fmt.Sprintf("/path/to/random/%d", r.Int63())
+
+	assert.False(isCgroupMounted(randPath), "%s does not exist", randPath)
+
+	assert.False(isCgroupMounted(os.TempDir()), "%s is not a cgroup", os.TempDir())
+
+	cgroupsDirPath = ""
+	cgroupRootPath, err := getCgroupsDirPath(procMountInfo)
+	if err != nil {
+		assert.NoError(err)
+	}
+	memoryCgroupPath := filepath.Join(cgroupRootPath, "memory")
+	if _, err := os.Stat(memoryCgroupPath); os.IsNotExist(err) {
+		t.Skipf("memory cgroup does not exist: %s", memoryCgroupPath)
+	}
+
+	assert.True(isCgroupMounted(memoryCgroupPath), "%s is a cgroup", memoryCgroupPath)
+}
+
+func TestGetCgroupsDirPath(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		contents       string
+		expectedResult string
+		expectError    bool
+	}
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		assert.NoError(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// make sure tested cgroupsDirPath is existed
+	testedCgroupDir := filepath.Join(dir, "weirdCgroup")
+	err = os.Mkdir(testedCgroupDir, testDirMode)
+	assert.NoError(err)
+
+	weirdCgroupPath := filepath.Join(testedCgroupDir, "memory")
+
+	data := []testData{
+		{fmt.Sprintf("num1 num2 num3 / %s num6 num7 - cgroup cgroup rw,memory", weirdCgroupPath), testedCgroupDir, false},
+		// cgroup mount is not properly formated, if fields post - less than 3
+		{fmt.Sprintf("num1 num2 num3 / %s num6 num7 - cgroup cgroup ", weirdCgroupPath), "", true},
+		{"a a a a a a a - b c d", "", true},
+		{"a \na b \na b c\na b c d", "", true},
+		{"", "", true},
+	}
+
+	file := filepath.Join(dir, "mountinfo")
+
+	//file does not exist, should error here
+	_, err = getCgroupsDirPath(file)
+	assert.Error(err)
+
+	for _, d := range data {
+		err := ioutil.WriteFile(file, []byte(d.contents), testFileMode)
+		assert.NoError(err)
+
+		cgroupsDirPath = ""
+		path, err := getCgroupsDirPath(file)
+		if d.expectError {
+			assert.Error(err, fmt.Sprintf("got %q, test data: %+v", path, d))
+		} else {
+			assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", path, d))
+		}
+
+		assert.Equal(d.expectedResult, path)
+	}
+}

src/runtime/cli/pause.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var noteText = `Use "` + name + ` list" to identify container statuses.`
+
+var pauseCLICommand = cli.Command{
+	Name:  "pause",
+	Usage: "suspend all processes in a container",
+	ArgsUsage: `<container-id>
+
+Where "<container-id>" is the container name to be paused.`,
+	Description: `The pause command suspends all processes in a container.
+
+	` + noteText,
+	Action: pause,
+}
+
+var resumeCLICommand = cli.Command{
+	Name:  "resume",
+	Usage: "unpause all previously paused processes in a container",
+	ArgsUsage: `<container-id>
+
+Where "<container-id>" is the container name to be resumed.`,
+	Description: `The resume command unpauses all processes in a container.
+
+	` + noteText,
+	Action: resume,
+}
+
+func pause(c *cli.Context) error {
+	return toggle(c, true)
+}
+
+func resume(c *cli.Context) error {
+	return toggle(c, false)
+}
+
+func toggle(c *cli.Context, pause bool) error {
+	ctx, err := cliContextToContext(c)
+	if err != nil {
+		return err
+	}
+
+	return toggleContainerPause(ctx, c.Args().First(), pause)
+}
+
+func toggleContainerPause(ctx context.Context, containerID string, pause bool) (err error) {
+	span, _ := katautils.Trace(ctx, "pause")
+	defer span.Finish()
+	span.SetTag("pause", pause)
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+	span.SetTag("sandbox", sandboxID)
+
+	if pause {
+		err = vci.PauseContainer(ctx, sandboxID, containerID)
+	} else {
+		err = vci.ResumeContainer(ctx, sandboxID, containerID)
+	}
+
+	return err
+}

src/runtime/cli/pause_test.go

@@ -0,0 +1,175 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+)
+
+var (
+	testPauseContainerFuncReturnNil = func(ctx context.Context, sandboxID, containerID string) error {
+		return nil
+	}
+
+	testResumeContainerFuncReturnNil = func(ctx context.Context, sandboxID, containerID string) error {
+		return nil
+	}
+)
+
+func TestPauseCLIFunctionSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	testingImpl.PauseContainerFunc = testPauseContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.PauseContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, pauseCLICommand, set, false)
+}
+
+func TestPauseCLIFunctionContainerNotExistFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	testingImpl.PauseContainerFunc = testPauseContainerFuncReturnNil
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	defer func() {
+		testingImpl.PauseContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, pauseCLICommand, set, true)
+}
+
+func TestPauseCLIFunctionPauseContainerFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, pauseCLICommand, set, true)
+}
+
+func TestResumeCLIFunctionSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	testingImpl.ResumeContainerFunc = testResumeContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.ResumeContainerFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, resumeCLICommand, set, false)
+}
+
+func TestResumeCLIFunctionContainerNotExistFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	testingImpl.ResumeContainerFunc = testResumeContainerFuncReturnNil
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	defer func() {
+		testingImpl.ResumeContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, resumeCLICommand, set, true)
+}
+
+func TestResumeCLIFunctionPauseContainerFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	state := types.ContainerState{
+		State: types.StateRunning,
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	set := flag.NewFlagSet("", 0)
+	set.Parse([]string{testContainerID})
+
+	execCLICommandFunc(assert, resumeCLICommand, set, true)
+}

src/runtime/cli/ps.go

@@ -0,0 +1,106 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var psCLICommand = cli.Command{
+	Name:      "ps",
+	Usage:     "ps displays the processes running inside a container",
+	ArgsUsage: `<container-id> [ps options]`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "format, f",
+			Value: "table",
+			Usage: `select one of: ` + formatOptions,
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		if !context.Args().Present() {
+			return fmt.Errorf("Missing container ID, should at least provide one")
+		}
+
+		var args []string
+		if len(context.Args()) > 1 {
+			// [1:] is to remove container_id:
+			// context.Args(): [container_id ps_arg1 ps_arg2 ...]
+			// args:           [ps_arg1 ps_arg2 ...]
+			args = context.Args()[1:]
+		}
+
+		return ps(ctx, context.Args().First(), context.String("format"), args)
+	},
+	SkipArgReorder: true,
+}
+
+func ps(ctx context.Context, containerID, format string, args []string) error {
+	span, _ := katautils.Trace(ctx, "ps")
+	defer span.Finish()
+
+	if containerID == "" {
+		return fmt.Errorf("Missing container ID")
+	}
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+	span.SetTag("sandbox", sandboxID)
+
+	// container MUST be running
+	if status.State.State != types.StateRunning {
+		return fmt.Errorf("Container %s is not running", containerID)
+	}
+
+	var options vc.ProcessListOptions
+
+	options.Args = args
+	if len(options.Args) == 0 {
+		options.Args = []string{"-ef"}
+	}
+
+	options.Format = format
+
+	msg, err := vci.ProcessListContainer(ctx, sandboxID, containerID, options)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print(string(msg))
+
+	return nil
+}

src/runtime/cli/ps_test.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func TestPSCLIAction(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := flag.NewFlagSet("flag", flag.ContinueOnError)
+	flagSet.Parse([]string{"runtime"})
+
+	// create a new fake context
+	ctx := createCLIContext(flagSet)
+
+	// get Action function
+	actionFunc, ok := psCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	err := actionFunc(ctx)
+	assert.Error(err, "Missing container ID")
+}
+
+func TestPSFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	// inexistent container
+	err = ps(context.Background(), "xyz123abc", "json", []string{"-ef"})
+	assert.Error(err)
+
+	// container is not running
+	err = ps(context.Background(), sandbox.ID(), "json", []string{"-ef"})
+	assert.Error(err)
+}
+
+func TestPSSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			State: types.ContainerState{
+				State: types.StateRunning,
+			},
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	testingImpl.ProcessListContainerFunc = func(ctx context.Context, sandboxID, containerID string, options vc.ProcessListOptions) (vc.ProcessList, error) {
+		return []byte("echo,sleep,grep"), nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.ProcessListContainerFunc = nil
+	}()
+
+	err = ps(context.Background(), sandbox.ID(), "json", []string{})
+	assert.NoError(err)
+}

src/runtime/cli/run.go

@@ -0,0 +1,129 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"syscall"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/urfave/cli"
+)
+
+var runCLICommand = cli.Command{
+	Name:  "run",
+	Usage: "create and run a container",
+	ArgsUsage: `<container-id>
+
+   <container-id> is your name for the instance of the container that you
+   are starting. The name you provide for the container instance must be unique
+   on your host.`,
+	Description: `The run command creates an instance of a container for a bundle. The bundle
+   is a directory with a specification file named "config.json" and a root
+   filesystem.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "bundle, b",
+			Value: "",
+			Usage: `path to the root of the bundle directory, defaults to the current directory`,
+		},
+		cli.StringFlag{
+			Name:  "console",
+			Value: "",
+			Usage: "path to a pseudo terminal",
+		},
+		cli.StringFlag{
+			Name:  "console-socket",
+			Value: "",
+			Usage: "path to an AF_UNIX socket which will receive a file descriptor referencing the master end of the console's pseudoterminal",
+		},
+		cli.StringFlag{
+			Name:  "pid-file",
+			Value: "",
+			Usage: "specify the file to write the process id to",
+		},
+		cli.BoolFlag{
+			Name:  "detach, d",
+			Usage: "detach from the container's process",
+		},
+		cli.BoolFlag{
+			Name:  "no-pivot",
+			Usage: "warning: this flag is meaningless to kata-runtime, just defined in order to be compatible with docker in ramdisk",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("invalid runtime config")
+		}
+
+		return run(ctx, context.Args().First(),
+			context.String("bundle"),
+			context.String("console"),
+			context.String("console-socket"),
+			context.String("pid-file"),
+			context.Bool("detach"),
+			context.Bool("systemd-cgroup"),
+			runtimeConfig)
+	},
+}
+
+func run(ctx context.Context, containerID, bundle, console, consoleSocket, pidFile string, detach, systemdCgroup bool,
+	runtimeConfig oci.RuntimeConfig) error {
+	span, ctx := katautils.Trace(ctx, "run")
+	defer span.Finish()
+
+	consolePath, err := setupConsole(console, consoleSocket)
+	if err != nil {
+		return err
+	}
+
+	if err := create(ctx, containerID, bundle, consolePath, pidFile, detach, systemdCgroup, runtimeConfig); err != nil {
+		return err
+	}
+
+	sandbox, err := start(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	if detach {
+		return nil
+	}
+
+	containers := sandbox.GetAllContainers()
+	if len(containers) == 0 {
+		return fmt.Errorf("There are no containers running in the sandbox: %s", sandbox.ID())
+	}
+
+	p, err := os.FindProcess(containers[0].GetPid())
+	if err != nil {
+		return err
+	}
+
+	ps, err := p.Wait()
+	if err != nil {
+		return fmt.Errorf("Process state %s: %s", ps.String(), err)
+	}
+
+	// delete container's resources
+	if err := delete(ctx, sandbox.ID(), true); err != nil {
+		return err
+	}
+
+	//runtime should forward container exit code to the system
+	return cli.NewExitError("", ps.Sys().(syscall.WaitStatus).ExitStatus())
+}

src/runtime/cli/run_test.go

@@ -0,0 +1,670 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+
+	ktu "github.com/kata-containers/runtime/pkg/katatestutils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+)
+
+func TestRunCliAction(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := flag.NewFlagSet("flag", flag.ContinueOnError)
+	flagSet.Parse([]string{"runtime"})
+
+	// create a new fake context
+	ctx := createCLIContext(flagSet)
+
+	// get Action function
+	actionFunc, ok := runCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	err := actionFunc(ctx)
+	assert.Error(err, "missing runtime configuration")
+
+	// temporal dir to place container files
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	// create a new runtime config
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, "/dev/ptmx", true)
+	assert.NoError(err)
+
+	ctx.App.Metadata = map[string]interface{}{
+		"runtimeConfig": runtimeConfig,
+	}
+
+	err = actionFunc(ctx)
+	assert.Error(err, "run without args")
+}
+
+func TestRunInvalidArgs(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+		},
+	}
+
+	// fake functions used to run containers
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+	}()
+
+	// temporal dir to place container files
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	// create a new bundle
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = os.MkdirAll(bundlePath, testDirMode)
+	assert.NoError(err)
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	// pid file
+	pidFilePath := filepath.Join(tmpdir, "pid")
+
+	// console file
+	consolePath := "/dev/ptmx"
+
+	// inexistent path
+	inexistentPath := "/this/path/does/not/exist"
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, consolePath, true)
+	assert.NoError(err)
+
+	type testArgs struct {
+		containerID   string
+		bundle        string
+		console       string
+		consoleSocket string
+		pidFile       string
+		detach        bool
+		systemdCgroup bool
+		runtimeConfig oci.RuntimeConfig
+	}
+
+	args := []testArgs{
+		{"", "", "", "", "", true, true, oci.RuntimeConfig{}},
+		{"", "", "", "", "", false, false, oci.RuntimeConfig{}},
+		{"", "", "", "", "", true, false, runtimeConfig},
+		{"", "", "", "", "", false, true, runtimeConfig},
+		{"", "", "", "", pidFilePath, false, false, runtimeConfig},
+		{"", "", "", "", inexistentPath, false, false, runtimeConfig},
+		{"", "", "", "", pidFilePath, false, true, runtimeConfig},
+		{"", "", "", inexistentPath, pidFilePath, false, true, runtimeConfig},
+		{"", "", inexistentPath, inexistentPath, pidFilePath, false, false, runtimeConfig},
+		{"", "", inexistentPath, "", pidFilePath, false, false, runtimeConfig},
+		{"", "", consolePath, "", pidFilePath, false, true, runtimeConfig},
+		{"", bundlePath, consolePath, "", pidFilePath, false, true, runtimeConfig},
+		{testContainerID, inexistentPath, consolePath, "", pidFilePath, false, true, oci.RuntimeConfig{}},
+		{testContainerID, inexistentPath, consolePath, "", inexistentPath, false, true, oci.RuntimeConfig{}},
+		{testContainerID, bundlePath, consolePath, "", pidFilePath, false, false, oci.RuntimeConfig{}},
+		{testContainerID, inexistentPath, consolePath, "", pidFilePath, false, false, runtimeConfig},
+		{testContainerID, inexistentPath, consolePath, "", inexistentPath, false, true, runtimeConfig},
+		{testContainerID, bundlePath, consolePath, "", pidFilePath, false, true, runtimeConfig},
+	}
+
+	for i, a := range args {
+		err := run(context.Background(), a.containerID, a.bundle, a.console, a.consoleSocket, a.pidFile, a.detach, a.systemdCgroup, a.runtimeConfig)
+		assert.Errorf(err, "test %d (%+v)", i, a)
+	}
+}
+
+type runContainerData struct {
+	pidFilePath   string
+	consolePath   string
+	bundlePath    string
+	spec          *specs.Spec
+	sandbox       *vcmock.Sandbox
+	runtimeConfig oci.RuntimeConfig
+	process       *os.Process
+	tmpDir        string
+}
+
+func testRunContainerSetup(t *testing.T) runContainerData {
+	assert := assert.New(t)
+
+	// create a fake container workload
+	workload := []string{"/bin/sleep", "10"}
+	cmd := exec.Command(workload[0], workload[1:]...)
+	err := cmd.Start()
+	assert.NoError(err, "unable to start fake container workload %+v: %s", workload, err)
+
+	// temporal dir to place container files
+	// Note - it is returned to the caller, who does the defer remove to clean up.
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+
+	// pid file
+	pidFilePath := filepath.Join(tmpdir, "pid")
+
+	// console file
+	consolePath := "/dev/ptmx"
+
+	// create a new bundle
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	// sandbox id and container id must be the same otherwise delete will not works
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockPid:     cmd.Process.Pid,
+			MockSandbox: sandbox,
+		},
+	}
+
+	// create a new runtime config
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, consolePath, true)
+	assert.NoError(err)
+
+	ociSpec, err := compatoci.ParseConfigJSON(bundlePath)
+	assert.NoError(err)
+
+	return runContainerData{
+		pidFilePath:   pidFilePath,
+		consolePath:   consolePath,
+		bundlePath:    bundlePath,
+		sandbox:       sandbox,
+		spec:          &ociSpec,
+		runtimeConfig: runtimeConfig,
+		process:       cmd.Process,
+		tmpDir:        tmpdir,
+	}
+}
+
+func TestRunContainerSuccessful(t *testing.T) {
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip(ktu.TestDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	d := testRunContainerSetup(t)
+	defer os.RemoveAll(d.tmpDir)
+
+	// this flags is used to detect if createSandboxFunc was called
+	flagCreate := false
+
+	// fake functions used to run containers
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		flagCreate = true
+		return d.sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return d.sandbox, nil
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		// return an empty list on create
+		if !flagCreate {
+			return vc.ContainerStatus{}, nil
+		}
+
+		// return a sandboxStatus with the container status
+		return vc.ContainerStatus{
+			ID: d.sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: d.spec,
+		}, nil
+	}
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		// now we can kill the fake container workload
+		err := d.process.Kill()
+		assert.NoError(err)
+
+		return d.sandbox.MockContainers[0], nil
+	}
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return d.sandbox, nil
+	}
+
+	testingImpl.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return d.sandbox.MockContainers[0], nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StartContainerFunc = nil
+		testingImpl.DeleteSandboxFunc = nil
+		testingImpl.DeleteContainerFunc = nil
+	}()
+
+	type errorTestArgs struct {
+		bundlePath string
+
+		// regex string for text of error message
+		errorRE string
+
+		// If true, expect a cli.ExitError, else expect any other type
+		// of error.
+		expectExitError bool
+	}
+
+	args := []errorTestArgs{
+		{"", "config.json: no such file or directory", false},
+		{d.bundlePath, "", true},
+	}
+
+	for i, a := range args {
+		err = run(context.Background(), d.sandbox.ID(), a.bundlePath, d.consolePath, "", d.pidFilePath, false, true, d.runtimeConfig)
+		assert.Errorf(err, "test args %d (%+v)", i, a)
+
+		if a.errorRE == "" {
+			assert.Empty(err.Error())
+		} else {
+			re := regexp.MustCompile(a.errorRE)
+			matches := re.FindAllStringSubmatch(err.Error(), -1)
+			assert.NotEmpty(matches)
+		}
+
+		e, ok := err.(*cli.ExitError)
+
+		if a.expectExitError {
+			// should return ExitError with the message and exit code
+			assert.Truef(ok, "test args %d (%+v): error should be a cli.ExitError: %s", i, a, err)
+			assert.NotZero(e.ExitCode())
+		} else {
+			assert.Falsef(ok, "test args %d (%+v): error should not be a cli.ExitError: %s", i, a, err)
+		}
+	}
+}
+
+func TestRunContainerDetachSuccessful(t *testing.T) {
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip(ktu.TestDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	d := testRunContainerSetup(t)
+	defer os.RemoveAll(d.tmpDir)
+
+	// this flags is used to detect if createSandboxFunc was called
+	flagCreate := false
+
+	// fake functions used to run containers
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		flagCreate = true
+		return d.sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return d.sandbox, nil
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		// return an empty list on create
+		if !flagCreate {
+			return vc.ContainerStatus{}, nil
+		}
+
+		// return a sandboxStatus with the container status
+		return vc.ContainerStatus{
+			ID: d.sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: d.spec,
+		}, nil
+	}
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		// now we can kill the fake container workload
+		err := d.process.Kill()
+		assert.NoError(err)
+
+		return d.sandbox.MockContainers[0], nil
+	}
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return d.sandbox, nil
+	}
+
+	testingImpl.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return d.sandbox.MockContainers[0], nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StartContainerFunc = nil
+		testingImpl.DeleteSandboxFunc = nil
+		testingImpl.DeleteContainerFunc = nil
+	}()
+
+	err = run(context.Background(), d.sandbox.ID(), d.bundlePath, d.consolePath, "", d.pidFilePath, true, true, d.runtimeConfig)
+
+	// should not return ExitError
+	assert.NoError(err)
+}
+
+func TestRunContainerDeleteFail(t *testing.T) {
+	assert := assert.New(t)
+
+	d := testRunContainerSetup(t)
+	defer os.RemoveAll(d.tmpDir)
+
+	// this flags is used to detect if createSandboxFunc was called
+	flagCreate := false
+
+	// fake functions used to run containers
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		flagCreate = true
+		return d.sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return d.sandbox, nil
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		// return an empty list on create
+		if !flagCreate {
+			return vc.ContainerStatus{}, nil
+		}
+
+		// return a sandboxStatus with the container status
+		return vc.ContainerStatus{
+			ID: d.sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: d.spec,
+		}, nil
+	}
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		// now we can kill the fake container workload
+		err := d.process.Kill()
+		assert.NoError(err)
+
+		return d.sandbox.MockContainers[0], nil
+	}
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		// return an error to provoke a failure in delete
+		return nil, fmt.Errorf("DeleteSandboxFunc")
+	}
+
+	testingImpl.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		// return an error to provoke a failure in delete
+		return d.sandbox.MockContainers[0], fmt.Errorf("DeleteContainerFunc")
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StartContainerFunc = nil
+		testingImpl.DeleteSandboxFunc = nil
+		testingImpl.DeleteContainerFunc = nil
+	}()
+
+	err = run(context.Background(), d.sandbox.ID(), d.bundlePath, d.consolePath, "", d.pidFilePath, false, true, d.runtimeConfig)
+
+	// should not return ExitError
+	err, ok := err.(*cli.ExitError)
+	assert.False(ok, "error should not be a cli.ExitError: %s", err)
+}
+
+func TestRunContainerWaitFail(t *testing.T) {
+	assert := assert.New(t)
+
+	d := testRunContainerSetup(t)
+	defer os.RemoveAll(d.tmpDir)
+
+	// this flags is used to detect if createSandboxFunc was called
+	flagCreate := false
+
+	// fake functions used to run containers
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		flagCreate = true
+		return d.sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return d.sandbox, nil
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		// return an empty list on create
+		if !flagCreate {
+			return vc.ContainerStatus{}, nil
+		}
+
+		// return a sandboxStatus with the container status
+		return vc.ContainerStatus{
+			ID: d.sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: d.spec,
+		}, nil
+	}
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		// now we can kill the fake container workload
+		err := d.process.Kill()
+		assert.NoError(err)
+
+		// change PID to provoke a failure in Wait
+		d.sandbox.MockContainers[0].MockPid = -1
+
+		return d.sandbox.MockContainers[0], nil
+	}
+
+	testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		// return an error to provoke a failure in delete
+		return nil, fmt.Errorf("DeleteSandboxFunc")
+	}
+
+	testingImpl.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		// return an error to provoke a failure in delete
+		return d.sandbox.MockContainers[0], fmt.Errorf("DeleteContainerFunc")
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StartContainerFunc = nil
+		testingImpl.DeleteSandboxFunc = nil
+		testingImpl.DeleteContainerFunc = nil
+	}()
+
+	err = run(context.Background(), d.sandbox.ID(), d.bundlePath, d.consolePath, "", d.pidFilePath, false, true, d.runtimeConfig)
+
+	// should not return ExitError
+	err, ok := err.(*cli.ExitError)
+	assert.False(ok, "error should not be a cli.ExitError: %s", err)
+}
+
+func TestRunContainerStartFail(t *testing.T) {
+	assert := assert.New(t)
+
+	d := testRunContainerSetup(t)
+	defer os.RemoveAll(d.tmpDir)
+
+	// now we can kill the fake container workload
+	err := d.process.Kill()
+	assert.NoError(err)
+
+	// this flags is used to detect if createSandboxFunc was called
+	flagCreate := false
+
+	// fake functions used to run containers
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		flagCreate = true
+		return d.sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		// start fails
+		return nil, fmt.Errorf("StartSandbox")
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		// return an empty list on create
+		if !flagCreate {
+			return vc.ContainerStatus{}, nil
+		}
+
+		// return a sandboxStatus with the container status
+		return vc.ContainerStatus{
+			ID: d.sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: d.spec,
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = run(context.Background(), d.sandbox.ID(), d.bundlePath, d.consolePath, "", d.pidFilePath, false, true, d.runtimeConfig)
+
+	// should not return ExitError
+	err, ok := err.(*cli.ExitError)
+	assert.False(ok, "error should not be a cli.ExitError: %s", err)
+}
+
+func TestRunContainerStartFailExistingContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	d := testRunContainerSetup(t)
+	defer os.RemoveAll(d.tmpDir)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		// return the container status
+		return vc.ContainerStatus{
+			ID: testContainerID,
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		// force no containers
+		sandbox.MockContainers = nil
+
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.CreateSandboxFunc = nil
+		testingImpl.StartSandboxFunc = nil
+	}()
+
+	err = run(context.Background(), d.sandbox.ID(), d.bundlePath, d.consolePath, "", d.pidFilePath, false, true, d.runtimeConfig)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}

src/runtime/cli/spec.go

@@ -0,0 +1,114 @@
+// Copyright (c) 2014,2015,2016,2017 Docker, Inc.
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/opencontainers/runc/libcontainer/specconv"
+	"github.com/urfave/cli"
+)
+
+var specCLICommand = cli.Command{
+	Name:      "spec",
+	Usage:     "create a new specification file",
+	ArgsUsage: "",
+	Description: `The spec command creates the new specification file named "` + specConfig + `" for
+the bundle.
+
+The spec generated is just a starter file. Editing of the spec is required to
+achieve desired results. For example, the newly generated spec includes an args
+parameter that is initially set to call the "sh" command when the container is
+started. Calling "sh" may work for an ubuntu container or busybox, but will not
+work for containers that do not include the "sh" program.
+
+EXAMPLE:
+  To run docker's hello-world container one needs to set the args parameter
+in the spec to call hello. This can be done using the sed command or a text
+editor. The following commands create a bundle for hello-world, change the
+default args parameter in the spec from "sh" to "/hello", then run the hello
+command in a new hello-world container named container1:
+
+    mkdir hello
+    cd hello
+    docker pull hello-world
+    docker export $(docker create hello-world) > hello-world.tar
+    mkdir rootfs
+    tar -C rootfs -xf hello-world.tar
+    kata-runtime spec
+    sed -i 's;"sh";"/hello";' ` + specConfig + `
+    kata-runtime run container1
+
+In the run command above, "container1" is the name for the instance of the
+container that you are starting. The name you provide for the container instance
+must be unique on your host.
+
+An alternative for generating a customized spec config is to use "oci-runtime-tool", the
+sub-command "oci-runtime-tool generate" has lots of options that can be used to do any
+customizations as you want, see runtime-tools (https://github.com/opencontainers/runtime-tools)
+to get more information.
+
+When starting a container through kata-runtime, kata-runtime needs root privilege. If not
+already running as root, you can use sudo to give kata-runtime root privilege. For
+example: "sudo kata-runtime start container1" will give kata-runtime root privilege to start the
+container on your host.
+
+Alternatively, you can start a rootless container, which has the ability to run
+without root privileges. For this to work, the specification file needs to be
+adjusted accordingly. You can pass the parameter --rootless to this command to
+generate a proper rootless spec file.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "bundle, b",
+			Value: "",
+			Usage: "path to the root of the bundle directory",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, _ := katautils.Trace(ctx, "spec")
+		defer span.Finish()
+
+		spec := specconv.Example()
+
+		checkNoFile := func(name string) error {
+			_, err := os.Stat(name)
+			if err == nil {
+				return fmt.Errorf("File %s exists. Remove it first", name)
+			}
+			if !os.IsNotExist(err) {
+				return err
+			}
+			return nil
+		}
+		bundle := context.String("bundle")
+		if bundle != "" {
+			if err := os.Chdir(bundle); err != nil {
+				return err
+			}
+		}
+		if err := checkNoFile(specConfig); err != nil {
+			return err
+		}
+		data, err := json.MarshalIndent(spec, "", "\t")
+		if err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(specConfig, data, 0640); err != nil {
+			return err
+		}
+		return nil
+	},
+}

src/runtime/cli/spec_test.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func TestSpecCliAction(t *testing.T) {
+	assert := assert.New(t)
+
+	actionFunc, ok := specCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	flagSet := flag.NewFlagSet("flag", flag.ContinueOnError)
+	ctx := createCLIContext(flagSet)
+	defer os.Remove(specConfig)
+	err := actionFunc(ctx)
+	assert.NoError(err)
+
+	pattern := "gid=5"
+	patternRootless := "uidMappings"
+	err = grep(pattern, specConfig)
+	assert.NoError(err)
+	err = grep(patternRootless, specConfig)
+	assert.Error(err)
+}

src/runtime/cli/start.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnot "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var startCLICommand = cli.Command{
+	Name:  "start",
+	Usage: "executes the user defined process in a created container",
+	ArgsUsage: `<container-id> [container-id...]
+
+   <container-id> is your name for the instance of the container that you
+   are starting. The name you provide for the container instance must be
+   unique on your host.`,
+	Description: `The start command executes the user defined process in a created container .`,
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		args := context.Args()
+		if !args.Present() {
+			return fmt.Errorf("Missing container ID, should at least provide one")
+		}
+
+		for _, cID := range []string(args) {
+			if _, err := start(ctx, cID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
+func start(ctx context.Context, containerID string) (vc.VCSandbox, error) {
+	span, _ := katautils.Trace(ctx, "start")
+	defer span.Finish()
+
+	kataLog = kataLog.WithField("container", containerID)
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	containerID = status.ID
+
+	kataLog = kataLog.WithFields(logrus.Fields{
+		"container": containerID,
+		"sandbox":   sandboxID,
+	})
+
+	setExternalLoggers(ctx, kataLog)
+	span.SetTag("container", containerID)
+	span.SetTag("sandbox", sandboxID)
+
+	containerType, err := oci.GetContainerType(status.Annotations)
+	if err != nil {
+		return nil, err
+	}
+
+	ociSpec, err := oci.GetOCIConfig(status)
+	if err != nil {
+		return nil, err
+	}
+
+	var sandbox vc.VCSandbox
+
+	if containerType.IsSandbox() {
+		s, err := vci.StartSandbox(ctx, sandboxID)
+		if err != nil {
+			return nil, err
+		}
+
+		sandbox = s
+	} else {
+		c, err := vci.StartContainer(ctx, sandboxID, containerID)
+		if err != nil {
+			return nil, err
+		}
+
+		sandbox = c.Sandbox()
+	}
+
+	// Run post-start OCI hooks.
+	err = katautils.EnterNetNS(sandbox.GetNetNs(), func() error {
+		return katautils.PostStartHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey])
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return sandbox, nil
+}

src/runtime/cli/start_test.go

@@ -0,0 +1,243 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+)
+
+func TestStartInvalidArgs(t *testing.T) {
+	assert := assert.New(t)
+
+	// Missing container id
+	_, err := start(context.Background(), "")
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	// Mock StatusContainer error
+	_, err = start(context.Background(), testContainerID)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	path, err = ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	katautils.SetCtrsMapTreePath(path)
+
+	// Container missing in container mapping
+	_, err = start(context.Background(), testContainerID)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestStartSandbox(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+			},
+			Spec: &specs.Spec{},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	_, err = start(context.Background(), sandbox.ID())
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StartSandboxFunc = nil
+	}()
+
+	_, err = start(context.Background(), sandbox.ID())
+	assert.Nil(err)
+}
+
+func TestStartMissingAnnotation(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          sandbox.ID(),
+			Annotations: map[string]string{},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	_, err = start(context.Background(), sandbox.ID())
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestStartContainerSucessFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: testContainerID,
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: &specs.Spec{},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	_, err = start(context.Background(), testContainerID)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return sandbox.MockContainers[0], nil
+	}
+
+	defer func() {
+		testingImpl.StartContainerFunc = nil
+	}()
+
+	_, err = start(context.Background(), testContainerID)
+	assert.Nil(err)
+}
+
+func TestStartCLIFunction(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := &flag.FlagSet{}
+	ctx := createCLIContext(flagSet)
+
+	fn, ok := startCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	// no container id in the Metadata
+	err := fn(ctx)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+
+	path, err := createTempContainerIDMapping("xyz", "xyz")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	flagSet = flag.NewFlagSet("container-id", flag.ContinueOnError)
+	flagSet.Parse([]string{"xyz"})
+	ctx = createCLIContext(flagSet)
+
+	err = fn(ctx)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestStartCLIFunctionSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: testContainerID,
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			Spec: &specs.Spec{},
+		}, nil
+	}
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return sandbox.MockContainers[0], nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.StartContainerFunc = nil
+	}()
+
+	fn, ok := startCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	flagSet := flag.NewFlagSet("test", 0)
+	flagSet.Parse([]string{testContainerID})
+	ctx := createCLIContext(flagSet)
+	assert.NotNil(ctx)
+
+	err = fn(ctx)
+	assert.NoError(err)
+}

src/runtime/cli/state.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/urfave/cli"
+)
+
+var stateCLICommand = cli.Command{
+	Name:  "state",
+	Usage: "output the state of a container",
+	ArgsUsage: `<container-id>
+
+   <container-id> is your name for the instance of the container`,
+	Description: `The state command outputs current state information for the
+instance of a container.`,
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		args := context.Args()
+		if len(args) != 1 {
+			return fmt.Errorf("Expecting only one container ID, got %d: %v", len(args), []string(args))
+		}
+
+		return state(ctx, args.First())
+	},
+}
+
+func state(ctx context.Context, containerID string) error {
+	span, _ := katautils.Trace(ctx, "state")
+	defer span.Finish()
+
+	kataLog = kataLog.WithField("container", containerID)
+	span.SetTag("container", containerID)
+
+	setExternalLoggers(ctx, kataLog)
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	status, _, err := getExistingContainerInfo(ctx, containerID)
+	if err != nil {
+		return err
+	}
+
+	// Convert the status to the expected State structure
+	state := oci.StatusToOCIState(status)
+
+	stateJSON, err := json.MarshalIndent(state, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	// Print stateJSON to stdout
+	fmt.Fprintf(os.Stdout, "%s", stateJSON)
+
+	return nil
+}

src/runtime/cli/state_test.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func TestStateCliAction(t *testing.T) {
+	assert := assert.New(t)
+
+	actionFunc, ok := stateCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	flagSet := flag.NewFlagSet("flag", flag.ContinueOnError)
+
+	// without container id
+	flagSet.Parse([]string{"runtime"})
+	ctx := createCLIContext(flagSet)
+	err := actionFunc(ctx)
+	assert.Error(err)
+
+	// with container id
+	flagSet.Parse([]string{"runtime", testContainerID})
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+}
+
+func TestStateSuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	// trying with an inexistent id
+	err = state(context.Background(), "123456789")
+	assert.Error(err)
+
+	path, err = createTempContainerIDMapping(testContainerID, sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: testContainerID,
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	err = state(context.Background(), testContainerID)
+	assert.NoError(err)
+}

src/runtime/cli/update.go

@@ -0,0 +1,288 @@
+// Copyright (c) 2016,2017 Docker, Inc.
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/docker/go-units"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func i64Ptr(i int64) *int64   { return &i }
+func u64Ptr(i uint64) *uint64 { return &i }
+func u16Ptr(i uint16) *uint16 { return &i }
+
+var updateCLICommand = cli.Command{
+	Name:      "update",
+	Usage:     "update container resource constraints",
+	ArgsUsage: `<container-id>`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "resources, r",
+			Value: "",
+			Usage: `path to the file containing the resources to update or '-' to read from the standard input
+
+The accepted format is as follow (unchanged values can be omitted):
+
+{
+  "memory": {
+    "limit": 0,
+    "reservation": 0,
+    "swap": 0,
+    "kernel": 0,
+    "kernelTCP": 0
+  },
+  "cpu": {
+    "shares": 0,
+    "quota": 0,
+    "period": 0,
+    "realtimeRuntime": 0,
+    "realtimePeriod": 0,
+    "cpus": "",
+    "mems": ""
+  },
+  "blockIO": {
+    "weight": 0
+  },
+  "pids": {
+    "limit": 0
+  }
+}
+
+Note: if data is to be read from a file or the standard input, all
+other options are ignored.
+`,
+		},
+
+		cli.IntFlag{
+			Name:  "blkio-weight",
+			Usage: "Specifies per cgroup weight, range is from 10 to 1000",
+		},
+		cli.StringFlag{
+			Name:  "cpu-period",
+			Usage: "CPU CFS period to be used for hardcapping (in usecs). 0 to use system default",
+		},
+		cli.StringFlag{
+			Name:  "cpu-quota",
+			Usage: "CPU CFS hardcap limit (in usecs). Allowed cpu time in a given period",
+		},
+		cli.StringFlag{
+			Name:  "cpu-share",
+			Usage: "CPU shares (relative weight vs. other containers)",
+		},
+		cli.StringFlag{
+			Name:  "cpu-rt-period",
+			Usage: "CPU realtime period to be used for hardcapping (in usecs). 0 to use system default",
+		},
+		cli.StringFlag{
+			Name:  "cpu-rt-runtime",
+			Usage: "CPU realtime hardcap limit (in usecs). Allowed cpu time in a given period",
+		},
+		cli.StringFlag{
+			Name:  "cpuset-cpus",
+			Usage: "CPU(s) to use",
+		},
+		cli.StringFlag{
+			Name:  "cpuset-mems",
+			Usage: "Memory node(s) to use",
+		},
+		cli.StringFlag{
+			Name:  "kernel-memory",
+			Usage: "Kernel memory limit (in bytes)",
+		},
+		cli.StringFlag{
+			Name:  "kernel-memory-tcp",
+			Usage: "Kernel memory limit (in bytes) for tcp buffer",
+		},
+		cli.StringFlag{
+			Name:  "memory",
+			Usage: "Memory limit (in bytes)",
+		},
+		cli.StringFlag{
+			Name:  "memory-reservation",
+			Usage: "Memory reservation or soft_limit (in bytes)",
+		},
+		cli.StringFlag{
+			Name:  "memory-swap",
+			Usage: "Total memory usage (memory + swap); set '-1' to enable unlimited swap",
+		},
+		cli.IntFlag{
+			Name:  "pids-limit",
+			Usage: "Maximum number of pids allowed in the container",
+		},
+		cli.StringFlag{
+			Name:  "l3-cache-schema",
+			Usage: "The string of Intel RDT/CAT L3 cache schema",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, _ := katautils.Trace(ctx, "update")
+		defer span.Finish()
+
+		if !context.Args().Present() {
+			return fmt.Errorf("Missing container ID, should at least provide one")
+		}
+
+		containerID := context.Args().First()
+
+		kataLog = kataLog.WithField("container", containerID)
+		setExternalLoggers(ctx, kataLog)
+		span.SetTag("container", containerID)
+
+		status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
+		if err != nil {
+			return err
+		}
+
+		containerID = status.ID
+
+		kataLog = kataLog.WithFields(logrus.Fields{
+			"container": containerID,
+			"sandbox":   sandboxID,
+		})
+
+		setExternalLoggers(ctx, kataLog)
+
+		span.SetTag("container", containerID)
+		span.SetTag("sandbox", sandboxID)
+
+		// container MUST be running
+		if state := status.State.State; !(state == types.StateRunning || state == types.StateReady) {
+			return fmt.Errorf("Container %s is not running or Ready, the state is %s", containerID, state)
+		}
+
+		r := specs.LinuxResources{
+			Memory: &specs.LinuxMemory{
+				Limit:       i64Ptr(0),
+				Reservation: i64Ptr(0),
+				Swap:        i64Ptr(0),
+				Kernel:      i64Ptr(0),
+				KernelTCP:   i64Ptr(0),
+			},
+			CPU: &specs.LinuxCPU{
+				Shares:          u64Ptr(0),
+				Quota:           i64Ptr(0),
+				Period:          u64Ptr(0),
+				RealtimeRuntime: i64Ptr(0),
+				RealtimePeriod:  u64Ptr(0),
+				Cpus:            "",
+				Mems:            "",
+			},
+			BlockIO: &specs.LinuxBlockIO{
+				Weight: u16Ptr(0),
+			},
+			Pids: &specs.LinuxPids{
+				Limit: 0,
+			},
+		}
+
+		if in := context.String("resources"); in != "" {
+			var (
+				f   *os.File
+				err error
+			)
+			switch in {
+			case "-":
+				f = os.Stdin
+			default:
+				f, err = os.Open(in)
+				if err != nil {
+					return err
+				}
+			}
+			err = json.NewDecoder(f).Decode(&r)
+			if err != nil {
+				return err
+			}
+		} else {
+			if val := context.Int("blkio-weight"); val != 0 {
+				r.BlockIO.Weight = u16Ptr(uint16(val))
+			}
+			if val := context.String("cpuset-cpus"); val != "" {
+				r.CPU.Cpus = val
+			}
+			if val := context.String("cpuset-mems"); val != "" {
+				r.CPU.Mems = val
+			}
+
+			for _, pair := range []struct {
+				opt  string
+				dest *uint64
+			}{
+
+				{"cpu-period", r.CPU.Period},
+				{"cpu-rt-period", r.CPU.RealtimePeriod},
+				{"cpu-share", r.CPU.Shares},
+			} {
+				if val := context.String(pair.opt); val != "" {
+					var err error
+					*pair.dest, err = strconv.ParseUint(val, 10, 64)
+					if err != nil {
+						return fmt.Errorf("invalid value for %s: %s", pair.opt, err)
+					}
+				}
+			}
+			for _, pair := range []struct {
+				opt  string
+				dest *int64
+			}{
+
+				{"cpu-quota", r.CPU.Quota},
+				{"cpu-rt-runtime", r.CPU.RealtimeRuntime},
+			} {
+				if val := context.String(pair.opt); val != "" {
+					var err error
+					*pair.dest, err = strconv.ParseInt(val, 10, 64)
+					if err != nil {
+						return fmt.Errorf("invalid value for %s: %s", pair.opt, err)
+					}
+				}
+			}
+			for _, pair := range []struct {
+				opt  string
+				dest *int64
+			}{
+				{"memory", r.Memory.Limit},
+				{"memory-swap", r.Memory.Swap},
+				{"kernel-memory", r.Memory.Kernel},
+				{"kernel-memory-tcp", r.Memory.KernelTCP},
+				{"memory-reservation", r.Memory.Reservation},
+			} {
+				if val := context.String(pair.opt); val != "" {
+					var v int64
+
+					if val != "-1" {
+						v, err = units.RAMInBytes(val)
+						if err != nil {
+							return fmt.Errorf("invalid value for %s: %s", pair.opt, err)
+						}
+					} else {
+						v = -1
+					}
+					*pair.dest = v
+				}
+			}
+			r.Pids.Limit = int64(context.Int("pids-limit"))
+		}
+
+		return vci.UpdateContainer(ctx, sandboxID, containerID, r)
+	},
+}

src/runtime/cli/update_test.go

@@ -0,0 +1,208 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/kata-containers/runtime/virtcontainers/types"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func TestUpdateCLIAction(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := flag.NewFlagSet("update", flag.ContinueOnError)
+	flagSet.Parse([]string{"resources"})
+
+	// create a new fake context
+	ctx := createCLIContext(flagSet)
+
+	// get Action function
+	actionFunc, ok := updateCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	err := actionFunc(ctx)
+	assert.Error(err, "Missing container ID")
+}
+
+func TestUpdateCLIFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	flagSet := flag.NewFlagSet("update", flag.ContinueOnError)
+	ctx := createCLIContext(flagSet)
+
+	actionFunc, ok := updateCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	// missing container ID
+	err := actionFunc(ctx)
+	assert.Error(err)
+
+	// container info
+	flagSet.Parse([]string{testContainerID})
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// not running
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// resources file does not exist
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			State: types.ContainerState{
+				State: types.StateRunning,
+			},
+		}, nil
+	}
+	testingImpl.UpdateContainerFunc = func(ctx context.Context, sandboxID, containerID string, resources specs.LinuxResources) error {
+		return nil
+	}
+	defer func() {
+		testingImpl.UpdateContainerFunc = nil
+	}()
+	flagSet.String("resources", "/abc/123/xyz/rgb", "")
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// json decode error
+	f, err := ioutil.TempFile("", "resources")
+	defer os.Remove(f.Name())
+	assert.NoError(err)
+	assert.NotNil(f)
+	f.WriteString("no json")
+	f.Close()
+	flagSet.Set("resources", f.Name())
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// ParseUint Error
+	flagSet = flag.NewFlagSet("update", flag.ContinueOnError)
+	flagSet.Parse([]string{testContainerID})
+	flagSet.String("cpu-period", "abcxyz", "")
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// ParseInt Error
+	flagSet = flag.NewFlagSet("update", flag.ContinueOnError)
+	flagSet.Parse([]string{testContainerID})
+	flagSet.String("cpu-quota", "abcxyz", "")
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+
+	// RAMInBytes Error
+	flagSet = flag.NewFlagSet("update", flag.ContinueOnError)
+	flagSet.Parse([]string{testContainerID})
+	flagSet.String("memory", "abcxyz", "")
+	ctx = createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.Error(err)
+}
+
+func TestUpdateCLISuccessful(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testContainerID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      sandbox.ID(),
+			MockSandbox: sandbox,
+		},
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+			State: types.ContainerState{
+				State: types.StateRunning,
+			},
+		}, nil
+	}
+	testingImpl.UpdateContainerFunc = func(ctx context.Context, sandboxID, containerID string, resources specs.LinuxResources) error {
+		return nil
+	}
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+		testingImpl.UpdateContainerFunc = nil
+	}()
+
+	path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	actionFunc, ok := updateCLICommand.Action.(func(ctx *cli.Context) error)
+	assert.True(ok)
+
+	flagSet := flag.NewFlagSet("update", flag.ContinueOnError)
+	flagSet.Parse([]string{testContainerID})
+	flagSet.Int("blkio-weight", 20, "")
+	flagSet.String("cpuset-cpus", "0-5", "")
+	flagSet.String("cpuset-mems", "0-5", "")
+	flagSet.String("cpu-period", "1000", "")
+	flagSet.String("cpu-rt-period", "1000", "")
+	flagSet.String("cpu-share", "1000", "")
+	flagSet.String("cpu-quota", "1000", "")
+	flagSet.String("cpu-rt-runtime", "1000", "")
+	flagSet.String("memory", "100M", "")
+	flagSet.String("memory-swap", "100M", "")
+	flagSet.String("kernel-memory", "100M", "")
+	flagSet.String("kernel-memory-tcp", "100M", "")
+	flagSet.String("memory-reservation", "100M", "")
+	ctx := createCLIContext(flagSet)
+	err = actionFunc(ctx)
+	assert.NoError(err)
+}

src/runtime/cli/utils.go

@@ -0,0 +1,203 @@
+// Copyright (c) 2014 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/blang/semver"
+	"github.com/kata-containers/runtime/pkg/katautils"
+)
+
+const (
+	unknown = "<<unknown>>"
+)
+
+// variables to allow tests to modify the values
+var (
+	procVersion = "/proc/version"
+	osRelease   = "/etc/os-release"
+
+	// Clear Linux has a different path (for stateless support)
+	osReleaseClr = "/usr/lib/os-release"
+
+	unknownVersionInfo = VersionInfo{
+		Semver: unknown,
+		Commit: unknown,
+	}
+)
+
+func getKernelVersion() (string, error) {
+	contents, err := katautils.GetFileContents(procVersion)
+	if err != nil {
+		return "", err
+	}
+
+	fields := strings.Fields(contents)
+
+	if len(fields) < 3 {
+		return "", fmt.Errorf("unexpected contents in %v", procVersion)
+	}
+
+	version := fields[2]
+
+	return version, nil
+}
+
+// getDistroDetails returns the distributions name and version string.
+// If it is not possible to determine both values an error is
+// returned.
+func getDistroDetails() (name, version string, err error) {
+	files := []string{osRelease, osReleaseClr}
+	name = ""
+	version = ""
+
+	for _, file := range files {
+		contents, err := katautils.GetFileContents(file)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+
+			return "", "", err
+		}
+
+		lines := strings.Split(contents, "\n")
+
+		for _, line := range lines {
+			if strings.HasPrefix(line, "NAME=") && name == "" {
+				fields := strings.Split(line, "=")
+				name = strings.Trim(fields[1], `"`)
+			} else if strings.HasPrefix(line, "VERSION_ID=") && version == "" {
+				fields := strings.Split(line, "=")
+				version = strings.Trim(fields[1], `"`)
+			}
+		}
+
+		if name != "" && version != "" {
+			return name, version, nil
+		}
+	}
+
+	if name == "" {
+		name = unknown
+	}
+
+	if version == "" {
+		version = unknown
+	}
+
+	return name, version, nil
+}
+
+// genericGetCPUDetails returns the vendor and model of the CPU.
+// If it is not possible to determine both values an error is
+// returned.
+func genericGetCPUDetails() (vendor, model string, err error) {
+	cpuinfo, err := getCPUInfo(procCPUInfo)
+	if err != nil {
+		return "", "", err
+	}
+
+	lines := strings.Split(cpuinfo, "\n")
+
+	for _, line := range lines {
+		if archCPUVendorField != "" {
+			if strings.HasPrefix(line, archCPUVendorField) {
+				fields := strings.Split(line, ":")
+				if len(fields) > 1 {
+					vendor = strings.TrimSpace(fields[1])
+				}
+			}
+		}
+
+		if archCPUModelField != "" {
+			if strings.HasPrefix(line, archCPUModelField) {
+				fields := strings.Split(line, ":")
+				if len(fields) > 1 {
+					model = strings.TrimSpace(fields[1])
+				}
+			}
+		}
+	}
+
+	if archCPUVendorField != "" && vendor == "" {
+		return "", "", fmt.Errorf("cannot find vendor field in file %v", procCPUInfo)
+	}
+
+	// model name is optional
+	if archCPUModelField != "" && model == "" {
+		return "", "", fmt.Errorf("cannot find model field in file %v", procCPUInfo)
+	}
+
+	return vendor, model, nil
+}
+
+// from runC
+// parseBoolOrAuto returns (nil, nil) if s is empty or "auto"
+func parseBoolOrAuto(s string) (*bool, error) {
+	if s == "" || strings.ToLower(s) == "auto" {
+		return nil, nil
+	}
+	b, err := strconv.ParseBool(s)
+	return &b, err
+}
+
+// constructVersionInfo constructs VersionInfo-type value from a version string
+// in the format of "Kata-Component version Major.Minor.Patch-rc_xxx-Commit".
+func constructVersionInfo(version string) VersionInfo {
+	fields := strings.Split(version, " ")
+	realVersion := fields[len(fields)-1]
+
+	sv, err := semver.Make(realVersion)
+	if err != nil {
+		return unknownVersionInfo
+	}
+
+	pres := strings.Split(sv.Pre[0].VersionStr, "-")
+
+	// version contains Commit info.
+	if len(pres) > 1 {
+		return VersionInfo{
+			Semver: realVersion,
+			Major:  sv.Major,
+			Minor:  sv.Minor,
+			Patch:  sv.Patch,
+			Commit: pres[1],
+		}
+	}
+
+	return VersionInfo{
+		Semver: realVersion,
+		Major:  sv.Major,
+		Minor:  sv.Minor,
+		Patch:  sv.Patch,
+		Commit: unknown,
+	}
+
+}
+
+func versionEqual(a VersionInfo, b VersionInfo) bool {
+	av, err := semver.Make(a.Semver)
+	if err != nil {
+		return false
+	}
+
+	bv, err := semver.Make(b.Semver)
+	if err != nil {
+		return false
+	}
+
+	if av.Major == bv.Major && av.Minor == bv.Minor && av.Patch == bv.Patch {
+		return true
+	}
+
+	return false
+}

src/runtime/cli/utils_arch_base.go

@@ -0,0 +1,10 @@
+// +build !s390x
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+func archConvertStatFs(cgroupFsType int) int64 {
+	return int64(cgroupFsType)
+}

src/runtime/cli/utils_s390x.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+func archConvertStatFs(cgroupFsType int) uint32 {
+	return uint32(cgroupFsType)
+}

src/runtime/cli/utils_test.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileExists(t *testing.T) {
+	dir, err := ioutil.TempDir(testDir, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "foo")
+
+	assert.False(t, katautils.FileExists(file),
+		fmt.Sprintf("File %q should not exist", file))
+
+	err = createEmptyFile(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.True(t, katautils.FileExists(file),
+		fmt.Sprintf("File %q should exist", file))
+}
+
+func TestGetKernelVersion(t *testing.T) {
+	type testData struct {
+		contents        string
+		expectedVersion string
+		expectError     bool
+	}
+
+	const validVersion = "1.2.3-4.5.x86_64"
+	validContents := fmt.Sprintf("Linux version %s blah blah blah ...", validVersion)
+
+	data := []testData{
+		{"", "", true},
+		{"invalid contents", "", true},
+		{"a b c", "c", false},
+		{validContents, validVersion, false},
+	}
+
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	subDir := filepath.Join(tmpdir, "subdir")
+	err = os.MkdirAll(subDir, testDirMode)
+	assert.NoError(t, err)
+
+	_, err = getKernelVersion()
+	assert.Error(t, err)
+
+	file := filepath.Join(tmpdir, "proc-version")
+
+	// override
+	procVersion = file
+
+	_, err = getKernelVersion()
+	// ENOENT
+	assert.Error(t, err)
+	assert.True(t, os.IsNotExist(err))
+
+	for _, d := range data {
+		err := createFile(file, d.contents)
+		assert.NoError(t, err)
+
+		version, err := getKernelVersion()
+		if d.expectError {
+			assert.Error(t, err, fmt.Sprintf("%+v", d))
+			continue
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("%+v", d))
+			assert.Equal(t, d.expectedVersion, version)
+		}
+	}
+}
+
+func TestGetDistroDetails(t *testing.T) {
+	type testData struct {
+		clrContents     string
+		nonClrContents  string
+		expectedName    string
+		expectedVersion string
+		expectError     bool
+	}
+
+	const unknown = "<<unknown>>"
+
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	testOSRelease := filepath.Join(tmpdir, "os-release")
+	testOSReleaseClr := filepath.Join(tmpdir, "os-release-clr")
+
+	const clrExpectedName = "clr"
+	const clrExpectedVersion = "1.2.3-4"
+	clrContents := fmt.Sprintf(`
+HELLO=world
+NAME="%s"
+FOO=bar
+VERSION_ID="%s"
+`, clrExpectedName, clrExpectedVersion)
+
+	const nonClrExpectedName = "not-clr"
+	const nonClrExpectedVersion = "999"
+	nonClrContents := fmt.Sprintf(`
+HELLO=world
+NAME="%s"
+FOO=bar
+VERSION_ID="%s"
+`, nonClrExpectedName, nonClrExpectedVersion)
+
+	subDir := filepath.Join(tmpdir, "subdir")
+	err = os.MkdirAll(subDir, testDirMode)
+	assert.NoError(t, err)
+
+	// override
+	osRelease = subDir
+
+	_, _, err = getDistroDetails()
+	assert.Error(t, err)
+
+	// override
+	osRelease = testOSRelease
+	osReleaseClr = testOSReleaseClr
+
+	_, _, err = getDistroDetails()
+	// ENOENT
+	assert.NoError(t, err)
+
+	data := []testData{
+		{"", "", unknown, unknown, false},
+		{"invalid", "", unknown, unknown, false},
+		{clrContents, "", clrExpectedName, clrExpectedVersion, false},
+		{"", nonClrContents, nonClrExpectedName, nonClrExpectedVersion, false},
+		{clrContents, nonClrContents, nonClrExpectedName, nonClrExpectedVersion, false},
+	}
+
+	for _, d := range data {
+		err := createFile(osRelease, d.nonClrContents)
+		assert.NoError(t, err)
+
+		err = createFile(osReleaseClr, d.clrContents)
+		assert.NoError(t, err)
+
+		name, version, err := getDistroDetails()
+		if d.expectError {
+			assert.Error(t, err, fmt.Sprintf("%+v", d))
+			continue
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("%+v", d))
+			assert.Equal(t, d.expectedName, name)
+			assert.Equal(t, d.expectedVersion, version)
+		}
+	}
+}
+
+func TestUtilsRunCommand(t *testing.T) {
+	output, err := katautils.RunCommand([]string{"true"})
+	assert.NoError(t, err)
+	assert.Equal(t, "", output)
+}
+
+func TestUtilsRunCommandCaptureStdout(t *testing.T) {
+	output, err := katautils.RunCommand([]string{"echo", "hello"})
+	assert.NoError(t, err)
+	assert.Equal(t, "hello", output)
+}
+
+func TestUtilsRunCommandIgnoreStderr(t *testing.T) {
+	args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"}
+
+	output, err := katautils.RunCommand(args)
+	assert.NoError(t, err)
+	assert.Equal(t, "", output)
+}
+
+func TestUtilsRunCommandInvalidCmds(t *testing.T) {
+	invalidCommands := [][]string{
+		{""},
+		{"", ""},
+		{" "},
+		{" ", " "},
+		{" ", ""},
+		{"\\"},
+		{"/"},
+		{"/.."},
+		{"../"},
+		{"/tmp"},
+		{"\t"},
+		{"\n"},
+		{"false"},
+	}
+
+	for _, args := range invalidCommands {
+		output, err := katautils.RunCommand(args)
+		assert.Error(t, err)
+		assert.Equal(t, "", output)
+	}
+}

src/runtime/cli/version.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/urfave/cli"
+)
+
+var versionCLICommand = cli.Command{
+	Name:  "version",
+	Usage: "display version details",
+	Action: func(context *cli.Context) error {
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+
+		span, _ := katautils.Trace(ctx, "version")
+		defer span.Finish()
+
+		cli.VersionPrinter(context)
+		return nil
+	},
+}

src/runtime/cli/version_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func TestVersion(t *testing.T) {
+	const testAppName = "foo"
+	const testAppVersion = "0.1.0"
+
+	resetCLIGlobals()
+
+	savedRuntimeVersionFunc := runtimeVersion
+
+	defer func() {
+		runtimeVersion = savedRuntimeVersionFunc
+	}()
+
+	runtimeVersion := func() string {
+		return testAppVersion
+	}
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = testAppName
+	ctx.App.Version = runtimeVersion()
+
+	fn, ok := versionCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(t, ok)
+
+	tmpfile, err := ioutil.TempFile("", "")
+	assert.NoError(t, err)
+	defer os.Remove(tmpfile.Name())
+
+	ctx.App.Writer = tmpfile
+
+	err = fn(ctx)
+	assert.NoError(t, err)
+
+	pattern := fmt.Sprintf("%s.*version.*%s", testAppName, testAppVersion)
+	err = grep(pattern, tmpfile.Name())
+	assert.NoError(t, err)
+}