Merge pull request #572 from hyperhq/shimv2

Implement containerd shim v2 API for Kata Containers

Gopkg.lock

@@ -9,6 +9,37 @@
   revision = "b26d9c308763d68093482582cea63d69be07a0f0"
   version = "v0.3.0"
 
+[[projects]]
+  digest = "1:2be791e7b333ff7c06f8fb3dc18a7d70580e9399dbdffd352621d067ff260b6e"
+  name = "github.com/Microsoft/go-winio"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "97e4973ce50b2ff5f09635a57e2b88a037aae829"
+  version = "v0.4.11"
+
+[[projects]]
+  digest = "1:46df22fa970c4b4384851338ada3a67f32d1c1400d27aa69b09e493dd01bb8bf"
+  name = "github.com/Microsoft/hcsshim"
+  packages = [
+    ".",
+    "internal/guestrequest",
+    "internal/guid",
+    "internal/hcs",
+    "internal/hcserror",
+    "internal/hns",
+    "internal/interop",
+    "internal/longpath",
+    "internal/mergemaps",
+    "internal/safefile",
+    "internal/schema1",
+    "internal/schema2",
+    "internal/timeout",
+    "internal/wclayer",
+  ]
+  pruneopts = "NUT"
+  revision = "4f64a598035b09da04155f7dfd76b63edf04fca1"
+  version = "v0.8.1"
+
 [[projects]]
   digest = "1:3ac89be767202cf44b33eec1b41b3c6255ec1c79d893304ff621cceada4e53d0"
   name = "github.com/clearcontainers/proxy"
@@ -35,6 +66,34 @@
   pruneopts = "NUT"
   revision = "5017d4e9a9cf2d4381db99eacd9baf84b95bfb14"
 
+[[projects]]
+  branch = "master"
+  digest = "1:da4daad2ec1737eec4ebeeed7afedb631711f96bbac0c361a17a4d0369d00c6d"
+  name = "github.com/containerd/console"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f"
+
+[[projects]]
+  digest = "1:d9120dea4d91818f1c859242cb96825faf6d375a4e0231263ecaec5905143757"
+  name = "github.com/containerd/containerd"
+  packages = [
+    "api/events",
+    "api/types",
+    "api/types/task",
+    "errdefs",
+    "events",
+    "log",
+    "mount",
+    "namespaces",
+    "runtime",
+    "runtime/v2/shim",
+    "runtime/v2/task",
+    "sys",
+  ]
+  pruneopts = "NUT"
+  revision = "29eab28b8e4e18231b6b2f077ab653c719d25dd5"
+
 [[projects]]
   digest = "1:3d1a50e9f27c661df8c5552e7f2f6b9d2a8b641c65aeac7373f8a5c60d9f6856"
   name = "github.com/containerd/cri-containerd"
@@ -42,6 +101,38 @@
   pruneopts = "NUT"
   revision = "3d382e2f5dabe3bae62ceb9ded56bdee847008ee"
 
+[[projects]]
+  branch = "master"
+  digest = "1:a16d601cf7295c29ccc3e7711be788bca1376e1f22e0826929f8883ed36c7c99"
+  name = "github.com/containerd/fifo"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "3d5202aec260678c48179c56f40e6f38a095738c"
+
+[[projects]]
+  branch = "master"
+  digest = "1:4455e7f226d40cb6e52bb570cc246e667cdfc6370ec2e8c3b6d5b8993ad38eeb"
+  name = "github.com/containerd/go-runc"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:1b7f9015632851c458c7b713d1a29c9d763c5a2bd85c4e3731139e198b2df2c8"
+  name = "github.com/containerd/ttrpc"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "f51df4475b76e0ab0315ee0684bef0703a070e6b"
+
+[[projects]]
+  branch = "master"
+  digest = "1:07ac073876dbf7df80789ba4c2959a969200b34875a34fc13848f76d60b51551"
+  name = "github.com/containerd/typeurl"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "461401dc8f19d80baa4b70178935e4501286c00b"
+
 [[projects]]
   digest = "1:827ed8a74e55981880c4d77f8472d638bceb899188104ba7bf24a9548fd97292"
   name = "github.com/containernetworking/cni"
@@ -186,12 +277,21 @@
   revision = "d0303fe809921458f417bcf828397a65db30a7e4"
 
 [[projects]]
-  digest = "1:7876a956f7aa42e9830566efbe4278e2963ae0c723b472a0dba27d5fffd066ab"
+  digest = "1:e0cc8395ea893c898ff5eb0850f4d9851c1f57c78c232304a026379a47a552d0"
+  name = "github.com/opencontainers/go-digest"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "279bed98673dd5bef374d3b6e4b09e2af76183bf"
+  version = "v1.0.0-rc1"
+
+[[projects]]
+  digest = "1:26537bc93f1e164bbc305117029de9933656ff81c4549609939212aca20a4710"
   name = "github.com/opencontainers/runc"
   packages = [
     "libcontainer/configs",
     "libcontainer/seccomp",
     "libcontainer/specconv",
+    "libcontainer/system",
     "libcontainer/utils",
   ]
   pruneopts = "NUT"
@@ -424,7 +524,18 @@
     "github.com/clearcontainers/proxy/api",
     "github.com/clearcontainers/proxy/client",
     "github.com/containerd/cgroups",
+    "github.com/containerd/containerd/api/events",
+    "github.com/containerd/containerd/api/types/task",
+    "github.com/containerd/containerd/errdefs",
+    "github.com/containerd/containerd/events",
+    "github.com/containerd/containerd/mount",
+    "github.com/containerd/containerd/namespaces",
+    "github.com/containerd/containerd/runtime",
+    "github.com/containerd/containerd/runtime/v2/shim",
+    "github.com/containerd/containerd/runtime/v2/task",
     "github.com/containerd/cri-containerd/pkg/annotations",
+    "github.com/containerd/fifo",
+    "github.com/containerd/typeurl",
     "github.com/containernetworking/plugins/pkg/ns",
     "github.com/dlespiau/covertool/pkg/cover",
     "github.com/docker/go-units",
@@ -443,6 +554,7 @@
     "github.com/opencontainers/runtime-spec/specs-go",
     "github.com/opentracing/opentracing-go",
     "github.com/opentracing/opentracing-go/log",
+    "github.com/pkg/errors",
     "github.com/safchain/ethtool",
     "github.com/sirupsen/logrus",
     "github.com/sirupsen/logrus/hooks/syslog",

Gopkg.toml

@@ -66,6 +66,10 @@
   name = "github.com/safchain/ethtool"
   revision = "79559b488d8848b53a8e34c330140c3fc37ee246"
 
+[[constraint]]
+ name = "github.com/containerd/containerd"
+ revision = "29eab28b8e4e18231b6b2f077ab653c719d25dd5"
+
 [[override]]
   branch = "master"
   name = "github.com/hashicorp/yamux"

Makefile

@@ -158,6 +158,10 @@ DEFHOTPLUGVFIOONROOTBUS := false
 SED = sed
 
 CLI_DIR = cli
+SHIMV2 = containerd-shim-kata-v2
+SHIMV2_OUTPUT = $(CURDIR)/$(SHIMV2)
+SHIMV2_DIR = $(CLI_DIR)/$(SHIMV2)
+
 SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$')
 VERSION := ${shell cat ./VERSION}
 COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
@@ -256,7 +260,9 @@ define SHOW_ARCH
   $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
 endef
 
-all: runtime netmon
+all: runtime containerd-shim-v2 netmon
+
+containerd-shim-v2: $(SHIMV2_OUTPUT)
 
 netmon: $(NETMON_TARGET_OUTPUT)
 
@@ -288,9 +294,6 @@ const project = "$(PROJECT_NAME)"
 // prefix used to denote non-standard CLI commands and options.
 const projectPrefix = "$(PROJECT_TYPE)"
 
-// systemdUnitName is the systemd(1) target used to launch the agent.
-const systemdUnitName = "$(PROJECT_TAG).target"
-
 // original URL for this project
 const projectURL = "$(PROJECT_URL)"
 
@@ -338,6 +341,9 @@ $(GENERATED_CONFIG): Makefile VERSION
 $(TARGET_OUTPUT): $(EXTRA_DEPS) $(SOURCES) $(GENERATED_GO_FILES) $(GENERATED_FILES) Makefile | show-summary
 	$(QUIET_BUILD)(cd $(CLI_DIR) && go build $(BUILDFLAGS) -o $@ .)
 
+$(SHIMV2_OUTPUT): $(TARGET_OUTPUT)
+	$(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && go build -i -o $@ .)
+
 .PHONY: \
 	check \
 	check-go-static \
@@ -416,11 +422,14 @@ check-go-static:
 coverage:
 	$(QUIET_TEST).ci/go-test.sh html-coverage
 
-install: default runtime install-scripts install-completions install-config install-bin install-bin-libexec
+install: default runtime install-scripts install-completions install-config install-bin install-containerd-shim-v2 install-bin-libexec
 
 install-bin: $(BINLIST)
 	$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR)))
 
+install-containerd-shim-v2: $(SHIMV2)
+	$(call INSTALL_EXEC,$<,$(BINDIR))
+
 install-bin-libexec: $(BINLIBEXECLIST)
 	$(foreach f,$(BINLIBEXECLIST),$(call INSTALL_EXEC,$f,$(PKGLIBEXECDIR)))
 
@@ -434,7 +443,7 @@ install-completions:
 	$(QUIET_INST)install --mode 0644 -D  $(BASH_COMPLETIONS) $(DESTDIR)/$(BASH_COMPLETIONSDIR)/$(notdir $(BASH_COMPLETIONS));
 
 clean:
-	$(QUIET_CLEAN)rm -f $(TARGET) $(NETMON_TARGET) $(CONFIG) $(GENERATED_GO_FILES) $(GENERATED_FILES) $(COLLECT_SCRIPT)
+	$(QUIET_CLEAN)rm -f $(TARGET) $(SHIMV2) $(NETMON_TARGET) $(CONFIG) $(GENERATED_GO_FILES) $(GENERATED_FILES) $(COLLECT_SCRIPT)
 
 show-usage: show-header
 	@printf "• Overview:\n"
@@ -497,6 +506,8 @@ show-summary: show-header
 	@printf "\tbinaries to install                   :\n"
 	@printf \
           "$(foreach b,$(sort $(BINLIST)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
+	@printf \
+          "$(foreach b,$(sort $(SHIMV2)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
 	@printf \
           "$(foreach b,$(sort $(BINLIBEXECLIST)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(PKGLIBEXECDIR)/$(b))\\\n"))"
 	@printf \

cli/containerd-shim-kata-v2/main.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/kata-containers/runtime/containerd-shim-v2"
+)
+
+func main() {
+	shim.Run("io.containerd.kata.v2", containerdshim.New)
+}

cli/create.go

@@ -11,10 +11,9 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"strings"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
-	vf "github.com/kata-containers/runtime/virtcontainers/factory"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/urfave/cli"
 )
@@ -87,44 +86,11 @@ var createCLICommand = cli.Command{
 	},
 }
 
-// Use a variable to allow tests to modify its value
-var getKernelParamsFunc = getKernelParams
-
-func handleFactory(ctx context.Context, runtimeConfig oci.RuntimeConfig) {
-	if !runtimeConfig.FactoryConfig.Template {
-		return
-	}
-
-	factoryConfig := vf.Config{
-		Template: true,
-		VMConfig: vc.VMConfig{
-			HypervisorType:   runtimeConfig.HypervisorType,
-			HypervisorConfig: runtimeConfig.HypervisorConfig,
-			AgentType:        runtimeConfig.AgentType,
-			AgentConfig:      runtimeConfig.AgentConfig,
-		},
-	}
-
-	kataLog.WithField("factory", factoryConfig).Info("load vm factory")
-
-	f, err := vf.NewFactory(ctx, factoryConfig, true)
-	if err != nil {
-		kataLog.WithError(err).Warn("load vm factory failed, about to create new one")
-		f, err = vf.NewFactory(ctx, factoryConfig, false)
-		if err != nil {
-			kataLog.WithError(err).Warn("create vm factory failed")
-			return
-		}
-	}
-
-	vci.SetFactory(ctx, f)
-}
-
 func create(ctx context.Context, containerID, bundlePath, console, pidFilePath string, detach, systemdCgroup bool,
 	runtimeConfig oci.RuntimeConfig) error {
 	var err error
 
-	span, ctx := trace(ctx, "create")
+	span, ctx := katautils.Trace(ctx, "create")
 	defer span.Finish()
 
 	kataLog = kataLog.WithField("container", containerID)
@@ -157,19 +123,19 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
 		return err
 	}
 
-	handleFactory(ctx, runtimeConfig)
+	katautils.HandleFactory(ctx, vci, &runtimeConfig)
 
 	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
 
 	var process vc.Process
 	switch containerType {
 	case vc.PodSandbox:
-		process, err = createSandbox(ctx, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup)
+		_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
 		if err != nil {
 			return err
 		}
 	case vc.PodContainer:
-		process, err = createContainer(ctx, ociSpec, containerID, bundlePath, console, disableOutput)
+		process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, containerID, bundlePath, console, disableOutput, false)
 		if err != nil {
 			return err
 		}
@@ -181,184 +147,8 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
 	return createPIDFile(ctx, pidFilePath, process.Pid)
 }
 
-var systemdKernelParam = []vc.Param{
-	{
-		Key:   "init",
-		Value: "/usr/lib/systemd/systemd",
-	},
-	{
-		Key:   "systemd.unit",
-		Value: systemdUnitName,
-	},
-	{
-		Key:   "systemd.mask",
-		Value: "systemd-networkd.service",
-	},
-	{
-		Key:   "systemd.mask",
-		Value: "systemd-networkd.socket",
-	},
-}
-
-func getKernelParams(needSystemd bool) []vc.Param {
-	p := []vc.Param{}
-
-	if needSystemd {
-		p = append(p, systemdKernelParam...)
-	}
-
-	return p
-}
-
-func needSystemd(config vc.HypervisorConfig) bool {
-	return config.ImagePath != ""
-}
-
-// setKernelParams adds the user-specified kernel parameters (from the
-// configuration file) to the defaults so that the former take priority.
-func setKernelParams(containerID string, runtimeConfig *oci.RuntimeConfig) error {
-	defaultKernelParams := getKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig))
-
-	if runtimeConfig.HypervisorConfig.Debug {
-		strParams := vc.SerializeParams(defaultKernelParams, "=")
-		formatted := strings.Join(strParams, " ")
-
-		kataLog.WithField("default-kernel-parameters", formatted).Debug()
-	}
-
-	// retrieve the parameters specified in the config file
-	userKernelParams := runtimeConfig.HypervisorConfig.KernelParams
-
-	// reset
-	runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{}
-
-	// first, add default values
-	for _, p := range defaultKernelParams {
-		if err := (runtimeConfig).AddKernelParam(p); err != nil {
-			return err
-		}
-	}
-
-	// now re-add the user-specified values so that they take priority.
-	for _, p := range userKernelParams {
-		if err := (runtimeConfig).AddKernelParam(p); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func createSandbox(ctx context.Context, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig,
-	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (vc.Process, error) {
-	span, ctx := trace(ctx, "createSandbox")
-	defer span.Finish()
-
-	err := setKernelParams(containerID, &runtimeConfig)
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	// Important to create the network namespace before the sandbox is
-	// created, because it is not responsible for the creation of the
-	// netns if it does not exist.
-	if err := setupNetworkNamespace(&sandboxConfig.NetworkConfig); err != nil {
-		return vc.Process{}, err
-	}
-
-	// Run pre-start OCI hooks.
-	err = enterNetNS(sandboxConfig.NetworkConfig.NetNSPath, func() error {
-		return preStartHooks(ctx, ociSpec, containerID, bundlePath)
-	})
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	sandbox, err := vci.CreateSandbox(ctx, sandboxConfig)
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	sid := sandbox.ID()
-	kataLog = kataLog.WithField("sandbox", sid)
-	setExternalLoggers(ctx, kataLog)
-	span.SetTag("sandbox", sid)
-
-	containers := sandbox.GetAllContainers()
-	if len(containers) != 1 {
-		return vc.Process{}, fmt.Errorf("BUG: Container list from sandbox is wrong, expecting only one container, found %d containers", len(containers))
-	}
-
-	if err := addContainerIDMapping(ctx, containerID, sandbox.ID()); err != nil {
-		return vc.Process{}, err
-	}
-
-	return containers[0].Process(), nil
-}
-
-// setEphemeralStorageType sets the mount type to 'ephemeral'
-// if the mount source path is provisioned by k8s for ephemeral storage.
-// For the given pod ephemeral volume is created only once
-// backed by tmpfs inside the VM. For successive containers
-// of the same pod the already existing volume is reused.
-func setEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec {
-	for idx, mnt := range ociSpec.Mounts {
-		if IsEphemeralStorage(mnt.Source) {
-			ociSpec.Mounts[idx].Type = "ephemeral"
-		}
-	}
-	return ociSpec
-}
-
-func createContainer(ctx context.Context, ociSpec oci.CompatOCISpec, containerID, bundlePath,
-	console string, disableOutput bool) (vc.Process, error) {
-
-	span, ctx := trace(ctx, "createContainer")
-	defer span.Finish()
-
-	ociSpec = setEphemeralStorageType(ociSpec)
-
-	contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput)
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	sandboxID, err := ociSpec.SandboxID()
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	kataLog = kataLog.WithField("sandbox", sandboxID)
-	setExternalLoggers(ctx, kataLog)
-	span.SetTag("sandbox", sandboxID)
-
-	s, c, err := vci.CreateContainer(ctx, sandboxID, contConfig)
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	// Run pre-start OCI hooks.
-	err = enterNetNS(s.GetNetNs(), func() error {
-		return preStartHooks(ctx, ociSpec, containerID, bundlePath)
-	})
-	if err != nil {
-		return vc.Process{}, err
-	}
-
-	if err := addContainerIDMapping(ctx, containerID, sandboxID); err != nil {
-		return vc.Process{}, err
-	}
-
-	return c.Process(), nil
-}
-
 func createPIDFile(ctx context.Context, pidFilePath string, pid int) error {
-	span, _ := trace(ctx, "createPIDFile")
+	span, _ := katautils.Trace(ctx, "createPIDFile")
 	defer span.Finish()
 
 	if pidFilePath == "" {

cli/create_test.go

@@ -7,16 +7,15 @@ package main
 
 import (
 	"context"
-	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
-	"strings"
 	"testing"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
@@ -29,34 +28,14 @@ const (
 	testPID                     = 100
 	testConsole                 = "/dev/pts/999"
 	testContainerTypeAnnotation = "io.kubernetes.cri-o.ContainerType"
-	testSandboxIDAnnotation     = "io.kubernetes.cri-o.SandboxID"
 	testContainerTypeSandbox    = "sandbox"
 	testContainerTypeContainer  = "container"
 )
 
-var testStrPID = fmt.Sprintf("%d", testPID)
-
-// return the value of the *last* param with the specified key
-func findLastParam(key string, params []vc.Param) (string, error) {
-	if key == "" {
-		return "", errors.New("ERROR: need non-nil key")
-	}
-
-	l := len(params)
-	if l == 0 {
-		return "", errors.New("ERROR: no params")
-	}
-
-	for i := l - 1; i >= 0; i-- {
-		p := params[i]
-
-		if key == p.Key {
-			return p.Value, nil
-		}
-	}
-
-	return "", fmt.Errorf("no param called %q found", name)
-}
+var (
+	testStrPID      = fmt.Sprintf("%d", testPID)
+	ctrsMapTreePath = "/var/run/kata-containers/containers-mapping"
+)
 
 func TestCreatePIDFileSuccessful(t *testing.T) {
 	pidDirPath, err := ioutil.TempDir(testDir, "pid-path-")
@@ -230,6 +209,7 @@ func TestCreateInvalidArgs(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	defer func() {
 		testingImpl.CreateSandboxFunc = nil
@@ -280,6 +260,7 @@ func TestCreateInvalidConfigJSON(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	tmpdir, err := ioutil.TempDir("", "")
 	assert.NoError(err)
@@ -296,7 +277,7 @@ func TestCreateInvalidConfigJSON(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	f, err := os.OpenFile(ociConfigFile, os.O_APPEND|os.O_WRONLY, testFileMode)
 	assert.NoError(err)
@@ -321,6 +302,7 @@ func TestCreateInvalidContainerType(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	tmpdir, err := ioutil.TempDir("", "")
 	assert.NoError(err)
@@ -337,7 +319,7 @@ func TestCreateInvalidContainerType(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
@@ -365,6 +347,7 @@ func TestCreateContainerInvalid(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	tmpdir, err := ioutil.TempDir("", "")
 	assert.NoError(err)
@@ -381,7 +364,7 @@ func TestCreateContainerInvalid(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 
@@ -421,6 +404,7 @@ func TestCreateProcessCgroupsPathSuccessful(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
 		return sandbox, nil
@@ -445,7 +429,7 @@ func TestCreateProcessCgroupsPathSuccessful(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
@@ -523,6 +507,7 @@ func TestCreateCreateCgroupsFilesFail(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
 		return sandbox, nil
@@ -547,7 +532,7 @@ func TestCreateCreateCgroupsFilesFail(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
@@ -608,6 +593,7 @@ func TestCreateCreateCreatePidFileFail(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
 		return sandbox, nil
@@ -633,7 +619,7 @@ func TestCreateCreateCreatePidFileFail(t *testing.T) {
 	pidFilePath := filepath.Join(pidDir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
@@ -683,6 +669,7 @@ func TestCreate(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
 		return sandbox, nil
@@ -707,7 +694,7 @@ func TestCreate(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
@@ -741,6 +728,7 @@ func TestCreateInvalidKernelParams(t *testing.T) {
 	assert.NoError(err)
 	defer os.RemoveAll(path)
 	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 
 	tmpdir, err := ioutil.TempDir("", "")
 	assert.NoError(err)
@@ -757,7 +745,7 @@ func TestCreateInvalidKernelParams(t *testing.T) {
 	pidFilePath := filepath.Join(tmpdir, "pidfile.txt")
 
 	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
+	assert.True(katautils.FileExists(ociConfigFile))
 
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
@@ -770,12 +758,12 @@ func TestCreateInvalidKernelParams(t *testing.T) {
 	err = writeOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
-	savedFunc := getKernelParamsFunc
+	savedFunc := katautils.GetKernelParamsFunc
 	defer func() {
-		getKernelParamsFunc = savedFunc
+		katautils.GetKernelParamsFunc = savedFunc
 	}()
 
-	getKernelParamsFunc = func(needSystemd bool) []vc.Param {
+	katautils.GetKernelParamsFunc = func(needSystemd bool) []vc.Param {
 		return []vc.Param{
 			{
 				Key:   "",
@@ -791,281 +779,3 @@ func TestCreateInvalidKernelParams(t *testing.T) {
 		os.RemoveAll(path)
 	}
 }
-
-func TestCreateSandboxConfigFail(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
-	assert.NoError(err)
-
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
-
-	spec, err := readOCIConfigFile(ociConfigFile)
-	assert.NoError(err)
-
-	quota := int64(0)
-	limit := int64(0)
-
-	spec.Linux.Resources.Memory = &specs.LinuxMemory{
-		Limit: &limit,
-	}
-
-	spec.Linux.Resources.CPU = &specs.LinuxCPU{
-		// specify an invalid value
-		Quota: &quota,
-	}
-
-	_, err = createSandbox(context.Background(), spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true)
-	assert.Error(err)
-}
-
-func TestCreateCreateSandboxFail(t *testing.T) {
-	if os.Geteuid() != 0 {
-		t.Skip(testDisabledNeedNonRoot)
-	}
-
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
-	assert.NoError(err)
-
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
-
-	spec, err := readOCIConfigFile(ociConfigFile)
-	assert.NoError(err)
-
-	_, err = createSandbox(context.Background(), spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true)
-	assert.Error(err)
-	assert.True(vcmock.IsMockError(err))
-}
-
-func TestCreateCreateContainerContainerConfigFail(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
-
-	spec, err := readOCIConfigFile(ociConfigFile)
-	assert.NoError(err)
-
-	// Set invalid container type
-	containerType := "你好，世界"
-	spec.Annotations = make(map[string]string)
-	spec.Annotations[testContainerTypeAnnotation] = containerType
-
-	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
-	assert.NoError(err)
-
-	for _, disableOutput := range []bool{true, false} {
-		_, err = createContainer(context.Background(), spec, testContainerID, bundlePath, testConsole, disableOutput)
-		assert.Error(err)
-		assert.False(vcmock.IsMockError(err))
-		assert.True(strings.Contains(err.Error(), containerType))
-		os.RemoveAll(path)
-	}
-}
-
-func TestCreateCreateContainerFail(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
-
-	spec, err := readOCIConfigFile(ociConfigFile)
-	assert.NoError(err)
-
-	// set expected container type and sandboxID
-	spec.Annotations = make(map[string]string)
-	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
-	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
-
-	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
-	assert.NoError(err)
-
-	for _, disableOutput := range []bool{true, false} {
-		_, err = createContainer(context.Background(), spec, testContainerID, bundlePath, testConsole, disableOutput)
-		assert.Error(err)
-		assert.True(vcmock.IsMockError(err))
-		os.RemoveAll(path)
-	}
-}
-
-func TestSetEphemeralStorageType(t *testing.T) {
-	assert := assert.New(t)
-
-	ociSpec := oci.CompatOCISpec{}
-	var ociMounts []specs.Mount
-	mount := specs.Mount{
-		Source: "/var/lib/kubelet/pods/366c3a77-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume",
-	}
-
-	ociMounts = append(ociMounts, mount)
-	ociSpec.Mounts = ociMounts
-	ociSpec = setEphemeralStorageType(ociSpec)
-
-	mountType := ociSpec.Mounts[0].Type
-	assert.Equal(mountType, "ephemeral",
-		"Unexpected mount type, got %s expected ephemeral", mountType)
-}
-
-func TestCreateCreateContainer(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) {
-		return &vcmock.Sandbox{}, &vcmock.Container{}, nil
-	}
-
-	defer func() {
-		testingImpl.CreateContainerFunc = nil
-	}()
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	bundlePath := filepath.Join(tmpdir, "bundle")
-
-	err = makeOCIBundle(bundlePath)
-	assert.NoError(err)
-
-	ociConfigFile := filepath.Join(bundlePath, "config.json")
-	assert.True(fileExists(ociConfigFile))
-
-	spec, err := readOCIConfigFile(ociConfigFile)
-	assert.NoError(err)
-
-	// set expected container type and sandboxID
-	spec.Annotations = make(map[string]string)
-	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
-	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
-
-	// rewrite file
-	err = writeOCIConfigFile(spec, ociConfigFile)
-	assert.NoError(err)
-
-	for _, disableOutput := range []bool{true, false} {
-		_, err = createContainer(context.Background(), spec, testContainerID, bundlePath, testConsole, disableOutput)
-		assert.NoError(err)
-		os.RemoveAll(path)
-	}
-}
-
-func TestSetKernelParams(t *testing.T) {
-	assert := assert.New(t)
-
-	config := oci.RuntimeConfig{}
-
-	assert.Empty(config.HypervisorConfig.KernelParams)
-
-	err := setKernelParams(testContainerID, &config)
-	assert.NoError(err)
-
-	if needSystemd(config.HypervisorConfig) {
-		assert.NotEmpty(config.HypervisorConfig.KernelParams)
-	}
-}
-
-func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) {
-	assert := assert.New(t)
-
-	initName := "init"
-	initValue := "/sbin/myinit"
-
-	ipName := "ip"
-	ipValue := "127.0.0.1"
-
-	params := []vc.Param{
-		{Key: initName, Value: initValue},
-		{Key: ipName, Value: ipValue},
-	}
-
-	hypervisorConfig := vc.HypervisorConfig{
-		KernelParams: params,
-	}
-
-	// Config containing user-specified kernel parameters
-	config := oci.RuntimeConfig{
-		HypervisorConfig: hypervisorConfig,
-	}
-
-	assert.NotEmpty(config.HypervisorConfig.KernelParams)
-
-	err := setKernelParams(testContainerID, &config)
-	assert.NoError(err)
-
-	kernelParams := config.HypervisorConfig.KernelParams
-
-	init, err := findLastParam(initName, kernelParams)
-	assert.NoError(err)
-	assert.Equal(initValue, init)
-
-	ip, err := findLastParam(ipName, kernelParams)
-	assert.NoError(err)
-	assert.Equal(ipValue, ip)
-
-}

cli/delete.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	vcAnnot "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
@@ -60,7 +61,7 @@ EXAMPLE:
 }
 
 func delete(ctx context.Context, containerID string, force bool) error {
-	span, ctx := trace(ctx, "delete")
+	span, ctx := katautils.Trace(ctx, "delete")
 	defer span.Finish()
 
 	kataLog = kataLog.WithField("container", containerID)
@@ -119,15 +120,15 @@ func delete(ctx context.Context, containerID string, force bool) error {
 	}
 
 	// Run post-stop OCI hooks.
-	if err := postStopHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]); err != nil {
+	if err := katautils.PostStopHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]); err != nil {
 		return err
 	}
 
-	return delContainerIDMapping(ctx, containerID)
+	return katautils.DelContainerIDMapping(ctx, containerID)
 }
 
 func deleteSandbox(ctx context.Context, sandboxID string) error {
-	span, _ := trace(ctx, "deleteSandbox")
+	span, _ := katautils.Trace(ctx, "deleteSandbox")
 	defer span.Finish()
 
 	status, err := vci.StatusSandbox(ctx, sandboxID)
@@ -149,7 +150,7 @@ func deleteSandbox(ctx context.Context, sandboxID string) error {
 }
 
 func deleteContainer(ctx context.Context, sandboxID, containerID string, forceStop bool) error {
-	span, _ := trace(ctx, "deleteContainer")
+	span, _ := katautils.Trace(ctx, "deleteContainer")
 	defer span.Finish()
 
 	if forceStop {
@@ -166,7 +167,7 @@ func deleteContainer(ctx context.Context, sandboxID, containerID string, forceSt
 }
 
 func removeCgroupsPath(ctx context.Context, containerID string, cgroupsPathList []string) error {
-	span, _ := trace(ctx, "removeCgroupsPath")
+	span, _ := katautils.Trace(ctx, "removeCgroupsPath")
 	defer span.Finish()
 
 	if len(cgroupsPathList) == 0 {

cli/events.go

@@ -15,6 +15,7 @@ import (
 
 	vc "github.com/kata-containers/runtime/virtcontainers"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -141,7 +142,7 @@ information is displayed once every 5 seconds.`,
 			return err
 		}
 
-		span, _ := trace(ctx, "events")
+		span, _ := katautils.Trace(ctx, "events")
 		defer span.Finish()
 
 		containerID := context.Args().First()

cli/exec.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"syscall"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -188,7 +189,7 @@ func generateExecParams(context *cli.Context, specProcess *oci.CompatOCIProcess)
 }
 
 func execute(ctx context.Context, context *cli.Context) error {
-	span, ctx := trace(ctx, "execute")
+	span, ctx := katautils.Trace(ctx, "execute")
 	defer span.Finish()
 
 	containerID := context.Args().First()

cli/kata-check.go

@@ -117,7 +117,7 @@ func getCPUFlags(cpuinfo string) string {
 func haveKernelModule(module string) bool {
 	// First, check to see if the module is already loaded
 	path := filepath.Join(sysModuleDir, module)
-	if fileExists(path) {
+	if katautils.FileExists(path) {
 		return true
 	}
 
@@ -289,7 +289,7 @@ var kataCheckCLICommand = cli.Command{
 			return err
 		}
 
-		span, _ := trace(ctx, "kata-check")
+		span, _ := katautils.Trace(ctx, "kata-check")
 		defer span.Finish()
 
 		err = setCPUtype()

cli/kata-check_test.go

@@ -15,6 +15,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/urfave/cli"
@@ -59,7 +60,7 @@ func createModules(assert *assert.Assertions, cpuInfoFile string, moduleData []t
 		}
 
 		err = hostIsVMContainerCapable(details)
-		if fileExists(cpuInfoFile) {
+		if katautils.FileExists(cpuInfoFile) {
 			assert.NoError(err)
 		} else {
 			assert.Error(err)

cli/kata-env.go

@@ -14,6 +14,7 @@ import (
 	runtim "runtime"
 
 	"github.com/BurntSushi/toml"
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	vcUtils "github.com/kata-containers/runtime/virtcontainers/utils"
@@ -276,7 +277,7 @@ func getNetmonInfo(config oci.RuntimeConfig) (NetmonInfo, error) {
 }
 
 func getCommandVersion(cmd string) (string, error) {
-	return runCommand([]string{cmd, "--version"})
+	return katautils.RunCommand([]string{cmd, "--version"})
 }
 
 func getShimInfo(config oci.RuntimeConfig) (ShimInfo, error) {
@@ -456,7 +457,7 @@ var kataEnvCLICommand = cli.Command{
 			return err
 		}
 
-		span, _ := trace(ctx, "kata-env")
+		span, _ := katautils.Trace(ctx, "kata-env")
 		defer span.Finish()
 
 		return handleSettings(defaultOutputFile, context)

cli/kata-env_test.go

@@ -196,7 +196,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		return "", oci.RuntimeConfig{}, err
 	}
 
-	_, config, _, err = katautils.LoadConfiguration(configFile, true, false)
+	_, config, err = katautils.LoadConfiguration(configFile, true, false)
 	if err != nil {
 		return "", oci.RuntimeConfig{}, err
 	}

cli/kill.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/sirupsen/logrus"
@@ -97,7 +98,7 @@ var signalList = map[string]syscall.Signal{
 }
 
 func kill(ctx context.Context, containerID, signal string, all bool) error {
-	span, _ := trace(ctx, "kill")
+	span, _ := katautils.Trace(ctx, "kill")
 	defer span.Finish()
 
 	kataLog = kataLog.WithField("container", containerID)

cli/list.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/urfave/cli"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	oci "github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 )
@@ -114,7 +115,7 @@ To list containers created using a non-default value for "--root":
 			return err
 		}
 
-		span, ctx := trace(ctx, "list")
+		span, ctx := katautils.Trace(ctx, "list")
 		defer span.Finish()
 
 		s, err := getContainers(ctx, context)

cli/main.go

@@ -60,9 +60,6 @@ var originalLoggerLevel logrus.Level
 
 var debug = false
 
-// if true, enable opentracing support.
-var tracing = false
-
 // if true, coredump when an internal error occurs or a fatal signal is received
 var crashOnError = false
 
@@ -193,7 +190,7 @@ func setupSignalHandler(ctx context.Context) {
 	}
 
 	dieCb := func() {
-		stopTracing(ctx)
+		katautils.StopTracing(ctx)
 	}
 
 	go func() {
@@ -229,7 +226,7 @@ func setExternalLoggers(ctx context.Context, logger *logrus.Entry) {
 	// created.
 
 	if opentracing.SpanFromContext(ctx) != nil {
-		span, ctx = trace(ctx, "setExternalLoggers")
+		span, ctx = katautils.Trace(ctx, "setExternalLoggers")
 		defer span.Finish()
 	}
 
@@ -307,7 +304,7 @@ func beforeSubcommands(c *cli.Context) error {
 
 	katautils.SetConfigOptions(name, defaultRuntimeConfiguration, defaultSysConfRuntimeConfiguration)
 
-	configFile, runtimeConfig, tracing, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false)
+	configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false)
 	if err != nil {
 		fatal(err)
 	}
@@ -360,7 +357,7 @@ func handleShowConfig(context *cli.Context) {
 }
 
 func setupTracing(context *cli.Context, rootSpanName string) error {
-	tracer, err := createTracer(name)
+	tracer, err := katautils.CreateTracer(name)
 	if err != nil {
 		fatal(err)
 	}
@@ -397,7 +394,7 @@ func afterSubcommands(c *cli.Context) error {
 		return err
 	}
 
-	stopTracing(ctx)
+	katautils.StopTracing(ctx)
 
 	return nil
 }
@@ -546,7 +543,7 @@ func main() {
 	ctx := context.Background()
 
 	dieCb := func() {
-		stopTracing(ctx)
+		katautils.StopTracing(ctx)
 	}
 
 	defer signals.HandlePanic(dieCb)

cli/main_test.go

@@ -33,7 +33,6 @@ import (
 )
 
 const (
-	testDisabledNeedRoot    = "Test disabled as requires root user"
 	testDisabledNeedNonRoot = "Test disabled as requires non-root user"
 	testDirMode             = os.FileMode(0750)
 	testFileMode            = os.FileMode(0640)
@@ -90,7 +89,7 @@ func init() {
 	fmt.Printf("INFO: test directory is %v\n", testDir)
 
 	fmt.Printf("INFO: ensuring docker is running\n")
-	output, err := runCommandFull([]string{"docker", "version"}, true)
+	output, err := katautils.RunCommandFull([]string{"docker", "version"}, true)
 	if err != nil {
 		panic(fmt.Sprintf("ERROR: docker daemon is not installed, not running, or not accessible to current user: %v (error %v)",
 			output, err))
@@ -101,11 +100,11 @@ func init() {
 	fmt.Printf("INFO: ensuring required docker image (%v) is available\n", testDockerImage)
 
 	// Only hit the network if the image doesn't exist locally
-	_, err = runCommand([]string{"docker", "inspect", "--type=image", testDockerImage})
+	_, err = katautils.RunCommand([]string{"docker", "inspect", "--type=image", testDockerImage})
 	if err == nil {
 		fmt.Printf("INFO: docker image %v already exists locally\n", testDockerImage)
 	} else {
-		_, err = runCommand([]string{"docker", "pull", testDockerImage})
+		_, err = katautils.RunCommand([]string{"docker", "pull", testDockerImage})
 		if err != nil {
 			panic(err)
 		}
@@ -256,7 +255,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("BUG: Need bundle directory")
 	}
 
-	if !fileExists(bundleDir) {
+	if !katautils.FileExists(bundleDir) {
 		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
 	}
 
@@ -276,13 +275,13 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := runCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}
 
 	specFile := filepath.Join(bundleDir, specConfig)
-	if !fileExists(specFile) {
+	if !katautils.FileExists(specFile) {
 		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
 	}
 
@@ -297,7 +296,7 @@ func createRootfs(dir string) error {
 		return err
 	}
 
-	container, err := runCommand([]string{"docker", "create", testDockerImage})
+	container, err := katautils.RunCommand([]string{"docker", "create", testDockerImage})
 	if err != nil {
 		return err
 	}
@@ -328,7 +327,7 @@ func createRootfs(dir string) error {
 	}
 
 	// Clean up
-	_, err = runCommand([]string{"docker", "rm", container})
+	_, err = katautils.RunCommand([]string{"docker", "rm", container})
 	if err != nil {
 		return err
 	}
@@ -346,7 +345,7 @@ func realMakeOCIBundle(bundleDir string) error {
 		return errors.New("BUG: Need bundle directory")
 	}
 
-	if !fileExists(bundleDir) {
+	if !katautils.FileExists(bundleDir) {
 		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
 	}
 
@@ -396,12 +395,12 @@ func makeOCIBundle(bundleDir string) error {
 	base := filepath.Dir(bundleDir)
 
 	for _, dir := range []string{from, base} {
-		if !fileExists(dir) {
+		if !katautils.FileExists(dir) {
 			return fmt.Errorf("BUG: directory %v should exist", dir)
 		}
 	}
 
-	output, err := runCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}
@@ -501,7 +500,7 @@ func TestMakeOCIBundle(t *testing.T) {
 	assert.NoError(err)
 
 	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(fileExists(specFile))
+	assert.True(katautils.FileExists(specFile))
 }
 
 func TestCreateOCIConfig(t *testing.T) {
@@ -524,7 +523,7 @@ func TestCreateOCIConfig(t *testing.T) {
 	assert.NoError(err)
 
 	specFile := filepath.Join(bundleDir, specConfig)
-	assert.True(fileExists(specFile))
+	assert.True(katautils.FileExists(specFile))
 }
 
 func TestCreateRootfs(t *testing.T) {
@@ -535,7 +534,7 @@ func TestCreateRootfs(t *testing.T) {
 	defer os.RemoveAll(tmpdir)
 
 	rootfsDir := filepath.Join(tmpdir, "rootfs")
-	assert.False(fileExists(rootfsDir))
+	assert.False(katautils.FileExists(rootfsDir))
 
 	err = createRootfs(rootfsDir)
 	assert.NoError(err)
@@ -543,11 +542,11 @@ func TestCreateRootfs(t *testing.T) {
 	// non-comprehensive list of expected directories
 	expectedDirs := []string{"bin", "dev", "etc", "usr", "var"}
 
-	assert.True(fileExists(rootfsDir))
+	assert.True(katautils.FileExists(rootfsDir))
 
 	for _, dir := range expectedDirs {
 		dirPath := filepath.Join(rootfsDir, dir)
-		assert.True(fileExists(dirPath))
+		assert.True(katautils.FileExists(dirPath))
 	}
 }
 
@@ -1122,5 +1121,6 @@ func createTempContainerIDMapping(containerID, sandboxID string) (string, error)
 		return "", err
 	}
 
+	katautils.SetCtrsMapTreePath(ctrsMapTreePath)
 	return tmpDir, nil
 }

cli/network.go

@@ -6,17 +6,11 @@
 package main
 
 import (
-	"bufio"
 	"context"
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
-	"strings"
 
-	"golang.org/x/sys/unix"
-
-	"github.com/containernetworking/plugins/pkg/ns"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/sirupsen/logrus"
@@ -233,133 +227,3 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
 	}
 	return err
 }
-
-const procMountInfoFile = "/proc/self/mountinfo"
-
-// getNetNsFromBindMount returns the network namespace for the bind-mounted path
-func getNetNsFromBindMount(nsPath string, procMountFile string) (string, error) {
-	netNsMountType := "nsfs"
-
-	// Resolve all symlinks in the path as the mountinfo file contains
-	// resolved paths.
-	nsPath, err := filepath.EvalSymlinks(nsPath)
-	if err != nil {
-		return "", err
-	}
-
-	f, err := os.Open(procMountFile)
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		text := scanner.Text()
-
-		// Scan the mountinfo file to search for the network namespace path
-		// This file contains mounts in the eg format:
-		// "711 26 0:3 net:[4026532009] /run/docker/netns/default rw shared:535 - nsfs nsfs rw"
-		//
-		// Reference: https://www.kernel.org/doc/Documentation/filesystems/proc.txt
-
-		// We are interested in the first 9 fields of this file,
-		// to check for the correct mount type.
-		fields := strings.Split(text, " ")
-		if len(fields) < 9 {
-			continue
-		}
-
-		// We check here if the mount type is a network namespace mount type, namely "nsfs"
-		mountTypeFieldIdx := 8
-		if fields[mountTypeFieldIdx] != netNsMountType {
-			continue
-		}
-
-		// This is the mount point/destination for the mount
-		mntDestIdx := 4
-		if fields[mntDestIdx] != nsPath {
-			continue
-		}
-
-		// This is the root/source of the mount
-		return fields[3], nil
-	}
-
-	return "", nil
-}
-
-// hostNetworkingRequested checks if the network namespace requested is the
-// same as the current process.
-func hostNetworkingRequested(configNetNs string) (bool, error) {
-	var evalNS, nsPath, currentNsPath string
-	var err error
-
-	// Net namespace provided as "/proc/pid/ns/net" or "/proc/<pid>/task/<tid>/ns/net"
-	if strings.HasPrefix(configNetNs, "/proc") && strings.HasSuffix(configNetNs, "/ns/net") {
-		if _, err := os.Stat(configNetNs); err != nil {
-			return false, err
-		}
-
-		// Here we are trying to resolve the path but it fails because
-		// namespaces links don't really exist. For this reason, the
-		// call to EvalSymlinks will fail when it will try to stat the
-		// resolved path found. As we only care about the path, we can
-		// retrieve it from the PathError structure.
-		if _, err = filepath.EvalSymlinks(configNetNs); err != nil {
-			nsPath = err.(*os.PathError).Path
-		} else {
-			return false, fmt.Errorf("Net namespace path %s is not a symlink", configNetNs)
-		}
-
-		_, evalNS = filepath.Split(nsPath)
-
-	} else {
-		// Bind-mounted path provided
-		evalNS, _ = getNetNsFromBindMount(configNetNs, procMountInfoFile)
-	}
-
-	currentNS := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
-	if _, err = filepath.EvalSymlinks(currentNS); err != nil {
-		currentNsPath = err.(*os.PathError).Path
-	} else {
-		return false, fmt.Errorf("Unexpected: Current network namespace path is not a symlink")
-	}
-
-	_, evalCurrentNS := filepath.Split(currentNsPath)
-
-	if evalNS == evalCurrentNS {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func setupNetworkNamespace(config *vc.NetworkConfig) error {
-	if config.DisableNewNetNs {
-		kataLog.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns")
-		return nil
-	}
-
-	if config.NetNSPath == "" {
-		n, err := ns.NewNS()
-		if err != nil {
-			return err
-		}
-
-		config.NetNSPath = n.Path()
-		config.NetNsCreated = true
-
-		return nil
-	}
-
-	isHostNs, err := hostNetworkingRequested(config.NetNSPath)
-	if err != nil {
-		return err
-	}
-	if isHostNs {
-		return fmt.Errorf("Host networking requested, not supported by runtime")
-	}
-
-	return nil
-}

cli/network_test.go

@@ -8,16 +8,10 @@ package main
 import (
 	"context"
 	"flag"
-	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
-	"syscall"
 	"testing"
 
-	"golang.org/x/sys/unix"
-
-	"github.com/containernetworking/plugins/pkg/ns"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/stretchr/testify/assert"
@@ -93,133 +87,3 @@ func TestNetworkCliFunction(t *testing.T) {
 	f.Close()
 	execCLICommandFunc(assert, updateRoutesCommand, set, false)
 }
-
-func TestGetNetNsFromBindMount(t *testing.T) {
-	assert := assert.New(t)
-
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	mountFile := filepath.Join(tmpdir, "mountInfo")
-	nsPath := filepath.Join(tmpdir, "ns123")
-
-	// Non-existent namespace path
-	_, err = getNetNsFromBindMount(nsPath, mountFile)
-	assert.NotNil(err)
-
-	tmpNSPath := filepath.Join(tmpdir, "testNetNs")
-	f, err := os.Create(tmpNSPath)
-	assert.NoError(err)
-	defer f.Close()
-
-	type testData struct {
-		contents       string
-		expectedResult string
-	}
-
-	data := []testData{
-		{fmt.Sprintf("711 26 0:3 net:[4026532008] %s rw shared:535 - nsfs nsfs rw", tmpNSPath), "net:[4026532008]"},
-		{"711 26 0:3 net:[4026532008] /run/netns/ns123 rw shared:535 - tmpfs tmpfs rw", ""},
-		{"a a a a a a a - b c d", ""},
-		{"", ""},
-	}
-
-	for i, d := range data {
-		err := ioutil.WriteFile(mountFile, []byte(d.contents), 0640)
-		assert.NoError(err)
-
-		path, err := getNetNsFromBindMount(tmpNSPath, mountFile)
-		assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", path, d))
-
-		assert.Equal(d.expectedResult, path, "Test %d, expected %s, got %s", i, d.expectedResult, path)
-	}
-}
-
-func TestHostNetworkingRequested(t *testing.T) {
-	assert := assert.New(t)
-
-	if os.Geteuid() != 0 {
-		t.Skip(testDisabledNeedRoot)
-	}
-
-	// Network namespace same as the host
-	selfNsPath := "/proc/self/ns/net"
-	isHostNs, err := hostNetworkingRequested(selfNsPath)
-	assert.NoError(err)
-	assert.True(isHostNs)
-
-	// Non-existent netns path
-	nsPath := "/proc/123456789/ns/net"
-	_, err = hostNetworkingRequested(nsPath)
-	assert.Error(err)
-
-	// Bind-mounted Netns
-	tmpdir, err := ioutil.TempDir("", "")
-	assert.NoError(err)
-	defer os.RemoveAll(tmpdir)
-
-	// Create a bind mount to the current network namespace.
-	tmpFile := filepath.Join(tmpdir, "testNetNs")
-	f, err := os.Create(tmpFile)
-	assert.NoError(err)
-	defer f.Close()
-
-	err = syscall.Mount(selfNsPath, tmpFile, "bind", syscall.MS_BIND, "")
-	assert.Nil(err)
-
-	isHostNs, err = hostNetworkingRequested(tmpFile)
-	assert.NoError(err)
-	assert.True(isHostNs)
-
-	syscall.Unmount(tmpFile, 0)
-}
-
-func TestSetupNetworkNamespace(t *testing.T) {
-	if os.Geteuid() != 0 {
-		t.Skip(testDisabledNeedNonRoot)
-	}
-
-	assert := assert.New(t)
-
-	// Network namespace same as the host
-	config := &vc.NetworkConfig{
-		NetNSPath: "/proc/self/ns/net",
-	}
-	err := setupNetworkNamespace(config)
-	assert.Error(err)
-
-	// Non-existent netns path
-	config = &vc.NetworkConfig{
-		NetNSPath: "/proc/123456789/ns/net",
-	}
-	err = setupNetworkNamespace(config)
-	assert.Error(err)
-
-	// Existent netns path
-	n, err := ns.NewNS()
-	assert.NoError(err)
-	config = &vc.NetworkConfig{
-		NetNSPath: n.Path(),
-	}
-	err = setupNetworkNamespace(config)
-	assert.NoError(err)
-	n.Close()
-
-	// Empty netns path
-	config = &vc.NetworkConfig{}
-	err = setupNetworkNamespace(config)
-	assert.NoError(err)
-	n, err = ns.GetNS(config.NetNSPath)
-	assert.NoError(err)
-	assert.NotNil(n)
-	assert.True(config.NetNsCreated)
-	n.Close()
-	unix.Unmount(config.NetNSPath, unix.MNT_DETACH)
-	os.RemoveAll(config.NetNSPath)
-
-	// Config with DisableNewNetNs
-	config = &vc.NetworkConfig{DisableNewNetNs: true}
-	err = setupNetworkNamespace(config)
-	assert.NoError(err)
-}

cli/oci.go

@@ -9,15 +9,12 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
-	goruntime "runtime"
 	"strings"
 	"syscall"
 
-	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/opencontainers/runc/libcontainer/utils"
@@ -25,8 +22,6 @@ import (
 
 // Contants related to cgroup memory directory
 const (
-	ctrsMappingDirMode = os.FileMode(0750)
-
 	// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
 	// here: http://man7.org/linux/man-pages/man2/statfs.2.html
 	cgroupFsType = 0x27e0eb
@@ -36,8 +31,6 @@ var cgroupsDirPath string
 
 var procMountInfo = "/proc/self/mountinfo"
 
-var ctrsMapTreePath = "/var/run/kata-containers/containers-mapping"
-
 // getContainerInfo returns the container status and its sandbox ID.
 func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStatus, string, error) {
 	// container ID MUST be provided.
@@ -45,7 +38,7 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat
 		return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
 	}
 
-	sandboxID, err := fetchContainerIDMapping(containerID)
+	sandboxID, err := katautils.FetchContainerIDMapping(containerID)
 	if err != nil {
 		return vc.ContainerStatus{}, "", err
 	}
@@ -218,100 +211,3 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) {
 
 	return cgroupRootPath, nil
 }
-
-// This function assumes it should find only one file inside the container
-// ID directory. If there are several files, we could not determine which
-// file name corresponds to the sandbox ID associated, and this would throw
-// an error.
-func fetchContainerIDMapping(containerID string) (string, error) {
-	if containerID == "" {
-		return "", fmt.Errorf("Missing container ID")
-	}
-
-	dirPath := filepath.Join(ctrsMapTreePath, containerID)
-
-	files, err := ioutil.ReadDir(dirPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return "", nil
-		}
-
-		return "", err
-	}
-
-	if len(files) != 1 {
-		return "", fmt.Errorf("Too many files (%d) in %q", len(files), dirPath)
-	}
-
-	return files[0].Name(), nil
-}
-
-func addContainerIDMapping(ctx context.Context, containerID, sandboxID string) error {
-	span, _ := trace(ctx, "addContainerIDMapping")
-	defer span.Finish()
-
-	if containerID == "" {
-		return fmt.Errorf("Missing container ID")
-	}
-
-	if sandboxID == "" {
-		return fmt.Errorf("Missing sandbox ID")
-	}
-
-	parentPath := filepath.Join(ctrsMapTreePath, containerID)
-
-	if err := os.RemoveAll(parentPath); err != nil {
-		return err
-	}
-
-	path := filepath.Join(parentPath, sandboxID)
-
-	if err := os.MkdirAll(path, ctrsMappingDirMode); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func delContainerIDMapping(ctx context.Context, containerID string) error {
-	span, _ := trace(ctx, "delContainerIDMapping")
-	defer span.Finish()
-
-	if containerID == "" {
-		return fmt.Errorf("Missing container ID")
-	}
-
-	path := filepath.Join(ctrsMapTreePath, containerID)
-
-	return os.RemoveAll(path)
-}
-
-// enterNetNS is free from any call to a go routine, and it calls
-// into runtime.LockOSThread(), meaning it won't be executed in a
-// different thread than the one expected by the caller.
-func enterNetNS(netNSPath string, cb func() error) error {
-	if netNSPath == "" {
-		return cb()
-	}
-
-	goruntime.LockOSThread()
-	defer goruntime.UnlockOSThread()
-
-	currentNS, err := ns.GetCurrentNS()
-	if err != nil {
-		return err
-	}
-	defer currentNS.Close()
-
-	targetNS, err := ns.GetNS(netNSPath)
-	if err != nil {
-		return err
-	}
-
-	if err := targetNS.Set(); err != nil {
-		return err
-	}
-	defer currentNS.Set()
-
-	return cb()
-}

cli/oci_test.go

@@ -317,106 +317,3 @@ func TestGetCgroupsDirPath(t *testing.T) {
 		assert.Equal(d.expectedResult, path)
 	}
 }
-
-func TestFetchContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
-	assert := assert.New(t)
-
-	sandboxID, err := fetchContainerIDMapping("")
-	assert.Error(err)
-	assert.Empty(sandboxID)
-}
-
-func TestFetchContainerIDMappingEmptyMappingSuccess(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	sandboxID, err := fetchContainerIDMapping(testContainerID)
-	assert.NoError(err)
-	assert.Empty(sandboxID)
-}
-
-func TestFetchContainerIDMappingTooManyFilesFailure(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	err = os.MkdirAll(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID+"2"), ctrsMappingDirMode)
-	assert.NoError(err)
-
-	sandboxID, err := fetchContainerIDMapping(testContainerID)
-	assert.Error(err)
-	assert.Empty(sandboxID)
-}
-
-func TestFetchContainerIDMappingSuccess(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-
-	sandboxID, err := fetchContainerIDMapping(testContainerID)
-	assert.NoError(err)
-	assert.Equal(sandboxID, testSandboxID)
-}
-
-func TestAddContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
-	assert := assert.New(t)
-
-	err := addContainerIDMapping(context.Background(), "", testSandboxID)
-	assert.Error(err)
-}
-
-func TestAddContainerIDMappingSandboxIDEmptyFailure(t *testing.T) {
-	assert := assert.New(t)
-
-	err := addContainerIDMapping(context.Background(), testContainerID, "")
-	assert.Error(err)
-}
-
-func TestAddContainerIDMappingSuccess(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := ioutil.TempDir("", "containers-mapping")
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
-
-	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
-	assert.True(os.IsNotExist(err))
-
-	err = addContainerIDMapping(context.Background(), testContainerID, testSandboxID)
-	assert.NoError(err)
-
-	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
-	assert.NoError(err)
-}
-
-func TestDelContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
-	assert := assert.New(t)
-
-	err := delContainerIDMapping(context.Background(), "")
-	assert.Error(err)
-}
-
-func TestDelContainerIDMappingSuccess(t *testing.T) {
-	assert := assert.New(t)
-
-	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
-	assert.NoError(err)
-	defer os.RemoveAll(path)
-
-	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
-	assert.NoError(err)
-
-	err = delContainerIDMapping(context.Background(), testContainerID)
-	assert.NoError(err)
-
-	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
-	assert.True(os.IsNotExist(err))
-}

cli/pause.go

@@ -9,6 +9,7 @@ package main
 import (
 	"context"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -57,7 +58,7 @@ func toggle(c *cli.Context, pause bool) error {
 }
 
 func toggleContainerPause(ctx context.Context, containerID string, pause bool) (err error) {
-	span, _ := trace(ctx, "pause")
+	span, _ := katautils.Trace(ctx, "pause")
 	defer span.Finish()
 	span.SetTag("pause", pause)
 

cli/ps.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -50,7 +51,7 @@ var psCLICommand = cli.Command{
 }
 
 func ps(ctx context.Context, containerID, format string, args []string) error {
-	span, _ := trace(ctx, "ps")
+	span, _ := katautils.Trace(ctx, "ps")
 	defer span.Finish()
 
 	if containerID == "" {

cli/run.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"syscall"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/urfave/cli"
 )
@@ -82,7 +83,7 @@ var runCLICommand = cli.Command{
 
 func run(ctx context.Context, containerID, bundle, console, consoleSocket, pidFile string, detach, systemdCgroup bool,
 	runtimeConfig oci.RuntimeConfig) error {
-	span, ctx := trace(ctx, "run")
+	span, ctx := katautils.Trace(ctx, "run")
 	defer span.Finish()
 
 	consolePath, err := setupConsole(console, consoleSocket)

cli/spec.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/opencontainers/runc/libcontainer/specconv"
 	"github.com/urfave/cli"
 )
@@ -77,7 +78,7 @@ generate a proper rootless spec file.`,
 			return err
 		}
 
-		span, _ := trace(ctx, "spec")
+		span, _ := katautils.Trace(ctx, "spec")
 		defer span.Finish()
 
 		spec := specconv.Example()

cli/start.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	vcAnnot "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
@@ -48,7 +49,7 @@ var startCLICommand = cli.Command{
 }
 
 func start(ctx context.Context, containerID string) (vc.VCSandbox, error) {
-	span, _ := trace(ctx, "start")
+	span, _ := katautils.Trace(ctx, "start")
 	defer span.Finish()
 
 	kataLog = kataLog.WithField("container", containerID)
@@ -101,8 +102,8 @@ func start(ctx context.Context, containerID string) (vc.VCSandbox, error) {
 	}
 
 	// Run post-start OCI hooks.
-	err = enterNetNS(sandbox.GetNetNs(), func() error {
-		return postStartHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey])
+	err = katautils.EnterNetNS(sandbox.GetNetNs(), func() error {
+		return katautils.PostStartHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey])
 	})
 	if err != nil {
 		return nil, err

cli/start_test.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
@@ -41,7 +42,7 @@ func TestStartInvalidArgs(t *testing.T) {
 	path, err = ioutil.TempDir("", "containers-mapping")
 	assert.NoError(err)
 	defer os.RemoveAll(path)
-	ctrsMapTreePath = path
+	katautils.SetCtrsMapTreePath(path)
 
 	// Container missing in container mapping
 	_, err = start(context.Background(), testContainerID)

cli/state.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/urfave/cli"
 )
@@ -40,7 +41,7 @@ instance of a container.`,
 }
 
 func state(ctx context.Context, containerID string) error {
-	span, _ := trace(ctx, "state")
+	span, _ := katautils.Trace(ctx, "state")
 	defer span.Finish()
 
 	kataLog = kataLog.WithField("container", containerID)

cli/update.go

@@ -13,6 +13,7 @@ import (
 	"strconv"
 
 	"github.com/docker/go-units"
+	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
@@ -132,7 +133,7 @@ other options are ignored.
 			return err
 		}
 
-		span, _ := trace(ctx, "update")
+		span, _ := katautils.Trace(ctx, "update")
 		defer span.Finish()
 
 		if context.Args().Present() == false {

cli/utils.go

@@ -8,15 +8,13 @@ package main
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"strings"
 
 	"github.com/kata-containers/runtime/pkg/katautils"
 )
 
 const (
-	unknown     = "<<unknown>>"
-	k8sEmptyDir = "kubernetes.io~empty-dir"
+	unknown = "<<unknown>>"
 )
 
 // variables to allow tests to modify the values
@@ -28,34 +26,6 @@ var (
 	osReleaseClr = "/usr/lib/os-release"
 )
 
-func fileExists(path string) bool {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return false
-	}
-
-	return true
-}
-
-// IsEphemeralStorage returns true if the given path
-// to the storage belongs to kubernetes ephemeral storage
-//
-// This method depends on a specific path used by k8s
-// to detect if it's of type ephemeral. As of now,
-// this is a very k8s specific solution that works
-// but in future there should be a better way for this
-// method to determine if the path is for ephemeral
-// volume type
-func IsEphemeralStorage(path string) bool {
-	splitSourceSlice := strings.Split(path, "/")
-	if len(splitSourceSlice) > 1 {
-		storageType := splitSourceSlice[len(splitSourceSlice)-2]
-		if storageType == k8sEmptyDir {
-			return true
-		}
-	}
-	return false
-}
-
 func getKernelVersion() (string, error) {
 	contents, err := katautils.GetFileContents(procVersion)
 	if err != nil {
@@ -151,27 +121,3 @@ func genericGetCPUDetails() (vendor, model string, err error) {
 
 	return vendor, model, nil
 }
-
-// runCommandFull returns the commands space-trimmed standard output and
-// error on success. Note that if the command fails, the requested output will
-// still be returned, along with an error.
-func runCommandFull(args []string, includeStderr bool) (string, error) {
-	cmd := exec.Command(args[0], args[1:]...)
-	var err error
-	var bytes []byte
-
-	if includeStderr {
-		bytes, err = cmd.CombinedOutput()
-	} else {
-		bytes, err = cmd.Output()
-	}
-
-	trimmed := strings.TrimSpace(string(bytes))
-
-	return trimmed, err
-}
-
-// runCommand returns the commands space-trimmed standard output on success
-func runCommand(args []string) (string, error) {
-	return runCommandFull(args, false)
-}

cli/utils_test.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -24,7 +25,7 @@ func TestFileExists(t *testing.T) {
 
 	file := filepath.Join(dir, "foo")
 
-	assert.False(t, fileExists(file),
+	assert.False(t, katautils.FileExists(file),
 		fmt.Sprintf("File %q should not exist", file))
 
 	err = createEmptyFile(file)
@@ -32,24 +33,10 @@ func TestFileExists(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	assert.True(t, fileExists(file),
+	assert.True(t, katautils.FileExists(file),
 		fmt.Sprintf("File %q should exist", file))
 }
 
-func TestIsEphemeralStorage(t *testing.T) {
-	sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume"
-	isEphe := IsEphemeralStorage(sampleEphePath)
-	if !isEphe {
-		t.Fatalf("Unable to correctly determine volume type")
-	}
-
-	sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume"
-	isEphe = IsEphemeralStorage(sampleEphePath)
-	if isEphe {
-		t.Fatalf("Unable to correctly determine volume type")
-	}
-}
-
 func TestGetKernelVersion(t *testing.T) {
 	type testData struct {
 		contents        string
@@ -187,13 +174,13 @@ VERSION_ID="%s"
 }
 
 func TestUtilsRunCommand(t *testing.T) {
-	output, err := runCommand([]string{"true"})
+	output, err := katautils.RunCommand([]string{"true"})
 	assert.NoError(t, err)
 	assert.Equal(t, "", output)
 }
 
 func TestUtilsRunCommandCaptureStdout(t *testing.T) {
-	output, err := runCommand([]string{"echo", "hello"})
+	output, err := katautils.RunCommand([]string{"echo", "hello"})
 	assert.NoError(t, err)
 	assert.Equal(t, "hello", output)
 }
@@ -201,7 +188,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) {
 func TestUtilsRunCommandIgnoreStderr(t *testing.T) {
 	args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"}
 
-	output, err := runCommand(args)
+	output, err := katautils.RunCommand(args)
 	assert.NoError(t, err)
 	assert.Equal(t, "", output)
 }
@@ -224,7 +211,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) {
 	}
 
 	for _, args := range invalidCommands {
-		output, err := runCommand(args)
+		output, err := katautils.RunCommand(args)
 		assert.Error(t, err)
 		assert.Equal(t, "", output)
 	}

cli/version.go

@@ -6,6 +6,7 @@
 package main
 
 import (
+	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/urfave/cli"
 )
 
@@ -18,7 +19,7 @@ var versionCLICommand = cli.Command{
 			return err
 		}
 
-		span, _ := trace(ctx, "version")
+		span, _ := katautils.Trace(ctx, "version")
 		defer span.Finish()
 
 		cli.VersionPrinter(context)

containerd-shim-v2/container.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/api/types/task"
+	"github.com/containerd/containerd/errdefs"
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+)
+
+type container struct {
+	s        *service
+	ttyio    *ttyIO
+	spec     *oci.CompatOCISpec
+	time     time.Time
+	execs    map[string]*exec
+	exitIOch chan struct{}
+	exitCh   chan uint32
+	id       string
+	stdin    string
+	stdout   string
+	stderr   string
+	bundle   string
+	cType    vc.ContainerType
+	mu       sync.Mutex
+	exit     uint32
+	status   task.Status
+	terminal bool
+}
+
+func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *oci.CompatOCISpec) (*container, error) {
+	if r == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrInvalidArgument, " CreateTaskRequest points to nil")
+	}
+
+	// in order to avoid deferencing a nil pointer in test
+	if spec == nil {
+		spec = &oci.CompatOCISpec{}
+	}
+
+	c := &container{
+		s:        s,
+		spec:     spec,
+		id:       r.ID,
+		bundle:   r.Bundle,
+		stdin:    r.Stdin,
+		stdout:   r.Stdout,
+		stderr:   r.Stderr,
+		terminal: r.Terminal,
+		cType:    containerType,
+		execs:    make(map[string]*exec),
+		status:   task.StatusCreated,
+		exitIOch: make(chan struct{}),
+		exitCh:   make(chan uint32, 1),
+		time:     time.Now(),
+	}
+	return c, nil
+}

containerd-shim-v2/container_test.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestNewContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	_, err := newContainer(nil, nil, "", nil)
+
+	assert.Error(err)
+}
+
+func TestGetExec(t *testing.T) {
+	assert := assert.New(t)
+
+	r := &taskAPI.CreateTaskRequest{}
+
+	c, err := newContainer(nil, r, "", nil)
+	assert.NoError(err)
+
+	_, err = c.getExec("")
+	assert.Error(err)
+
+	c.execs = make(map[string]*exec)
+	_, err = c.getExec("")
+	assert.Error(err)
+
+	c.execs[TestID] = &exec{}
+	_, err = c.getExec(TestID)
+	assert.NoError(err)
+}

containerd-shim-v2/create.go

@@ -0,0 +1,102 @@
+// Copyright (c) 2014,2015,2016 Docker, Inc.
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns string,
+	runtimeConfig *oci.RuntimeConfig) (*container, error) {
+
+	detach := !r.Terminal
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	bundlePath, err := validBundle(r.ID, r.Bundle)
+	if err != nil {
+		return nil, err
+	}
+
+	ociSpec, err := oci.ParseConfigJSON(bundlePath)
+	if err != nil {
+		return nil, err
+	}
+
+	containerType, err := ociSpec.ContainerType()
+	if err != nil {
+		return nil, err
+	}
+
+	// Todo:
+	// Since there is a bug in kata for sharedPidNs, here to
+	// remove the pidns to disable the sharePidNs temporarily,
+	// once kata fixed this issue, we can remove this line.
+	// For the bug, please see:
+	// https://github.com/kata-containers/runtime/issues/930
+	removeNamespace(&ociSpec, specs.PIDNamespace)
+
+	//set the network namespace path
+	//this set will be applied to sandbox's
+	//network config and has nothing to
+	//do with containers in the sandbox since
+	//networkNamespace has been ignored by
+	//kata-agent in sandbox.
+
+	for _, n := range ociSpec.Linux.Namespaces {
+		if n.Type != specs.NetworkNamespace {
+			continue
+		}
+
+		if n.Path == "" {
+			n.Path = netns
+		}
+	}
+
+	katautils.HandleFactory(ctx, vci, runtimeConfig)
+
+	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
+
+	switch containerType {
+	case vc.PodSandbox:
+		if s.sandbox != nil {
+			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
+		}
+
+		sandbox, _, err := katautils.CreateSandbox(ctx, vci, ociSpec, *runtimeConfig, r.ID, bundlePath, "", disableOutput, false, true)
+		if err != nil {
+			return nil, err
+		}
+		s.sandbox = sandbox
+
+	case vc.PodContainer:
+		if s.sandbox == nil {
+			return nil, fmt.Errorf("BUG: Cannot start the container, since the sandbox hasn't been created")
+		}
+
+		_, err = katautils.CreateContainer(ctx, vci, s.sandbox, ociSpec, r.ID, bundlePath, "", disableOutput, true)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	container, err := newContainer(s, r, containerType, &ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	return container, nil
+}

containerd-shim-v2/create_test.go

@@ -0,0 +1,363 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/namespaces"
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateSandboxSuccess(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip(testDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+		MockContainers: []*vcmock.Container{
+			{MockID: testContainerID},
+		},
+	}
+
+	testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.CreateSandboxFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	// Force sandbox-type container
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeSandbox
+
+	// Set a limit to ensure processCgroupsPath() considers the
+	// cgroup part of the spec
+	limit := int64(1024 * 1024)
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	// Rewrite the file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	s := &service{
+		id:         testSandboxID,
+		containers: make(map[string]*container),
+		config:     &runtimeConfig,
+	}
+
+	req := &taskAPI.CreateTaskRequest{
+		ID:       testSandboxID,
+		Bundle:   bundlePath,
+		Terminal: true,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Create(ctx, req)
+	assert.NoError(err)
+}
+
+func TestCreateSandboxFail(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip(testDisabledNeedRoot)
+	}
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	s := &service{
+		id:         testSandboxID,
+		containers: make(map[string]*container),
+		config:     &runtimeConfig,
+	}
+
+	req := &taskAPI.CreateTaskRequest{
+		ID:       testSandboxID,
+		Bundle:   bundlePath,
+		Terminal: true,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Create(ctx, req)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestCreateSandboxConfigFail(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip(testDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	quota := int64(0)
+	limit := int64(0)
+
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	// specify an invalid spec
+	spec.Linux.Resources.CPU = &specs.LinuxCPU{
+		Quota: &quota,
+	}
+
+	s := &service{
+		id:         testSandboxID,
+		containers: make(map[string]*container),
+		config:     &runtimeConfig,
+	}
+
+	req := &taskAPI.CreateTaskRequest{
+		ID:       testSandboxID,
+		Bundle:   bundlePath,
+		Terminal: true,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Create(ctx, req)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestCreateContainerSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) {
+		return sandbox, &vcmock.Container{}, nil
+	}
+
+	defer func() {
+		testingImpl.CreateContainerFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	// set expected container type and sandboxID
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
+	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
+
+	// rewrite file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	s := &service{
+		id:         testContainerID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+		config:     &runtimeConfig,
+	}
+
+	req := &taskAPI.CreateTaskRequest{
+		ID:       testContainerID,
+		Bundle:   bundlePath,
+		Terminal: true,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Create(ctx, req)
+	assert.NoError(err)
+}
+
+func TestCreateContainerFail(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
+	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
+
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	// doesn't create sandbox first
+	s := &service{
+		id:         testContainerID,
+		containers: make(map[string]*container),
+		config:     &runtimeConfig,
+	}
+
+	req := &taskAPI.CreateTaskRequest{
+		ID:       testContainerID,
+		Bundle:   bundlePath,
+		Terminal: true,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Create(ctx, req)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestCreateContainerConfigFail(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) {
+		return sandbox, &vcmock.Container{}, nil
+	}
+
+	defer func() {
+		testingImpl.CreateContainerFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(katautils.FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	// set the error containerType
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = "errorType"
+	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
+
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	s := &service{
+		id:         testContainerID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+		config:     &runtimeConfig,
+	}
+
+	req := &taskAPI.CreateTaskRequest{
+		ID:       testContainerID,
+		Bundle:   bundlePath,
+		Terminal: true,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Create(ctx, req)
+	assert.Error(err)
+}

containerd-shim-v2/delete.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"path"
+
+	"github.com/containerd/containerd/mount"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/sirupsen/logrus"
+)
+
+func deleteContainer(ctx context.Context, s *service, c *container) error {
+
+	status, err := s.sandbox.StatusContainer(c.id)
+	if err != nil {
+		return err
+	}
+	if status.State.State != vc.StateStopped {
+		_, err = s.sandbox.StopContainer(c.id)
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err = s.sandbox.DeleteContainer(c.id)
+	if err != nil {
+		return err
+	}
+
+	// Run post-stop OCI hooks.
+	if err := katautils.PostStopHooks(ctx, *c.spec, s.sandbox.ID(), c.bundle); err != nil {
+		return err
+	}
+
+	rootfs := path.Join(c.bundle, "rootfs")
+	if err := mount.UnmountAll(rootfs, 0); err != nil {
+		logrus.WithError(err).Warn("failed to cleanup rootfs mount")
+	}
+
+	delete(s.containers, c.id)
+
+	return nil
+}

containerd-shim-v2/delete_test.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDeleteContainerSuccessAndFail(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	rootPath, configPath := testConfigSetup(t)
+	defer os.RemoveAll(rootPath)
+	_, err := readOCIConfigJSON(configPath)
+	assert.NoError(err)
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testContainerID,
+	}
+	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil)
+	assert.NoError(err)
+}
+
+func testConfigSetup(t *testing.T) (rootPath string, configPath string) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+	err = os.MkdirAll(bundlePath, testDirMode)
+	assert.NoError(err)
+
+	err = createOCIConfig(bundlePath)
+	assert.NoError(err)
+
+	// config json path
+	configPath = filepath.Join(bundlePath, "config.json")
+	return tmpdir, configPath
+}

containerd-shim-v2/exec.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/api/types/task"
+	"github.com/containerd/containerd/errdefs"
+	googleProtobuf "github.com/gogo/protobuf/types"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type exec struct {
+	container *container
+	cmds      *vc.Cmd
+	tty       *tty
+	ttyio     *ttyIO
+	id        string
+
+	exitCode int32
+
+	status task.Status
+
+	exitIOch chan struct{}
+	exitCh   chan uint32
+
+	exitTime time.Time
+}
+
+type tty struct {
+	stdin    string
+	stdout   string
+	stderr   string
+	height   uint32
+	width    uint32
+	terminal bool
+}
+
+func getEnvs(envs []string) []vc.EnvVar {
+	var vcEnvs = []vc.EnvVar{}
+	var env vc.EnvVar
+
+	for _, v := range envs {
+		pair := strings.SplitN(v, "=", 2)
+
+		if len(pair) == 2 {
+			env = vc.EnvVar{Var: pair[0], Value: pair[1]}
+		} else if len(pair) == 1 {
+			env = vc.EnvVar{Var: pair[0], Value: ""}
+		}
+
+		vcEnvs = append(vcEnvs, env)
+	}
+
+	return vcEnvs
+}
+
+func newExec(c *container, stdin, stdout, stderr string, terminal bool, jspec *googleProtobuf.Any) (*exec, error) {
+	var height uint32
+	var width uint32
+
+	if jspec == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrInvalidArgument, "googleProtobuf.Any points to nil")
+	}
+
+	// process exec request
+	var spec specs.Process
+	if err := json.Unmarshal(jspec.Value, &spec); err != nil {
+		return nil, err
+	}
+
+	if spec.ConsoleSize != nil {
+		height = uint32(spec.ConsoleSize.Height)
+		width = uint32(spec.ConsoleSize.Width)
+	}
+
+	tty := &tty{
+		stdin:    stdin,
+		stdout:   stdout,
+		stderr:   stderr,
+		height:   height,
+		width:    width,
+		terminal: terminal,
+	}
+
+	cmds := &vc.Cmd{
+		Args:            spec.Args,
+		Envs:            getEnvs(spec.Env),
+		User:            fmt.Sprintf("%d", spec.User.UID),
+		PrimaryGroup:    fmt.Sprintf("%d", spec.User.GID),
+		WorkDir:         spec.Cwd,
+		Interactive:     terminal,
+		Detach:          !terminal,
+		NoNewPrivileges: spec.NoNewPrivileges,
+	}
+
+	exec := &exec{
+		container: c,
+		cmds:      cmds,
+		tty:       tty,
+		exitCode:  exitCode255,
+		exitIOch:  make(chan struct{}),
+		exitCh:    make(chan uint32, 1),
+		status:    task.StatusCreated,
+	}
+
+	return exec, nil
+}
+
+func (c *container) getExec(id string) (*exec, error) {
+	if c.execs == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "exec does not exist %s", id)
+	}
+
+	exec := c.execs[id]
+
+	if exec == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "exec does not exist %s", id)
+	}
+
+	return exec, nil
+}

containerd-shim-v2/exec_test.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"testing"
+
+	"github.com/containerd/containerd/namespaces"
+
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExecNoSpecFail(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testContainerID,
+	}
+
+	var err error
+	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil)
+	assert.NoError(err)
+
+	reqExec := &taskAPI.ExecProcessRequest{
+		ID:     testContainerID,
+		ExecID: testContainerID,
+	}
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+
+	_, err = s.Exec(ctx, reqExec)
+	assert.Error(err)
+}

containerd-shim-v2/metrics.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"github.com/containerd/cgroups"
+	"github.com/containerd/typeurl"
+
+	google_protobuf "github.com/gogo/protobuf/types"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+)
+
+func marshalMetrics(s *service, containerID string) (*google_protobuf.Any, error) {
+	stats, err := s.sandbox.StatsContainer(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	metrics := statsToMetrics(stats.CgroupStats)
+
+	data, err := typeurl.MarshalAny(metrics)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+func statsToMetrics(cgStats *vc.CgroupStats) *cgroups.Metrics {
+	var hugetlb []*cgroups.HugetlbStat
+	for _, v := range cgStats.HugetlbStats {
+		hugetlb = append(
+			hugetlb,
+			&cgroups.HugetlbStat{
+				Usage:   v.Usage,
+				Max:     v.MaxUsage,
+				Failcnt: v.Failcnt,
+			})
+	}
+
+	var perCPU []uint64
+	for _, v := range cgStats.CPUStats.CPUUsage.PercpuUsage {
+		perCPU = append(perCPU, v)
+	}
+
+	metrics := &cgroups.Metrics{
+		Hugetlb: hugetlb,
+		Pids: &cgroups.PidsStat{
+			Current: cgStats.PidsStats.Current,
+			Limit:   cgStats.PidsStats.Limit,
+		},
+		CPU: &cgroups.CPUStat{
+			Usage: &cgroups.CPUUsage{
+				Total:  cgStats.CPUStats.CPUUsage.TotalUsage,
+				PerCPU: perCPU,
+			},
+		},
+		Memory: &cgroups.MemoryStat{
+			Cache: cgStats.MemoryStats.Cache,
+			Usage: &cgroups.MemoryEntry{
+				Limit: cgStats.MemoryStats.Usage.Limit,
+				Usage: cgStats.MemoryStats.Usage.Usage,
+			},
+		},
+	}
+
+	return metrics
+}

containerd-shim-v2/pause_test.go

@@ -0,0 +1,202 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"testing"
+
+	"github.com/containerd/containerd/namespaces"
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPauseContainerSuccess(t *testing.T) {
+	assert := assert.New(t)
+	var err error
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.PauseContainerFunc = func(ctx context.Context, sandboxID, containerID string) error {
+		return nil
+	}
+	defer func() {
+		testingImpl.PauseContainerFunc = nil
+	}()
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          testContainerID,
+			Annotations: make(map[string]string),
+			State: vc.State{
+				State: vc.StateRunning,
+			},
+		}, nil
+	}
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testContainerID,
+	}
+	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil)
+	assert.NoError(err)
+
+	reqPause := &taskAPI.PauseRequest{
+		ID: testContainerID,
+	}
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+
+	_, err = s.Pause(ctx, reqPause)
+	assert.NoError(err)
+}
+
+func TestPauseContainerFail(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.PauseContainerFunc = func(ctx context.Context, sandboxID, containerID string) error {
+		return nil
+	}
+	defer func() {
+		testingImpl.PauseContainerFunc = nil
+	}()
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          testContainerID,
+			Annotations: make(map[string]string),
+			State: vc.State{
+				State: vc.StateRunning,
+			},
+		}, nil
+	}
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqPause := &taskAPI.PauseRequest{
+		ID: testContainerID,
+	}
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+
+	_, err := s.Pause(ctx, reqPause)
+	assert.Error(err)
+}
+
+func TestResumeContainerSuccess(t *testing.T) {
+	assert := assert.New(t)
+	var err error
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.ResumeContainerFunc = func(ctx context.Context, sandboxID, containerID string) error {
+		return nil
+	}
+	defer func() {
+		testingImpl.ResumeContainerFunc = nil
+	}()
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          testContainerID,
+			Annotations: make(map[string]string),
+			State: vc.State{
+				State: vc.StateRunning,
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testContainerID,
+	}
+	s.containers[testContainerID], err = newContainer(s, reqCreate, "", nil)
+	assert.NoError(err)
+
+	reqResume := &taskAPI.ResumeRequest{
+		ID: testContainerID,
+	}
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+
+	_, err = s.Resume(ctx, reqResume)
+	assert.NoError(err)
+}
+
+func TestResumeContainerFail(t *testing.T) {
+	assert := assert.New(t)
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.ResumeContainerFunc = func(ctx context.Context, sandboxID, containerID string) error {
+		return nil
+	}
+	defer func() {
+		testingImpl.ResumeContainerFunc = nil
+	}()
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          testContainerID,
+			Annotations: make(map[string]string),
+			State: vc.State{
+				State: vc.StateRunning,
+			},
+		}, nil
+	}
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqResume := &taskAPI.ResumeRequest{
+		ID: testContainerID,
+	}
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+
+	_, err := s.Resume(ctx, reqResume)
+	assert.Error(err)
+}

containerd-shim-v2/service.go

@@ -0,0 +1,788 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	sysexec "os/exec"
+	"path/filepath"
+	"sync"
+	"syscall"
+	"time"
+
+	eventstypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/namespaces"
+	cdruntime "github.com/containerd/containerd/runtime"
+	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+
+	"github.com/containerd/containerd/api/types/task"
+	ptypes "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// Define the service's channel size, which is used for
+	// reaping the exited processes exit state and forwarding
+	// it to containerd as the containerd event format.
+	bufferSize = 32
+
+	chSize      = 128
+	exitCode255 = 255
+)
+
+var (
+	empty                     = &ptypes.Empty{}
+	_     taskAPI.TaskService = (taskAPI.TaskService)(&service{})
+)
+
+// concrete virtcontainer implementation
+var vci vc.VC = &vc.VCImpl{}
+
+// New returns a new shim service that can be used via GRPC
+func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shim, error) {
+	logger := logrus.WithField("ID", id)
+	vci.SetLogger(ctx, logger)
+	katautils.SetLogger(ctx, logger, logger.Logger.Level)
+	_, runtimeConfig, err := katautils.LoadConfiguration("", true, true)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &service{
+		id:         id,
+		pid:        uint32(os.Getpid()),
+		context:    ctx,
+		config:     &runtimeConfig,
+		containers: make(map[string]*container),
+		events:     make(chan interface{}, chSize),
+		ec:         make(chan exit, bufferSize),
+	}
+
+	go s.processExits()
+
+	go s.forward(publisher)
+
+	return s, nil
+}
+
+type exit struct {
+	id        string
+	execid    string
+	pid       uint32
+	status    int
+	timestamp time.Time
+}
+
+// service is the shim implementation of a remote shim over GRPC
+type service struct {
+	sync.Mutex
+
+	// pid Since this shimv2 cannot get the container processes pid from VM,
+	// thus for the returned values needed pid, just return this shim's
+	// pid directly.
+	pid uint32
+
+	context    context.Context
+	sandbox    vc.VCSandbox
+	containers map[string]*container
+	config     *oci.RuntimeConfig
+	events     chan interface{}
+
+	ec chan exit
+	id string
+}
+
+func newCommand(ctx context.Context, containerdBinary, id, containerdAddress string) (*sysexec.Cmd, error) {
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, err
+	}
+	self, err := os.Executable()
+	if err != nil {
+		return nil, err
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	args := []string{
+		"-namespace", ns,
+		"-address", containerdAddress,
+		"-publish-binary", containerdBinary,
+		"-id", id,
+		"-debug",
+	}
+	cmd := sysexec.Command(self, args...)
+	cmd.Dir = cwd
+
+	// Set the go max process to 2 in case the shim forks too much process
+	cmd.Env = append(os.Environ(), "GOMAXPROCS=2")
+
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	return cmd, nil
+}
+
+// StartShim willl start a kata shimv2 daemon which will implemented the
+// ShimV2 APIs such as create/start/update etc containers.
+func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
+	bundlePath, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	address, err := getAddress(ctx, bundlePath, id)
+	if err != nil {
+		return "", err
+	}
+	if address != "" {
+		if err := cdshim.WriteAddress("address", address); err != nil {
+			return "", err
+		}
+		return address, nil
+	}
+
+	cmd, err := newCommand(ctx, containerdBinary, id, containerdAddress)
+	if err != nil {
+		return "", err
+	}
+
+	address, err = cdshim.SocketAddress(ctx, id)
+	if err != nil {
+		return "", err
+	}
+
+	socket, err := cdshim.NewSocket(address)
+	if err != nil {
+		return "", err
+	}
+	defer socket.Close()
+	f, err := socket.File()
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, f)
+
+	if err := cmd.Start(); err != nil {
+		return "", err
+	}
+	defer func() {
+		if err != nil {
+			cmd.Process.Kill()
+		}
+	}()
+
+	// make sure to wait after start
+	go cmd.Wait()
+	if err := cdshim.WritePidFile("shim.pid", cmd.Process.Pid); err != nil {
+		return "", err
+	}
+	if err := cdshim.WriteAddress("address", address); err != nil {
+		return "", err
+	}
+	return address, nil
+}
+
+func (s *service) forward(publisher events.Publisher) {
+	for e := range s.events {
+		if err := publisher.Publish(s.context, getTopic(s.context, e), e); err != nil {
+			logrus.WithError(err).Error("post event")
+		}
+	}
+}
+
+func getTopic(ctx context.Context, e interface{}) string {
+	switch e.(type) {
+	case *eventstypes.TaskCreate:
+		return cdruntime.TaskCreateEventTopic
+	case *eventstypes.TaskStart:
+		return cdruntime.TaskStartEventTopic
+	case *eventstypes.TaskOOM:
+		return cdruntime.TaskOOMEventTopic
+	case *eventstypes.TaskExit:
+		return cdruntime.TaskExitEventTopic
+	case *eventstypes.TaskDelete:
+		return cdruntime.TaskDeleteEventTopic
+	case *eventstypes.TaskExecAdded:
+		return cdruntime.TaskExecAddedEventTopic
+	case *eventstypes.TaskExecStarted:
+		return cdruntime.TaskExecStartedEventTopic
+	case *eventstypes.TaskPaused:
+		return cdruntime.TaskPausedEventTopic
+	case *eventstypes.TaskResumed:
+		return cdruntime.TaskResumedEventTopic
+	case *eventstypes.TaskCheckpointed:
+		return cdruntime.TaskCheckpointedEventTopic
+	default:
+		logrus.Warnf("no topic for type %#v", e)
+	}
+	return cdruntime.TaskUnknownTopic
+}
+
+func (s *service) Cleanup(ctx context.Context) (*taskAPI.DeleteResponse, error) {
+	//Since the binary cleanup will return the DeleteResponse from stdout to
+	//containerd, thus we must make sure there is no any outputs in stdout except
+	//the returned response, thus here redirect the log to stderr in case there's
+	//any log output to stdout.
+	logrus.SetOutput(os.Stderr)
+
+	if s.id == "" {
+		return nil, errdefs.ToGRPCf(errdefs.ErrInvalidArgument, "the container id is empty, please specify the container id")
+	}
+
+	path, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+
+	ociSpec, err := oci.ParseConfigJSON(path)
+	if err != nil {
+		return nil, err
+	}
+
+	containerType, err := ociSpec.ContainerType()
+	if err != nil {
+		return nil, err
+	}
+
+	switch containerType {
+	case vc.PodSandbox:
+		err = cleanupContainer(ctx, s.id, s.id, path)
+		if err != nil {
+			return nil, err
+		}
+	case vc.PodContainer:
+		sandboxID, err := ociSpec.SandboxID()
+		if err != nil {
+			return nil, err
+		}
+
+		err = cleanupContainer(ctx, sandboxID, s.id, path)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &taskAPI.DeleteResponse{
+		ExitedAt:   time.Now(),
+		ExitStatus: 128 + uint32(unix.SIGKILL),
+	}, nil
+}
+
+// Create a new sandbox or container with the underlying OCI runtime
+func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *taskAPI.CreateTaskResponse, err error) {
+	s.Lock()
+	defer s.Unlock()
+
+	//the network namespace created by cni plugin
+	netns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "create namespace")
+	}
+
+	rootfs := filepath.Join(r.Bundle, "rootfs")
+	defer func() {
+		if err != nil {
+			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+				logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
+			}
+		}
+	}()
+	for _, rm := range r.Rootfs {
+		m := &mount.Mount{
+			Type:    rm.Type,
+			Source:  rm.Source,
+			Options: rm.Options,
+		}
+		if err := m.Mount(rootfs); err != nil {
+			return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
+		}
+	}
+
+	container, err := create(ctx, s, r, netns, s.config)
+	if err != nil {
+		return nil, err
+	}
+
+	container.status = task.StatusCreated
+
+	s.containers[r.ID] = container
+
+	return &taskAPI.CreateTaskResponse{
+		Pid: s.pid,
+	}, nil
+}
+
+// Start a process
+func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (*taskAPI.StartResponse, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	//start a container
+	if r.ExecID == "" {
+		err = startContainer(ctx, s, c)
+		if err != nil {
+			return nil, errdefs.ToGRPC(err)
+		}
+	} else {
+		//start an exec
+		_, err = startExec(ctx, s, r.ID, r.ExecID)
+		if err != nil {
+			return nil, errdefs.ToGRPC(err)
+		}
+	}
+
+	return &taskAPI.StartResponse{
+		Pid: s.pid,
+	}, nil
+}
+
+// Delete the initial process and container
+func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAPI.DeleteResponse, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	if r.ExecID == "" {
+		err = deleteContainer(ctx, s, c)
+		if err != nil {
+			return nil, err
+		}
+
+		return &taskAPI.DeleteResponse{
+			ExitStatus: c.exit,
+			ExitedAt:   c.time,
+			Pid:        s.pid,
+		}, nil
+	}
+	//deal with the exec case
+	execs, err := c.getExec(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+
+	delete(c.execs, r.ExecID)
+
+	return &taskAPI.DeleteResponse{
+		ExitStatus: uint32(execs.exitCode),
+		ExitedAt:   execs.exitTime,
+		Pid:        s.pid,
+	}, nil
+}
+
+// Exec an additional process inside the container
+func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	if execs := c.execs[r.ExecID]; execs != nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrAlreadyExists, "id %s", r.ExecID)
+	}
+
+	execs, err := newExec(c, r.Stdin, r.Stdout, r.Stderr, r.Terminal, r.Spec)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+
+	c.execs[r.ExecID] = execs
+
+	return empty, nil
+}
+
+// ResizePty of a process
+func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	processID := c.id
+	if r.ExecID != "" {
+		execs, err := c.getExec(r.ExecID)
+		if err != nil {
+			return nil, err
+		}
+		execs.tty.height = r.Height
+		execs.tty.width = r.Width
+
+		processID = execs.id
+
+	}
+	err = s.sandbox.WinsizeProcess(c.id, processID, r.Height, r.Width)
+	if err != nil {
+		return nil, err
+	}
+
+	return empty, err
+}
+
+// State returns runtime state information for a process
+func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI.StateResponse, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	if r.ExecID == "" {
+		return &taskAPI.StateResponse{
+			ID:         c.id,
+			Bundle:     c.bundle,
+			Pid:        s.pid,
+			Status:     c.status,
+			Stdin:      c.stdin,
+			Stdout:     c.stdout,
+			Stderr:     c.stderr,
+			Terminal:   c.terminal,
+			ExitStatus: c.exit,
+		}, nil
+	}
+
+	//deal with exec case
+	execs, err := c.getExec(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+
+	return &taskAPI.StateResponse{
+		ID:         execs.id,
+		Bundle:     c.bundle,
+		Pid:        s.pid,
+		Status:     execs.status,
+		Stdin:      execs.tty.stdin,
+		Stdout:     execs.tty.stdout,
+		Stderr:     execs.tty.stderr,
+		Terminal:   execs.tty.terminal,
+		ExitStatus: uint32(execs.exitCode),
+	}, nil
+
+}
+
+// Pause the container
+func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	c.status = task.StatusPausing
+
+	err = s.sandbox.PauseContainer(r.ID)
+	if err == nil {
+		c.status = task.StatusPaused
+		return empty, nil
+	}
+
+	c.status, err = s.getContainerStatus(c.id)
+	if err != nil {
+		c.status = task.StatusUnknown
+	}
+
+	return empty, err
+}
+
+// Resume the container
+func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	err = s.sandbox.ResumeContainer(c.id)
+	if err == nil {
+		c.status = task.StatusRunning
+		return empty, nil
+	}
+
+	c.status, err = s.getContainerStatus(c.id)
+	if err != nil {
+		c.status = task.StatusUnknown
+	}
+
+	return empty, err
+}
+
+// Kill a process with the provided signal
+func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	signum := syscall.Signal(r.Signal)
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	processID := c.id
+	if r.ExecID != "" {
+		execs, err := c.getExec(r.ExecID)
+		if err != nil {
+			return nil, err
+		}
+		processID = execs.id
+	}
+
+	err = s.sandbox.SignalProcess(c.id, processID, signum, r.All)
+	if err != nil {
+		return nil, err
+	}
+
+	// Since the k8s will use the SIGTERM signal to stop a container by default, but
+	// some container processes would ignore this signal such as shell, thus it's better
+	// to resend another SIGKILL signal to make sure the container process terminated successfully.
+	if signum == syscall.SIGTERM {
+		err = s.sandbox.SignalProcess(c.id, processID, syscall.SIGKILL, r.All)
+	}
+
+	return empty, err
+}
+
+// Pids returns all pids inside the container
+// Since for kata, it cannot get the process's pid from VM,
+// thus only return the Shim's pid directly.
+func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (*taskAPI.PidsResponse, error) {
+	var processes []*task.ProcessInfo
+
+	pInfo := task.ProcessInfo{
+		Pid: s.pid,
+	}
+	processes = append(processes, &pInfo)
+
+	return &taskAPI.PidsResponse{
+		Processes: processes,
+	}, nil
+}
+
+// CloseIO of a process
+func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	tty := c.ttyio
+	if r.ExecID != "" {
+		execs, err := c.getExec(r.ExecID)
+		if err != nil {
+			return nil, err
+		}
+		tty = execs.ttyio
+	}
+
+	if tty != nil && tty.Stdin != nil {
+		if err := tty.Stdin.Close(); err != nil {
+			return nil, errors.Wrap(err, "close stdin")
+		}
+	}
+
+	return empty, nil
+}
+
+// Checkpoint the container
+func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (*ptypes.Empty, error) {
+	return nil, errdefs.ToGRPCf(errdefs.ErrNotImplemented, "service Checkpoint")
+}
+
+// Connect returns shim information such as the shim's pid
+func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (*taskAPI.ConnectResponse, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	return &taskAPI.ConnectResponse{
+		ShimPid: s.pid,
+		//Since kata cannot get the container's pid in VM, thus only return the shim's pid.
+		TaskPid: s.pid,
+	}, nil
+}
+
+func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	if len(s.containers) != 0 {
+		return empty, nil
+	}
+
+	defer os.Exit(0)
+
+	err := s.sandbox.Stop()
+	if err != nil {
+		logrus.WithField("sandbox", s.sandbox.ID()).Error("failed to stop sandbox")
+		return empty, err
+	}
+
+	err = s.sandbox.Delete()
+	if err != nil {
+		logrus.WithField("sandbox", s.sandbox.ID()).Error("failed to delete sandbox")
+	}
+
+	return empty, err
+}
+
+func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (*taskAPI.StatsResponse, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := marshalMetrics(s, c.id)
+	if err != nil {
+		return nil, err
+	}
+
+	return &taskAPI.StatsResponse{
+		Stats: data,
+	}, nil
+}
+
+// Update a running container
+func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (*ptypes.Empty, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	var resources specs.LinuxResources
+	if err := json.Unmarshal(r.Resources.Value, &resources); err != nil {
+		return empty, err
+	}
+
+	err := s.sandbox.UpdateContainer(r.ID, resources)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+
+	return empty, nil
+}
+
+// Wait for a process to exit
+func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (*taskAPI.WaitResponse, error) {
+	var ret uint32
+
+	s.Lock()
+	c, err := s.getContainer(r.ID)
+	s.Unlock()
+
+	if err != nil {
+		return nil, err
+	}
+
+	//wait for container
+	if r.ExecID == "" {
+		ret = <-c.exitCh
+	} else { //wait for exec
+		execs, err := c.getExec(r.ExecID)
+		if err != nil {
+			return nil, err
+		}
+		ret = <-execs.exitCh
+	}
+
+	return &taskAPI.WaitResponse{
+		ExitStatus: ret,
+	}, nil
+}
+
+func (s *service) processExits() {
+	for e := range s.ec {
+		s.checkProcesses(e)
+	}
+}
+
+func (s *service) checkProcesses(e exit) {
+	s.Lock()
+	defer s.Unlock()
+
+	id := e.execid
+	if id == "" {
+		id = e.id
+	}
+	s.events <- &eventstypes.TaskExit{
+		ContainerID: e.id,
+		ID:          id,
+		Pid:         e.pid,
+		ExitStatus:  uint32(e.status),
+		ExitedAt:    e.timestamp,
+	}
+	return
+}
+
+func (s *service) getContainer(id string) (*container, error) {
+	c := s.containers[id]
+
+	if c == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "container does not exist %s", id)
+	}
+
+	return c, nil
+}
+
+func (s *service) getContainerStatus(containerID string) (task.Status, error) {
+	cStatus, err := s.sandbox.StatusContainer(containerID)
+	if err != nil {
+		return task.StatusUnknown, err
+	}
+
+	var status task.Status
+	switch cStatus.State.State {
+	case vc.StateReady:
+		status = task.StatusCreated
+	case vc.StateRunning:
+		status = task.StatusRunning
+	case vc.StatePaused:
+		status = task.StatusPaused
+	case vc.StateStopped:
+		status = task.StatusStopped
+	}
+
+	return status, nil
+}

containerd-shim-v2/start.go

@@ -0,0 +1,115 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/containerd/containerd/api/types/task"
+	"github.com/kata-containers/runtime/pkg/katautils"
+)
+
+func startContainer(ctx context.Context, s *service, c *container) error {
+	//start a container
+	if c.cType == "" {
+		err := fmt.Errorf("Bug, the container %s type is empty", c.id)
+		return err
+	}
+
+	if s.sandbox == nil {
+		err := fmt.Errorf("Bug, the sandbox hasn't been created for this container %s", c.id)
+		return err
+	}
+
+	if c.cType.IsSandbox() {
+		err := s.sandbox.Start()
+		if err != nil {
+			return err
+		}
+	} else {
+		_, err := s.sandbox.StartContainer(c.id)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Run post-start OCI hooks.
+	err := katautils.EnterNetNS(s.sandbox.GetNetNs(), func() error {
+		return katautils.PostStartHooks(ctx, *c.spec, s.sandbox.ID(), c.bundle)
+	})
+	if err != nil {
+		return err
+	}
+
+	c.status = task.StatusRunning
+
+	stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id)
+	if err != nil {
+		return err
+	}
+
+	if c.stdin != "" || c.stdout != "" || c.stderr != "" {
+		tty, err := newTtyIO(ctx, c.stdin, c.stdout, c.stderr, c.terminal)
+		if err != nil {
+			return err
+		}
+		c.ttyio = tty
+		go ioCopy(c.exitIOch, tty, stdin, stdout, stderr)
+	} else {
+		//close the io exit channel, since there is no io for this container,
+		//otherwise the following wait goroutine will hang on this channel.
+		close(c.exitIOch)
+	}
+
+	go wait(s, c, "")
+
+	return nil
+}
+
+func startExec(ctx context.Context, s *service, containerID, execID string) (*exec, error) {
+	//start an exec
+	c, err := s.getContainer(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	execs, err := c.getExec(execID)
+	if err != nil {
+		return nil, err
+	}
+
+	_, proc, err := s.sandbox.EnterContainer(containerID, *execs.cmds)
+	if err != nil {
+		err := fmt.Errorf("cannot enter container %s, with err %s", containerID, err)
+		return nil, err
+	}
+	execs.id = proc.Token
+
+	execs.status = task.StatusRunning
+	if execs.tty.height != 0 && execs.tty.width != 0 {
+		err = s.sandbox.WinsizeProcess(c.id, execs.id, execs.tty.height, execs.tty.width)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, execs.id)
+	if err != nil {
+		return nil, err
+	}
+	tty, err := newTtyIO(ctx, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal)
+	if err != nil {
+		return nil, err
+	}
+	execs.ttyio = tty
+
+	go ioCopy(execs.exitIOch, tty, stdin, stdout, stderr)
+
+	go wait(s, c, execID)
+
+	return execs, nil
+}

containerd-shim-v2/start_test.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"testing"
+
+	"github.com/containerd/containerd/namespaces"
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStartStartSandboxSuccess(t *testing.T) {
+	assert := assert.New(t)
+	var err error
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: sandbox.ID(),
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodSandbox),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testSandboxID,
+	}
+	s.containers[testSandboxID], err = newContainer(s, reqCreate, vc.PodSandbox, nil)
+	assert.NoError(err)
+
+	reqStart := &taskAPI.StartRequest{
+		ID: testSandboxID,
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StartSandboxFunc = nil
+	}()
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Start(ctx, reqStart)
+	assert.NoError(err)
+}
+
+func TestStartMissingAnnotation(t *testing.T) {
+	assert := assert.New(t)
+	var err error
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID:          sandbox.ID(),
+			Annotations: map[string]string{},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testSandboxID,
+	}
+	s.containers[testSandboxID], err = newContainer(s, reqCreate, "", nil)
+	assert.NoError(err)
+
+	reqStart := &taskAPI.StartRequest{
+		ID: testSandboxID,
+	}
+
+	testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
+		return sandbox, nil
+	}
+
+	defer func() {
+		testingImpl.StartSandboxFunc = nil
+	}()
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Start(ctx, reqStart)
+	assert.Error(err)
+	assert.False(vcmock.IsMockError(err))
+}
+
+func TestStartStartContainerSucess(t *testing.T) {
+	assert := assert.New(t)
+	var err error
+
+	sandbox := &vcmock.Sandbox{
+		MockID: testSandboxID,
+	}
+
+	sandbox.MockContainers = []*vcmock.Container{
+		{
+			MockID:      testContainerID,
+			MockSandbox: sandbox,
+		},
+	}
+
+	testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
+		return vc.ContainerStatus{
+			ID: testContainerID,
+			Annotations: map[string]string{
+				vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
+			},
+		}, nil
+	}
+
+	defer func() {
+		testingImpl.StatusContainerFunc = nil
+	}()
+
+	testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) {
+		return sandbox.MockContainers[0], nil
+	}
+
+	defer func() {
+		testingImpl.StartContainerFunc = nil
+	}()
+
+	s := &service{
+		id:         testSandboxID,
+		sandbox:    sandbox,
+		containers: make(map[string]*container),
+	}
+
+	reqCreate := &taskAPI.CreateTaskRequest{
+		ID: testContainerID,
+	}
+	s.containers[testContainerID], err = newContainer(s, reqCreate, vc.PodContainer, nil)
+	assert.NoError(err)
+
+	reqStart := &taskAPI.StartRequest{
+		ID: testContainerID,
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "UnitTest")
+	_, err = s.Start(ctx, reqStart)
+	assert.NoError(err)
+}

containerd-shim-v2/stream.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"io"
+	"sync"
+	"syscall"
+
+	"github.com/containerd/fifo"
+)
+
+// The buffer size used to specify the buffer for IO streams copy
+const bufSize = 32 << 10
+
+var (
+	bufPool = sync.Pool{
+		New: func() interface{} {
+			buffer := make([]byte, bufSize)
+			return &buffer
+		},
+	}
+)
+
+type ttyIO struct {
+	Stdin  io.ReadCloser
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+func (tty *ttyIO) close() {
+
+	if tty.Stdin != nil {
+		tty.Stdin.Close()
+	}
+	cf := func(w io.Writer) {
+		if w == nil {
+			return
+		}
+		if c, ok := w.(io.WriteCloser); ok {
+			c.Close()
+		}
+	}
+	cf(tty.Stdout)
+	cf(tty.Stderr)
+}
+
+func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (*ttyIO, error) {
+	var in io.ReadCloser
+	var outw io.Writer
+	var errw io.Writer
+	var err error
+
+	if stdin != "" {
+		in, err = fifo.OpenFifo(ctx, stdin, syscall.O_RDONLY, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if stdout != "" {
+		outw, err = fifo.OpenFifo(ctx, stdout, syscall.O_WRONLY, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if !console && stderr != "" {
+		errw, err = fifo.OpenFifo(ctx, stderr, syscall.O_WRONLY, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	ttyIO := &ttyIO{
+		Stdin:  in,
+		Stdout: outw,
+		Stderr: errw,
+	}
+
+	return ttyIO, nil
+}
+
+func ioCopy(exitch chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) {
+	var wg sync.WaitGroup
+	var closeOnce sync.Once
+
+	if tty.Stdin != nil {
+		wg.Add(1)
+		go func() {
+			p := bufPool.Get().(*[]byte)
+			defer bufPool.Put(p)
+			io.CopyBuffer(stdinPipe, tty.Stdin, *p)
+			wg.Done()
+		}()
+	}
+
+	if tty.Stdout != nil {
+		wg.Add(1)
+
+		go func() {
+			p := bufPool.Get().(*[]byte)
+			defer bufPool.Put(p)
+			io.CopyBuffer(tty.Stdout, stdoutPipe, *p)
+			wg.Done()
+			closeOnce.Do(tty.close)
+		}()
+	}
+
+	if tty.Stderr != nil && stderrPipe != nil {
+		wg.Add(1)
+		go func() {
+			p := bufPool.Get().(*[]byte)
+			defer bufPool.Put(p)
+			io.CopyBuffer(tty.Stderr, stderrPipe, *p)
+			wg.Done()
+		}()
+	}
+
+	wg.Wait()
+	closeOnce.Do(tty.close)
+	close(exitch)
+}

containerd-shim-v2/utils.go

@@ -0,0 +1,169 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/mount"
+	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+func cReap(s *service, status int, id, execid string, exitat time.Time) {
+	s.ec <- exit{
+		timestamp: exitat,
+		pid:       s.pid,
+		status:    status,
+		id:        id,
+		execid:    execid,
+	}
+}
+
+func cleanupContainer(ctx context.Context, sid, cid, bundlePath string) error {
+	logrus.WithField("Service", "Cleanup").WithField("container", cid).Info("Cleanup container")
+
+	rootfs := filepath.Join(bundlePath, "rootfs")
+	sandbox, err := vci.FetchSandbox(ctx, sid)
+	if err != nil {
+		return err
+	}
+
+	status, err := sandbox.StatusContainer(cid)
+	if err != nil {
+		logrus.WithError(err).WithField("container", cid).Warn("failed to get container status")
+		return err
+	}
+
+	if oci.StateToOCIState(status.State) != oci.StateStopped {
+		err := sandbox.KillContainer(cid, syscall.SIGKILL, true)
+		if err != nil {
+			logrus.WithError(err).WithField("container", cid).Warn("failed to kill container")
+			return err
+		}
+	}
+
+	if _, err = sandbox.StopContainer(cid); err != nil {
+		logrus.WithError(err).WithField("container", cid).Warn("failed to stop container")
+		return err
+	}
+
+	if _, err := sandbox.DeleteContainer(cid); err != nil {
+		logrus.WithError(err).WithField("container", cid).Warn("failed to remove container")
+	}
+
+	if err := mount.UnmountAll(rootfs, 0); err != nil {
+		logrus.WithError(err).WithField("container", cid).Warn("failed to cleanup container rootfs")
+	}
+
+	if len(sandbox.GetAllContainers()) == 0 {
+		err = sandbox.Stop()
+		if err != nil {
+			logrus.WithError(err).WithField("sandbox", sid).Warn("failed to stop sandbox")
+			return err
+		}
+
+		err = sandbox.Delete()
+		if err != nil {
+			logrus.WithError(err).WithField("sandbox", sid).Warnf("failed to delete sandbox")
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validBundle(containerID, bundlePath string) (string, error) {
+	// container ID MUST be provided.
+	if containerID == "" {
+		return "", fmt.Errorf("Missing container ID")
+	}
+
+	// bundle path MUST be provided.
+	if bundlePath == "" {
+		return "", fmt.Errorf("Missing bundle path")
+	}
+
+	// bundle path MUST be valid.
+	fileInfo, err := os.Stat(bundlePath)
+	if err != nil {
+		return "", fmt.Errorf("Invalid bundle path '%s': %s", bundlePath, err)
+	}
+	if fileInfo.IsDir() == false {
+		return "", fmt.Errorf("Invalid bundle path '%s', it should be a directory", bundlePath)
+	}
+
+	resolved, err := katautils.ResolvePath(bundlePath)
+	if err != nil {
+		return "", err
+	}
+
+	return resolved, nil
+}
+
+func getAddress(ctx context.Context, bundlePath, id string) (string, error) {
+	var err error
+
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	if bundlePath, err = validBundle(id, bundlePath); err != nil {
+		return "", err
+	}
+
+	ociSpec, err := oci.ParseConfigJSON(bundlePath)
+	if err != nil {
+		return "", err
+	}
+
+	containerType, err := ociSpec.ContainerType()
+	if err != nil {
+		return "", err
+	}
+
+	if containerType == vc.PodContainer {
+		sandboxID, err := ociSpec.SandboxID()
+		if err != nil {
+			return "", err
+		}
+		address, err := cdshim.SocketAddress(ctx, sandboxID)
+		if err != nil {
+			return "", err
+		}
+		return address, nil
+	}
+
+	return "", nil
+}
+
+func noNeedForOutput(detach bool, tty bool) bool {
+	if !detach {
+		return false
+	}
+
+	if !tty {
+		return false
+	}
+
+	return true
+}
+
+func removeNamespace(s *oci.CompatOCISpec, nsType specs.LinuxNamespaceType) {
+	for i, n := range s.Linux.Namespaces {
+		if n.Type == nsType {
+			s.Linux.Namespaces = append(s.Linux.Namespaces[:i], s.Linux.Namespaces[i+1:]...)
+			return
+		}
+	}
+}

containerd-shim-v2/utils_test.go

@@ -0,0 +1,375 @@
+// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	sysExec "os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/kata-containers/runtime/pkg/katautils"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+)
+
+const (
+	// specConf is the name of the file holding the containers configuration
+	specConf = "config.json"
+
+	TestID = "container_test"
+
+	testDirMode  = os.FileMode(0750)
+	testFileMode = os.FileMode(0640)
+	// testExeFileMode         = os.FileMode(0750)
+
+	// small docker image used to create root filesystems from
+	testDockerImage = "busybox"
+
+	testSandboxID   = "777-77-77777777"
+	testContainerID = "42"
+	testBundle      = "bundle"
+	testConsole     = "/dev/pts/888"
+
+	testDisabledNeedRoot = "Test disabled as requires root user"
+
+	testContainerTypeAnnotation = "io.kubernetes.cri.container-type"
+	testSandboxIDAnnotation     = "io.kubernetes.cri.sandbox-id"
+	testContainerTypeSandbox    = "sandbox"
+	testContainerTypeContainer  = "container"
+)
+
+var (
+	// package variables set by calling TestMain()
+	testDir       = ""
+	testBundleDir = ""
+)
+
+// testingImpl is a concrete mock RVC implementation used for testing
+var testingImpl = &vcmock.VCMock{}
+
+func init() {
+	fmt.Printf("INFO: running as actual user %v (effective %v), actual group %v (effective %v)\n",
+		os.Getuid(), os.Geteuid(), os.Getgid(), os.Getegid())
+
+	fmt.Printf("INFO: switching to fake virtcontainers implementation for testing\n")
+	vci = testingImpl
+
+	var err error
+
+	fmt.Printf("INFO: creating test directory\n")
+	testDir, err = ioutil.TempDir("", fmt.Sprintf("shimV2-"))
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: failed to create test directory: %v", err))
+	}
+
+	fmt.Printf("INFO: test directory is %v\n", testDir)
+
+	fmt.Printf("INFO: ensuring docker is running\n")
+	output, err := katautils.RunCommandFull([]string{"docker", "version"}, true)
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: docker daemon is not installed, not running, or not accessible to current user: %v (error %v)",
+			output, err))
+	}
+
+	// Do this now to avoid hitting the test timeout value due to
+	// slow network response.
+	fmt.Printf("INFO: ensuring required docker image (%v) is available\n", testDockerImage)
+
+	// Only hit the network if the image doesn't exist locally
+	_, err = katautils.RunCommand([]string{"docker", "inspect", "--type=image", testDockerImage})
+	if err == nil {
+		fmt.Printf("INFO: docker image %v already exists locally\n", testDockerImage)
+	} else {
+		_, err = katautils.RunCommand([]string{"docker", "pull", testDockerImage})
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	testBundleDir = filepath.Join(testDir, testBundle)
+	err = os.MkdirAll(testBundleDir, testDirMode)
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
+	}
+
+	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
+	err = realMakeOCIBundle(testBundleDir)
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
+	}
+}
+
+// createOCIConfig creates an OCI configuration (spec) file in
+// the bundle directory specified (which must exist).
+func createOCIConfig(bundleDir string) error {
+	if bundleDir == "" {
+		return errors.New("BUG: Need bundle directory")
+	}
+
+	if !katautils.FileExists(bundleDir) {
+		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
+	}
+
+	var configCmd string
+
+	// Search for a suitable version of runc to use to generate
+	// the OCI config file.
+	for _, cmd := range []string{"docker-runc", "runc"} {
+		fullPath, err := sysExec.LookPath(cmd)
+		if err == nil {
+			configCmd = fullPath
+			break
+		}
+	}
+
+	if configCmd == "" {
+		return errors.New("Cannot find command to generate OCI config file")
+	}
+
+	_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	if err != nil {
+		return err
+	}
+
+	specFile := filepath.Join(bundleDir, specConf)
+	if !katautils.FileExists(specFile) {
+		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
+	}
+
+	return nil
+}
+
+func createEmptyFile(path string) (err error) {
+	return ioutil.WriteFile(path, []byte(""), testFileMode)
+}
+
+// newTestHypervisorConfig creaets a new virtcontainers
+// HypervisorConfig, ensuring that the required resources are also
+// created.
+//
+// Note: no parameter validation in case caller wishes to create an invalid
+// object.
+func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) {
+	kernelPath := path.Join(dir, "kernel")
+	imagePath := path.Join(dir, "image")
+	hypervisorPath := path.Join(dir, "hypervisor")
+
+	if create {
+		for _, file := range []string{kernelPath, imagePath, hypervisorPath} {
+			err := createEmptyFile(file)
+			if err != nil {
+				return vc.HypervisorConfig{}, err
+			}
+		}
+	}
+
+	return vc.HypervisorConfig{
+		KernelPath:            kernelPath,
+		ImagePath:             imagePath,
+		HypervisorPath:        hypervisorPath,
+		HypervisorMachineType: "pc-lite",
+	}, nil
+}
+
+// newTestRuntimeConfig creates a new RuntimeConfig
+func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
+	if dir == "" {
+		return oci.RuntimeConfig{}, errors.New("BUG: need directory")
+	}
+
+	hypervisorConfig, err := newTestHypervisorConfig(dir, create)
+	if err != nil {
+		return oci.RuntimeConfig{}, err
+	}
+
+	return oci.RuntimeConfig{
+		HypervisorType:   vc.QemuHypervisor,
+		HypervisorConfig: hypervisorConfig,
+		AgentType:        vc.KataContainersAgent,
+		ProxyType:        vc.KataBuiltInProxyType,
+		ShimType:         vc.KataBuiltInShimType,
+		Console:          consolePath,
+	}, nil
+}
+
+// readOCIConfig returns an OCI spec.
+func readOCIConfigFile(configPath string) (oci.CompatOCISpec, error) {
+	if configPath == "" {
+		return oci.CompatOCISpec{}, errors.New("BUG: need config file path")
+	}
+
+	data, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return oci.CompatOCISpec{}, err
+	}
+
+	var ociSpec oci.CompatOCISpec
+	if err := json.Unmarshal(data, &ociSpec); err != nil {
+		return oci.CompatOCISpec{}, err
+	}
+	caps, err := oci.ContainerCapabilities(ociSpec)
+	if err != nil {
+		return oci.CompatOCISpec{}, err
+	}
+	ociSpec.Process.Capabilities = caps
+	return ociSpec, nil
+}
+
+// realMakeOCIBundle will create an OCI bundle (including the "config.json"
+// config file) in the directory specified (which must already exist).
+//
+// XXX: Note that tests should *NOT* call this function - they should
+// XXX: instead call makeOCIBundle().
+func realMakeOCIBundle(bundleDir string) error {
+	if bundleDir == "" {
+		return errors.New("BUG: Need bundle directory")
+	}
+
+	if !katautils.FileExists(bundleDir) {
+		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
+	}
+
+	err := createOCIConfig(bundleDir)
+	if err != nil {
+		return err
+	}
+
+	// Note the unusual parameter (a directory, not the config
+	// file to parse!)
+	spec, err := oci.ParseConfigJSON(bundleDir)
+	if err != nil {
+		return err
+	}
+
+	// Determine the rootfs directory name the OCI config refers to
+	ociRootPath := spec.Root.Path
+
+	rootfsDir := filepath.Join(bundleDir, ociRootPath)
+
+	if strings.HasPrefix(ociRootPath, "/") {
+		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
+	}
+
+	err = createRootfs(rootfsDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Create an OCI bundle in the specified directory.
+//
+// Note that the directory will be created, but it's parent is expected to exist.
+//
+// This function works by copying the already-created test bundle. Ideally,
+// the bundle would be recreated for each test, but createRootfs() uses
+// docker which on some systems is too slow, resulting in the tests timing
+// out.
+func makeOCIBundle(bundleDir string) error {
+	from := testBundleDir
+	to := bundleDir
+
+	// only the basename of bundleDir needs to exist as bundleDir
+	// will get created by cp(1).
+	base := filepath.Dir(bundleDir)
+
+	for _, dir := range []string{from, base} {
+		if !katautils.FileExists(dir) {
+			return fmt.Errorf("BUG: directory %v should exist", dir)
+		}
+	}
+
+	output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
+	if err != nil {
+		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
+	}
+
+	return nil
+}
+
+// createRootfs creates a minimal root filesystem below the specified
+// directory.
+func createRootfs(dir string) error {
+	err := os.MkdirAll(dir, testDirMode)
+	if err != nil {
+		return err
+	}
+
+	container, err := katautils.RunCommand([]string{"docker", "create", testDockerImage})
+	if err != nil {
+		return err
+	}
+
+	cmd1 := sysExec.Command("docker", "export", container)
+	cmd2 := sysExec.Command("tar", "-C", dir, "-xvf", "-")
+
+	cmd1Stdout, err := cmd1.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	cmd2.Stdin = cmd1Stdout
+
+	err = cmd2.Start()
+	if err != nil {
+		return err
+	}
+
+	err = cmd1.Run()
+	if err != nil {
+		return err
+	}
+
+	err = cmd2.Wait()
+	if err != nil {
+		return err
+	}
+
+	// Clean up
+	_, err = katautils.RunCommand([]string{"docker", "rm", container})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func writeOCIConfigFile(spec oci.CompatOCISpec, configPath string) error {
+	if configPath == "" {
+		return errors.New("BUG: need config file path")
+	}
+
+	bytes, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(configPath, bytes, testFileMode)
+}
+
+// Read fail that should contain a CompatOCISpec and
+// return its JSON representation on success
+func readOCIConfigJSON(configFile string) (string, error) {
+	bundlePath := filepath.Dir(configFile)
+	ociSpec, err := oci.ParseConfigJSON(bundlePath)
+	if err != nil {
+		return "", nil
+	}
+	ociSpecJSON, err := json.Marshal(ociSpec)
+	if err != nil {
+		return "", err
+	}
+	return string(ociSpecJSON), err
+}

containerd-shim-v2/wait.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"time"
+
+	"github.com/containerd/containerd/api/types/task"
+	"github.com/sirupsen/logrus"
+)
+
+func wait(s *service, c *container, execID string) (int32, error) {
+	var execs *exec
+	var err error
+
+	processID := c.id
+
+	if execID == "" {
+		//wait until the io closed, then wait the container
+		<-c.exitIOch
+	} else {
+		execs, err = c.getExec(execID)
+		if err != nil {
+			return exitCode255, err
+		}
+		<-execs.exitIOch
+		//This wait could be triggered before exec start which
+		//will get the exec's id, thus this assignment must after
+		//the exec exit, to make sure it get the exec's id.
+		processID = execs.id
+	}
+
+	ret, err := s.sandbox.WaitProcess(c.id, processID)
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{
+			"container": c.id,
+			"pid":       processID,
+		}).Error("Wait for process failed")
+	}
+
+	if execID == "" {
+		c.exitCh <- uint32(ret)
+	} else {
+		execs.exitCh <- uint32(ret)
+	}
+
+	timeStamp := time.Now()
+	c.mu.Lock()
+	if execID == "" {
+		c.status = task.StatusStopped
+		c.exit = uint32(ret)
+		c.time = timeStamp
+	} else {
+		execs.status = task.StatusStopped
+		execs.exitCode = ret
+		execs.exitTime = timeStamp
+	}
+	c.mu.Unlock()
+
+	go cReap(s, int(ret), c.id, execID, timeStamp)
+
+	return ret, nil
+}

pkg/katautils/config-settings.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
@@ -15,6 +15,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
 var defaultFirmwarePath = ""
 var defaultMachineAccelerators = ""
 var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
+var systemdUnitName = "kata-containers.target"
 
 const defaultKernelParams = ""
 const defaultMachineType = "pc"

pkg/katautils/config.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
@@ -28,6 +28,9 @@ const (
 var (
 	defaultProxy = vc.KataProxyType
 	defaultShim  = vc.KataShimType
+
+	// if true, enable opentracing support.
+	tracing = false
 )
 
 // The TOML configuration file contains a number of sections (or
@@ -531,7 +534,7 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
 	return nil
 }
 
-func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) {
+func initConfig() (config oci.RuntimeConfig, err error) {
 	var defaultAgentConfig interface{}
 
 	defaultHypervisorConfig := vc.HypervisorConfig{
@@ -565,13 +568,6 @@ func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) {
 
 	defaultAgentConfig = vc.HyperConfig{}
 
-	if builtIn {
-		defaultProxy = vc.KataBuiltInProxyType
-		defaultShim = vc.KataBuiltInShimType
-
-		defaultAgentConfig = vc.KataAgentConfig{LongLiveConn: true}
-	}
-
 	config = oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: defaultHypervisorConfig,
@@ -592,12 +588,12 @@ func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) {
 //
 // All paths are resolved fully meaning if this function does not return an
 // error, all paths are valid at the time of the call.
-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, tracing bool, err error) {
+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
 	var resolved string
 
-	config, err = initConfig(builtIn)
+	config, err = initConfig()
 	if err != nil {
-		return "", oci.RuntimeConfig{}, tracing, err
+		return "", oci.RuntimeConfig{}, err
 	}
 
 	if configPath == "" {
@@ -607,18 +603,18 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	}
 
 	if err != nil {
-		return "", config, tracing, fmt.Errorf("Cannot find usable config file (%v)", err)
+		return "", config, fmt.Errorf("Cannot find usable config file (%v)", err)
 	}
 
 	configData, err := ioutil.ReadFile(resolved)
 	if err != nil {
-		return "", config, tracing, err
+		return "", config, err
 	}
 
 	var tomlConf tomlConfig
 	_, err = toml.Decode(string(configData), &tomlConf)
 	if err != nil {
-		return "", config, tracing, err
+		return "", config, err
 	}
 
 	config.Debug = tomlConf.Runtime.Debug
@@ -634,14 +630,14 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	if tomlConf.Runtime.InterNetworkModel != "" {
 		err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel)
 		if err != nil {
-			return "", config, tracing, err
+			return "", config, err
 		}
 	}
 
 	if !ignoreLogging {
 		err := handleSystemLog("", "")
 		if err != nil {
-			return "", config, tracing, err
+			return "", config, err
 		}
 
 		kataUtilsLogger.WithFields(
@@ -651,13 +647,13 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 			}).Info("loaded configuration")
 	}
 
-	if err := updateRuntimeConfig(resolved, tomlConf, &config); err != nil {
-		return "", config, tracing, err
+	if err := updateConfig(resolved, tomlConf, &config, builtIn); err != nil {
+		return "", config, err
 	}
 
 	config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
 	if err := checkNetNsConfig(config); err != nil {
-		return "", config, tracing, err
+		return "", config, err
 	}
 
 	// use no proxy if HypervisorConfig.UseVSock is true
@@ -668,10 +664,26 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	}
 
 	if err := checkHypervisorConfig(config.HypervisorConfig); err != nil {
-		return "", config, tracing, err
+		return "", config, err
 	}
 
-	return resolved, config, tracing, nil
+	return resolved, config, nil
+}
+
+func updateConfig(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error {
+
+	if err := updateRuntimeConfig(configPath, tomlConf, config); err != nil {
+		return err
+	}
+
+	if builtIn {
+		config.ProxyType = vc.KataBuiltInProxyType
+		config.ShimType = vc.KataBuiltInShimType
+		config.AgentType = vc.KataContainersAgent
+		config.AgentConfig = vc.KataAgentConfig{LongLiveConn: true}
+	}
+
+	return nil
 }
 
 // checkNetNsConfig performs sanity checks on disable_new_netns config.

pkg/katautils/config_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
@@ -249,7 +249,7 @@ func testLoadConfiguration(t *testing.T, dir string,
 					assert.NoError(t, err)
 				}
 
-				resolvedConfigPath, config, _, err := LoadConfiguration(file, ignoreLogging, false)
+				resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
 				if expectFail {
 					assert.Error(t, err)
 
@@ -558,7 +558,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, config, _, err := LoadConfiguration(configPath, false, false)
+	_, config, err := LoadConfiguration(configPath, false, false)
 	if err == nil {
 		t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
 	}
@@ -583,7 +583,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		t.Error(err)
 	}
 
-	_, config, _, err = LoadConfiguration(configPath, false, false)
+	_, config, err = LoadConfiguration(configPath, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -712,7 +712,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, config, _, err := LoadConfiguration(configPath, false, false)
+	_, config, err := LoadConfiguration(configPath, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/katautils/create.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package katautils
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vf "github.com/kata-containers/runtime/virtcontainers/factory"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+)
+
+// GetKernelParamsFunc use a variable to allow tests to modify its value
+var GetKernelParamsFunc = getKernelParams
+
+var systemdKernelParam = []vc.Param{
+	{
+		Key:   "init",
+		Value: "/usr/lib/systemd/systemd",
+	},
+	{
+		Key:   "systemd.unit",
+		Value: systemdUnitName,
+	},
+	{
+		Key:   "systemd.mask",
+		Value: "systemd-networkd.service",
+	},
+	{
+		Key:   "systemd.mask",
+		Value: "systemd-networkd.socket",
+	},
+}
+
+func getKernelParams(needSystemd bool) []vc.Param {
+	p := []vc.Param{}
+
+	if needSystemd {
+		p = append(p, systemdKernelParam...)
+	}
+
+	return p
+}
+
+func needSystemd(config vc.HypervisorConfig) bool {
+	return config.ImagePath != ""
+}
+
+// HandleFactory  set the factory
+func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) {
+	if !runtimeConfig.FactoryConfig.Template {
+		return
+	}
+
+	factoryConfig := vf.Config{
+		Template: true,
+		VMConfig: vc.VMConfig{
+			HypervisorType:   runtimeConfig.HypervisorType,
+			HypervisorConfig: runtimeConfig.HypervisorConfig,
+			AgentType:        runtimeConfig.AgentType,
+			AgentConfig:      runtimeConfig.AgentConfig,
+		},
+	}
+
+	kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory")
+
+	f, err := vf.NewFactory(ctx, factoryConfig, true)
+	if err != nil {
+		kataUtilsLogger.WithError(err).Warn("load vm factory failed, about to create new one")
+		f, err = vf.NewFactory(ctx, factoryConfig, false)
+		if err != nil {
+			kataUtilsLogger.WithError(err).Warn("create vm factory failed")
+			return
+		}
+	}
+
+	vci.SetFactory(ctx, f)
+}
+
+// SetKernelParams adds the user-specified kernel parameters (from the
+// configuration file) to the defaults so that the former take priority.
+func SetKernelParams(containerID string, runtimeConfig *oci.RuntimeConfig) error {
+	defaultKernelParams := GetKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig))
+
+	if runtimeConfig.HypervisorConfig.Debug {
+		strParams := vc.SerializeParams(defaultKernelParams, "=")
+		formatted := strings.Join(strParams, " ")
+
+		kataUtilsLogger.WithField("default-kernel-parameters", formatted).Debug()
+	}
+
+	// retrieve the parameters specified in the config file
+	userKernelParams := runtimeConfig.HypervisorConfig.KernelParams
+
+	// reset
+	runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{}
+
+	// first, add default values
+	for _, p := range defaultKernelParams {
+		if err := (runtimeConfig).AddKernelParam(p); err != nil {
+			return err
+		}
+	}
+
+	// now re-add the user-specified values so that they take priority.
+	for _, p := range userKernelParams {
+		if err := (runtimeConfig).AddKernelParam(p); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// SetEphemeralStorageType sets the mount type to 'ephemeral'
+// if the mount source path is provisioned by k8s for ephemeral storage.
+// For the given pod ephemeral volume is created only once
+// backed by tmpfs inside the VM. For successive containers
+// of the same pod the already existing volume is reused.
+func SetEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec {
+	for idx, mnt := range ociSpec.Mounts {
+		if IsEphemeralStorage(mnt.Source) {
+			ociSpec.Mounts[idx].Type = "ephemeral"
+		}
+	}
+	return ociSpec
+}
+
+// CreateSandbox create a sandbox container
+func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig,
+	containerID, bundlePath, console string, disableOutput, systemdCgroup, builtIn bool) (vc.VCSandbox, vc.Process, error) {
+	span, ctx := Trace(ctx, "createSandbox")
+	defer span.Finish()
+
+	err := SetKernelParams(containerID, &runtimeConfig)
+	if err != nil {
+		return nil, vc.Process{}, err
+	}
+
+	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
+	if err != nil {
+		return nil, vc.Process{}, err
+	}
+
+	if builtIn {
+		sandboxConfig.Stateful = true
+	}
+
+	// Important to create the network namespace before the sandbox is
+	// created, because it is not responsible for the creation of the
+	// netns if it does not exist.
+	if err := SetupNetworkNamespace(&sandboxConfig.NetworkConfig); err != nil {
+		return nil, vc.Process{}, err
+	}
+
+	// Run pre-start OCI hooks.
+	err = EnterNetNS(sandboxConfig.NetworkConfig.NetNSPath, func() error {
+		return PreStartHooks(ctx, ociSpec, containerID, bundlePath)
+	})
+	if err != nil {
+		return nil, vc.Process{}, err
+	}
+
+	sandbox, err := vci.CreateSandbox(ctx, sandboxConfig)
+	if err != nil {
+		return nil, vc.Process{}, err
+	}
+
+	sid := sandbox.ID()
+	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
+	span.SetTag("sandbox", sid)
+
+	containers := sandbox.GetAllContainers()
+	if len(containers) != 1 {
+		return nil, vc.Process{}, fmt.Errorf("BUG: Container list from sandbox is wrong, expecting only one container, found %d containers", len(containers))
+	}
+
+	if !builtIn {
+		err = AddContainerIDMapping(ctx, containerID, sandbox.ID())
+		if err != nil {
+			return nil, vc.Process{}, err
+		}
+	}
+
+	return sandbox, containers[0].Process(), nil
+}
+
+// CreateContainer create a container
+func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) {
+	var c vc.VCContainer
+
+	span, ctx := Trace(ctx, "createContainer")
+	defer span.Finish()
+
+	ociSpec = SetEphemeralStorageType(ociSpec)
+
+	contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput)
+	if err != nil {
+		return vc.Process{}, err
+	}
+
+	sandboxID, err := ociSpec.SandboxID()
+	if err != nil {
+		return vc.Process{}, err
+	}
+
+	span.SetTag("sandbox", sandboxID)
+
+	if builtIn {
+		c, err = sandbox.CreateContainer(contConfig)
+		if err != nil {
+			return vc.Process{}, err
+		}
+	} else {
+		kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sandboxID)
+
+		sandbox, c, err = vci.CreateContainer(ctx, sandboxID, contConfig)
+		if err != nil {
+			return vc.Process{}, err
+		}
+
+		if err := AddContainerIDMapping(ctx, containerID, sandboxID); err != nil {
+			return vc.Process{}, err
+		}
+
+		kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sandboxID)
+
+		if err := AddContainerIDMapping(ctx, containerID, sandboxID); err != nil {
+			return vc.Process{}, err
+		}
+	}
+
+	// Run pre-start OCI hooks.
+	err = EnterNetNS(sandbox.GetNetNs(), func() error {
+		return PreStartHooks(ctx, ociSpec, containerID, bundlePath)
+	})
+	if err != nil {
+		return vc.Process{}, err
+	}
+
+	return c.Process(), nil
+}

pkg/katautils/create_test.go

@@ -0,0 +1,455 @@
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package katautils
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testConsole                 = "/dev/pts/999"
+	testContainerTypeAnnotation = "io.kubernetes.cri-o.ContainerType"
+	testSandboxIDAnnotation     = "io.kubernetes.cri-o.SandboxID"
+	testContainerTypeContainer  = "container"
+)
+
+var (
+	testBundleDir = ""
+
+	// testingImpl is a concrete mock RVC implementation used for testing
+	testingImpl = &vcmock.VCMock{}
+)
+
+// readOCIConfig returns an OCI spec.
+func readOCIConfigFile(configPath string) (oci.CompatOCISpec, error) {
+	if configPath == "" {
+		return oci.CompatOCISpec{}, errors.New("BUG: need config file path")
+	}
+
+	data, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return oci.CompatOCISpec{}, err
+	}
+
+	var ociSpec oci.CompatOCISpec
+	if err := json.Unmarshal(data, &ociSpec); err != nil {
+		return oci.CompatOCISpec{}, err
+	}
+	caps, err := oci.ContainerCapabilities(ociSpec)
+	if err != nil {
+		return oci.CompatOCISpec{}, err
+	}
+	ociSpec.Process.Capabilities = caps
+	return ociSpec, nil
+}
+
+func writeOCIConfigFile(spec oci.CompatOCISpec, configPath string) error {
+	if configPath == "" {
+		return errors.New("BUG: need config file path")
+	}
+
+	bytes, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(configPath, bytes, testFileMode)
+}
+
+// Create an OCI bundle in the specified directory.
+//
+// Note that the directory will be created, but it's parent is expected to exist.
+//
+// This function works by copying the already-created test bundle. Ideally,
+// the bundle would be recreated for each test, but createRootfs() uses
+// docker which on some systems is too slow, resulting in the tests timing
+// out.
+func makeOCIBundle(bundleDir string) error {
+	from := testBundleDir
+	to := bundleDir
+
+	// only the basename of bundleDir needs to exist as bundleDir
+	// will get created by cp(1).
+	base := filepath.Dir(bundleDir)
+
+	for _, dir := range []string{from, base} {
+		if !FileExists(dir) {
+			return fmt.Errorf("BUG: directory %v should exist", dir)
+		}
+	}
+
+	output, err := RunCommandFull([]string{"cp", "-a", from, to}, true)
+	if err != nil {
+		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
+	}
+
+	return nil
+}
+
+// newTestRuntimeConfig creates a new RuntimeConfig
+func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
+	if dir == "" {
+		return oci.RuntimeConfig{}, errors.New("BUG: need directory")
+	}
+
+	hypervisorConfig, err := newTestHypervisorConfig(dir, create)
+	if err != nil {
+		return oci.RuntimeConfig{}, err
+	}
+
+	return oci.RuntimeConfig{
+		HypervisorType:   vc.QemuHypervisor,
+		HypervisorConfig: hypervisorConfig,
+		AgentType:        vc.KataContainersAgent,
+		ProxyType:        vc.CCProxyType,
+		ShimType:         vc.CCShimType,
+		Console:          consolePath,
+	}, nil
+}
+
+// newTestHypervisorConfig creaets a new virtcontainers
+// HypervisorConfig, ensuring that the required resources are also
+// created.
+//
+// Note: no parameter validation in case caller wishes to create an invalid
+// object.
+func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) {
+	kernelPath := path.Join(dir, "kernel")
+	imagePath := path.Join(dir, "image")
+	hypervisorPath := path.Join(dir, "hypervisor")
+
+	if create {
+		for _, file := range []string{kernelPath, imagePath, hypervisorPath} {
+			err := createEmptyFile(file)
+			if err != nil {
+				return vc.HypervisorConfig{}, err
+			}
+		}
+	}
+
+	return vc.HypervisorConfig{
+		KernelPath:            kernelPath,
+		ImagePath:             imagePath,
+		HypervisorPath:        hypervisorPath,
+		HypervisorMachineType: "pc-lite",
+	}, nil
+}
+
+// return the value of the *last* param with the specified key
+func findLastParam(key string, params []vc.Param) (string, error) {
+	if key == "" {
+		return "", errors.New("ERROR: need non-nil key")
+	}
+
+	l := len(params)
+	if l == 0 {
+		return "", errors.New("ERROR: no params")
+	}
+
+	for i := l - 1; i >= 0; i-- {
+		p := params[i]
+
+		if key == p.Key {
+			return p.Value, nil
+		}
+	}
+
+	return "", fmt.Errorf("no param called %q found", name)
+}
+
+func TestSetEphemeralStorageType(t *testing.T) {
+	assert := assert.New(t)
+
+	ociSpec := oci.CompatOCISpec{}
+	var ociMounts []specs.Mount
+	mount := specs.Mount{
+		Source: "/var/lib/kubelet/pods/366c3a77-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume",
+	}
+
+	ociMounts = append(ociMounts, mount)
+	ociSpec.Mounts = ociMounts
+	ociSpec = SetEphemeralStorageType(ociSpec)
+
+	mountType := ociSpec.Mounts[0].Type
+	assert.Equal(mountType, "ephemeral",
+		"Unexpected mount type, got %s expected ephemeral", mountType)
+}
+
+func TestSetKernelParams(t *testing.T) {
+	assert := assert.New(t)
+
+	config := oci.RuntimeConfig{}
+
+	assert.Empty(config.HypervisorConfig.KernelParams)
+
+	err := SetKernelParams(testContainerID, &config)
+	assert.NoError(err)
+
+	if needSystemd(config.HypervisorConfig) {
+		assert.NotEmpty(config.HypervisorConfig.KernelParams)
+	}
+}
+
+func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) {
+	assert := assert.New(t)
+
+	initName := "init"
+	initValue := "/sbin/myinit"
+
+	ipName := "ip"
+	ipValue := "127.0.0.1"
+
+	params := []vc.Param{
+		{Key: initName, Value: initValue},
+		{Key: ipName, Value: ipValue},
+	}
+
+	hypervisorConfig := vc.HypervisorConfig{
+		KernelParams: params,
+	}
+
+	// Config containing user-specified kernel parameters
+	config := oci.RuntimeConfig{
+		HypervisorConfig: hypervisorConfig,
+	}
+
+	assert.NotEmpty(config.HypervisorConfig.KernelParams)
+
+	err := SetKernelParams(testContainerID, &config)
+	assert.NoError(err)
+
+	kernelParams := config.HypervisorConfig.KernelParams
+
+	init, err := findLastParam(initName, kernelParams)
+	assert.NoError(err)
+	assert.Equal(initValue, init)
+
+	ip, err := findLastParam(ipName, kernelParams)
+	assert.NoError(err)
+	assert.Equal(ipValue, ip)
+
+}
+
+func TestCreateSandboxConfigFail(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	quota := int64(0)
+	limit := int64(0)
+
+	spec.Linux.Resources.Memory = &specs.LinuxMemory{
+		Limit: &limit,
+	}
+
+	spec.Linux.Resources.CPU = &specs.LinuxCPU{
+		// specify an invalid value
+		Quota: &quota,
+	}
+
+	_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false)
+	assert.Error(err)
+}
+
+func TestCreateSandboxFail(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip(testDisabledNeedNonRoot)
+	}
+
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
+	assert.NoError(err)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false)
+	assert.Error(err)
+	assert.True(vcmock.IsMockError(err))
+}
+
+func TestCreateContainerContainerConfigFail(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	// Set invalid container type
+	containerType := "你好，世界"
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = containerType
+
+	// rewrite file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for _, disableOutput := range []bool{true, false} {
+		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
+		assert.Error(err)
+		assert.False(vcmock.IsMockError(err))
+		assert.True(strings.Contains(err.Error(), containerType))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateContainerFail(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	// set expected container type and sandboxID
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
+	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
+
+	// rewrite file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for _, disableOutput := range []bool{true, false} {
+		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
+		assert.Error(err)
+		assert.True(vcmock.IsMockError(err))
+		os.RemoveAll(path)
+	}
+}
+
+func TestCreateContainer(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) {
+		return &vcmock.Sandbox{}, &vcmock.Container{}, nil
+	}
+
+	defer func() {
+		testingImpl.CreateContainerFunc = nil
+	}()
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	bundlePath := filepath.Join(tmpdir, "bundle")
+
+	err = makeOCIBundle(bundlePath)
+	assert.NoError(err)
+
+	ociConfigFile := filepath.Join(bundlePath, "config.json")
+	assert.True(FileExists(ociConfigFile))
+
+	spec, err := readOCIConfigFile(ociConfigFile)
+	assert.NoError(err)
+
+	// set expected container type and sandboxID
+	spec.Annotations = make(map[string]string)
+	spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
+	spec.Annotations[testSandboxIDAnnotation] = testSandboxID
+
+	// rewrite file
+	err = writeOCIConfigFile(spec, ociConfigFile)
+	assert.NoError(err)
+
+	for _, disableOutput := range []bool{true, false} {
+		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
+		assert.NoError(err)
+		os.RemoveAll(path)
+	}
+}

pkg/katautils/hook.go

@@ -1,9 +1,10 @@
 // Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 
-package main
+package katautils
 
 import (
 	"bytes"
@@ -24,11 +25,11 @@ import (
 
 // Logger returns a logrus logger appropriate for logging hook messages
 func hookLogger() *logrus.Entry {
-	return kataLog.WithField("subsystem", "hook")
+	return kataUtilsLogger.WithField("subsystem", "hook")
 }
 
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
-	span, _ := trace(ctx, "hook")
+	span, _ := Trace(ctx, "hook")
 	defer span.Finish()
 
 	span.SetTag("subsystem", "runHook")
@@ -91,7 +92,7 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 }
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
-	span, _ := trace(ctx, "hooks")
+	span, _ := Trace(ctx, "hooks")
 	defer span.Finish()
 
 	span.SetTag("subsystem", hookType)
@@ -110,7 +111,8 @@ func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType
 	return nil
 }
 
-func preStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
+// PreStartHooks run the hooks before start container
+func PreStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
 	// If no hook available, nothing needs to be done.
 	if spec.Hooks == nil {
 		return nil
@@ -119,7 +121,8 @@ func preStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath
 	return runHooks(ctx, spec.Hooks.Prestart, cid, bundlePath, "pre-start")
 }
 
-func postStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
+// PostStartHooks run the hooks just after start container
+func PostStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
 	// If no hook available, nothing needs to be done.
 	if spec.Hooks == nil {
 		return nil
@@ -128,7 +131,8 @@ func postStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath
 	return runHooks(ctx, spec.Hooks.Poststart, cid, bundlePath, "post-start")
 }
 
-func postStopHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
+// PostStopHooks run the hooks after stop container
+func PostStopHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
 	// If no hook available, nothing needs to be done.
 	if spec.Hooks == nil {
 		return nil

pkg/katautils/hook_test.go

@@ -1,9 +1,10 @@
 // Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 
-package main
+package katautils
 
 import (
 	"context"
@@ -95,7 +96,7 @@ func TestPreStartHooks(t *testing.T) {
 
 	// Hooks field is nil
 	spec := oci.CompatOCISpec{}
-	err := preStartHooks(ctx, spec, "", "")
+	err := PreStartHooks(ctx, spec, "", "")
 	assert.NoError(err)
 
 	// Hooks list is empty
@@ -104,7 +105,7 @@ func TestPreStartHooks(t *testing.T) {
 			Hooks: &specs.Hooks{},
 		},
 	}
-	err = preStartHooks(ctx, spec, "", "")
+	err = PreStartHooks(ctx, spec, "", "")
 	assert.NoError(err)
 
 	// Run with timeout 0
@@ -116,7 +117,7 @@ func TestPreStartHooks(t *testing.T) {
 			},
 		},
 	}
-	err = preStartHooks(ctx, spec, testSandboxID, testBundlePath)
+	err = PreStartHooks(ctx, spec, testSandboxID, testBundlePath)
 	assert.NoError(err)
 
 	// Failure due to wrong hook
@@ -128,7 +129,7 @@ func TestPreStartHooks(t *testing.T) {
 			},
 		},
 	}
-	err = preStartHooks(ctx, spec, testSandboxID, testBundlePath)
+	err = PreStartHooks(ctx, spec, testSandboxID, testBundlePath)
 	assert.Error(err)
 }
 
@@ -143,7 +144,7 @@ func TestPostStartHooks(t *testing.T) {
 
 	// Hooks field is nil
 	spec := oci.CompatOCISpec{}
-	err := postStartHooks(ctx, spec, "", "")
+	err := PostStartHooks(ctx, spec, "", "")
 	assert.NoError(err)
 
 	// Hooks list is empty
@@ -152,7 +153,7 @@ func TestPostStartHooks(t *testing.T) {
 			Hooks: &specs.Hooks{},
 		},
 	}
-	err = postStartHooks(ctx, spec, "", "")
+	err = PostStartHooks(ctx, spec, "", "")
 	assert.NoError(err)
 
 	// Run with timeout 0
@@ -164,7 +165,7 @@ func TestPostStartHooks(t *testing.T) {
 			},
 		},
 	}
-	err = postStartHooks(ctx, spec, testSandboxID, testBundlePath)
+	err = PostStartHooks(ctx, spec, testSandboxID, testBundlePath)
 	assert.NoError(err)
 
 	// Failure due to wrong hook
@@ -176,7 +177,7 @@ func TestPostStartHooks(t *testing.T) {
 			},
 		},
 	}
-	err = postStartHooks(ctx, spec, testSandboxID, testBundlePath)
+	err = PostStartHooks(ctx, spec, testSandboxID, testBundlePath)
 	assert.Error(err)
 }
 
@@ -191,7 +192,7 @@ func TestPostStopHooks(t *testing.T) {
 
 	// Hooks field is nil
 	spec := oci.CompatOCISpec{}
-	err := postStopHooks(ctx, spec, "", "")
+	err := PostStopHooks(ctx, spec, "", "")
 	assert.NoError(err)
 
 	// Hooks list is empty
@@ -200,7 +201,7 @@ func TestPostStopHooks(t *testing.T) {
 			Hooks: &specs.Hooks{},
 		},
 	}
-	err = postStopHooks(ctx, spec, "", "")
+	err = PostStopHooks(ctx, spec, "", "")
 	assert.NoError(err)
 
 	// Run with timeout 0
@@ -212,7 +213,7 @@ func TestPostStopHooks(t *testing.T) {
 			},
 		},
 	}
-	err = postStopHooks(ctx, spec, testSandboxID, testBundlePath)
+	err = PostStopHooks(ctx, spec, testSandboxID, testBundlePath)
 	assert.NoError(err)
 
 	// Failure due to wrong hook
@@ -224,6 +225,6 @@ func TestPostStopHooks(t *testing.T) {
 			},
 		},
 	}
-	err = postStopHooks(ctx, spec, testSandboxID, testBundlePath)
+	err = PostStopHooks(ctx, spec, testSandboxID, testBundlePath)
 	assert.Error(err)
 }

pkg/katautils/logger.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0

pkg/katautils/logger_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0

pkg/katautils/network.go

@@ -0,0 +1,181 @@
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package katautils
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	goruntime "runtime"
+	"strings"
+
+	"github.com/containernetworking/plugins/pkg/ns"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"golang.org/x/sys/unix"
+)
+
+const procMountInfoFile = "/proc/self/mountinfo"
+
+// EnterNetNS is free from any call to a go routine, and it calls
+// into runtime.LockOSThread(), meaning it won't be executed in a
+// different thread than the one expected by the caller.
+func EnterNetNS(netNSPath string, cb func() error) error {
+	if netNSPath == "" {
+		return cb()
+	}
+
+	goruntime.LockOSThread()
+	defer goruntime.UnlockOSThread()
+
+	currentNS, err := ns.GetCurrentNS()
+	if err != nil {
+		return err
+	}
+	defer currentNS.Close()
+
+	targetNS, err := ns.GetNS(netNSPath)
+	if err != nil {
+		return err
+	}
+
+	if err := targetNS.Set(); err != nil {
+		return err
+	}
+	defer currentNS.Set()
+
+	return cb()
+}
+
+// SetupNetworkNamespace create a network namespace
+func SetupNetworkNamespace(config *vc.NetworkConfig) error {
+	if config.DisableNewNetNs {
+		kataUtilsLogger.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns")
+		return nil
+	}
+
+	if config.NetNSPath == "" {
+		n, err := ns.NewNS()
+		if err != nil {
+			return err
+		}
+
+		config.NetNSPath = n.Path()
+		config.NetNsCreated = true
+
+		return nil
+	}
+
+	isHostNs, err := hostNetworkingRequested(config.NetNSPath)
+	if err != nil {
+		return err
+	}
+	if isHostNs {
+		return fmt.Errorf("Host networking requested, not supported by runtime")
+	}
+
+	return nil
+}
+
+// getNetNsFromBindMount returns the network namespace for the bind-mounted path
+func getNetNsFromBindMount(nsPath string, procMountFile string) (string, error) {
+	netNsMountType := "nsfs"
+
+	// Resolve all symlinks in the path as the mountinfo file contains
+	// resolved paths.
+	nsPath, err := filepath.EvalSymlinks(nsPath)
+	if err != nil {
+		return "", err
+	}
+
+	f, err := os.Open(procMountFile)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		text := scanner.Text()
+
+		// Scan the mountinfo file to search for the network namespace path
+		// This file contains mounts in the eg format:
+		// "711 26 0:3 net:[4026532009] /run/docker/netns/default rw shared:535 - nsfs nsfs rw"
+		//
+		// Reference: https://www.kernel.org/doc/Documentation/filesystems/proc.txt
+
+		// We are interested in the first 9 fields of this file,
+		// to check for the correct mount type.
+		fields := strings.Split(text, " ")
+		if len(fields) < 9 {
+			continue
+		}
+
+		// We check here if the mount type is a network namespace mount type, namely "nsfs"
+		mountTypeFieldIdx := 8
+		if fields[mountTypeFieldIdx] != netNsMountType {
+			continue
+		}
+
+		// This is the mount point/destination for the mount
+		mntDestIdx := 4
+		if fields[mntDestIdx] != nsPath {
+			continue
+		}
+
+		// This is the root/source of the mount
+		return fields[3], nil
+	}
+
+	return "", nil
+}
+
+// hostNetworkingRequested checks if the network namespace requested is the
+// same as the current process.
+func hostNetworkingRequested(configNetNs string) (bool, error) {
+	var evalNS, nsPath, currentNsPath string
+	var err error
+
+	// Net namespace provided as "/proc/pid/ns/net" or "/proc/<pid>/task/<tid>/ns/net"
+	if strings.HasPrefix(configNetNs, "/proc") && strings.HasSuffix(configNetNs, "/ns/net") {
+		if _, err := os.Stat(configNetNs); err != nil {
+			return false, err
+		}
+
+		// Here we are trying to resolve the path but it fails because
+		// namespaces links don't really exist. For this reason, the
+		// call to EvalSymlinks will fail when it will try to stat the
+		// resolved path found. As we only care about the path, we can
+		// retrieve it from the PathError structure.
+		if _, err = filepath.EvalSymlinks(configNetNs); err != nil {
+			nsPath = err.(*os.PathError).Path
+		} else {
+			return false, fmt.Errorf("Net namespace path %s is not a symlink", configNetNs)
+		}
+
+		_, evalNS = filepath.Split(nsPath)
+
+	} else {
+		// Bind-mounted path provided
+		evalNS, _ = getNetNsFromBindMount(configNetNs, procMountInfoFile)
+	}
+
+	currentNS := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
+	if _, err = filepath.EvalSymlinks(currentNS); err != nil {
+		currentNsPath = err.(*os.PathError).Path
+	} else {
+		return false, fmt.Errorf("Unexpected: Current network namespace path is not a symlink")
+	}
+
+	_, evalCurrentNS := filepath.Split(currentNsPath)
+
+	if evalNS == evalCurrentNS {
+		return true, nil
+	}
+
+	return false, nil
+}

pkg/katautils/network_test.go

@@ -0,0 +1,150 @@
+// Copyright (c) 2018 Huawei Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package katautils
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	"github.com/containernetworking/plugins/pkg/ns"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/sys/unix"
+)
+
+func TestGetNetNsFromBindMount(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	mountFile := filepath.Join(tmpdir, "mountInfo")
+	nsPath := filepath.Join(tmpdir, "ns123")
+
+	// Non-existent namespace path
+	_, err = getNetNsFromBindMount(nsPath, mountFile)
+	assert.NotNil(err)
+
+	tmpNSPath := filepath.Join(tmpdir, "testNetNs")
+	f, err := os.Create(tmpNSPath)
+	assert.NoError(err)
+	defer f.Close()
+
+	type testData struct {
+		contents       string
+		expectedResult string
+	}
+
+	data := []testData{
+		{fmt.Sprintf("711 26 0:3 net:[4026532008] %s rw shared:535 - nsfs nsfs rw", tmpNSPath), "net:[4026532008]"},
+		{"711 26 0:3 net:[4026532008] /run/netns/ns123 rw shared:535 - tmpfs tmpfs rw", ""},
+		{"a a a a a a a - b c d", ""},
+		{"", ""},
+	}
+
+	for i, d := range data {
+		err := ioutil.WriteFile(mountFile, []byte(d.contents), 0640)
+		assert.NoError(err)
+
+		path, err := getNetNsFromBindMount(tmpNSPath, mountFile)
+		assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", path, d))
+
+		assert.Equal(d.expectedResult, path, "Test %d, expected %s, got %s", i, d.expectedResult, path)
+	}
+}
+
+func TestHostNetworkingRequested(t *testing.T) {
+	assert := assert.New(t)
+
+	if os.Geteuid() != 0 {
+		t.Skip(testDisabledNeedRoot)
+	}
+
+	// Network namespace same as the host
+	selfNsPath := "/proc/self/ns/net"
+	isHostNs, err := hostNetworkingRequested(selfNsPath)
+	assert.NoError(err)
+	assert.True(isHostNs)
+
+	// Non-existent netns path
+	nsPath := "/proc/123456789/ns/net"
+	_, err = hostNetworkingRequested(nsPath)
+	assert.Error(err)
+
+	// Bind-mounted Netns
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	// Create a bind mount to the current network namespace.
+	tmpFile := filepath.Join(tmpdir, "testNetNs")
+	f, err := os.Create(tmpFile)
+	assert.NoError(err)
+	defer f.Close()
+
+	err = syscall.Mount(selfNsPath, tmpFile, "bind", syscall.MS_BIND, "")
+	assert.Nil(err)
+
+	isHostNs, err = hostNetworkingRequested(tmpFile)
+	assert.NoError(err)
+	assert.True(isHostNs)
+
+	syscall.Unmount(tmpFile, 0)
+}
+
+func TestSetupNetworkNamespace(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip(testDisabledNeedNonRoot)
+	}
+
+	assert := assert.New(t)
+
+	// Network namespace same as the host
+	config := &vc.NetworkConfig{
+		NetNSPath: "/proc/self/ns/net",
+	}
+	err := SetupNetworkNamespace(config)
+	assert.Error(err)
+
+	// Non-existent netns path
+	config = &vc.NetworkConfig{
+		NetNSPath: "/proc/123456789/ns/net",
+	}
+	err = SetupNetworkNamespace(config)
+	assert.Error(err)
+
+	// Existent netns path
+	n, err := ns.NewNS()
+	assert.NoError(err)
+	config = &vc.NetworkConfig{
+		NetNSPath: n.Path(),
+	}
+	err = SetupNetworkNamespace(config)
+	assert.NoError(err)
+	n.Close()
+
+	// Empty netns path
+	config = &vc.NetworkConfig{}
+	err = SetupNetworkNamespace(config)
+	assert.NoError(err)
+	n, err = ns.GetNS(config.NetNSPath)
+	assert.NoError(err)
+	assert.NotNil(n)
+	assert.True(config.NetNsCreated)
+	n.Close()
+	unix.Unmount(config.NetNSPath, unix.MNT_DETACH)
+	os.RemoveAll(config.NetNSPath)
+
+	// Config with DisableNewNetNs
+	config = &vc.NetworkConfig{DisableNewNetNs: true}
+	err = SetupNetworkNamespace(config)
+	assert.NoError(err)
+}

pkg/katautils/oci.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package katautils
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const ctrsMappingDirMode = os.FileMode(0750)
+
+var ctrsMapTreePath = "/var/run/kata-containers/containers-mapping"
+
+// SetCtrsMapTreePath let the testcases change the ctrsMapTreePath to a test dir
+func SetCtrsMapTreePath(path string) {
+	ctrsMapTreePath = path
+}
+
+// FetchContainerIDMapping This function assumes it should find only one file inside the container
+// ID directory. If there are several files, we could not determine which
+// file name corresponds to the sandbox ID associated, and this would throw
+// an error.
+func FetchContainerIDMapping(containerID string) (string, error) {
+	if containerID == "" {
+		return "", fmt.Errorf("Missing container ID")
+	}
+
+	dirPath := filepath.Join(ctrsMapTreePath, containerID)
+
+	files, err := ioutil.ReadDir(dirPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+
+		return "", err
+	}
+
+	if len(files) != 1 {
+		return "", fmt.Errorf("Too many files (%d) in %q", len(files), dirPath)
+	}
+
+	return files[0].Name(), nil
+}
+
+// AddContainerIDMapping add a container id mapping to sandbox id
+func AddContainerIDMapping(ctx context.Context, containerID, sandboxID string) error {
+	span, _ := Trace(ctx, "addContainerIDMapping")
+	defer span.Finish()
+
+	if containerID == "" {
+		return fmt.Errorf("Missing container ID")
+	}
+
+	if sandboxID == "" {
+		return fmt.Errorf("Missing sandbox ID")
+	}
+
+	parentPath := filepath.Join(ctrsMapTreePath, containerID)
+
+	if err := os.RemoveAll(parentPath); err != nil {
+		return err
+	}
+
+	path := filepath.Join(parentPath, sandboxID)
+
+	if err := os.MkdirAll(path, ctrsMappingDirMode); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// DelContainerIDMapping delete container id mapping from a sandbox
+func DelContainerIDMapping(ctx context.Context, containerID string) error {
+	span, _ := Trace(ctx, "delContainerIDMapping")
+	defer span.Finish()
+
+	if containerID == "" {
+		return fmt.Errorf("Missing container ID")
+	}
+
+	path := filepath.Join(ctrsMapTreePath, containerID)
+
+	return os.RemoveAll(path)
+}

pkg/katautils/oci_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2018 Intel Corporation
+// Copyright (c) 2018 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package katautils
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createTempContainerIDMapping(containerID, sandboxID string) (string, error) {
+	tmpDir, err := ioutil.TempDir("", "containers-mapping")
+	if err != nil {
+		return "", err
+	}
+	ctrsMapTreePath = tmpDir
+
+	path := filepath.Join(ctrsMapTreePath, containerID, sandboxID)
+	if err := os.MkdirAll(path, 0750); err != nil {
+		return "", err
+	}
+
+	return tmpDir, nil
+}
+
+func TestFetchContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	sandboxID, err := FetchContainerIDMapping("")
+	assert.Error(err)
+	assert.Empty(sandboxID)
+}
+
+func TestFetchContainerIDMappingEmptyMappingSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	sandboxID, err := FetchContainerIDMapping(testContainerID)
+	assert.NoError(err)
+	assert.Empty(sandboxID)
+}
+
+func TestFetchContainerIDMappingTooManyFilesFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	err = os.MkdirAll(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID+"2"), ctrsMappingDirMode)
+	assert.NoError(err)
+
+	sandboxID, err := FetchContainerIDMapping(testContainerID)
+	assert.Error(err)
+	assert.Empty(sandboxID)
+}
+
+func TestFetchContainerIDMappingSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	sandboxID, err := FetchContainerIDMapping(testContainerID)
+	assert.NoError(err)
+	assert.Equal(sandboxID, testSandboxID)
+}
+
+func TestAddContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	err := AddContainerIDMapping(context.Background(), "", testSandboxID)
+	assert.Error(err)
+}
+
+func TestAddContainerIDMappingSandboxIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	err := AddContainerIDMapping(context.Background(), testContainerID, "")
+	assert.Error(err)
+}
+
+func TestAddContainerIDMappingSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := ioutil.TempDir("", "containers-mapping")
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+	ctrsMapTreePath = path
+
+	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
+	assert.True(os.IsNotExist(err))
+
+	err = AddContainerIDMapping(context.Background(), testContainerID, testSandboxID)
+	assert.NoError(err)
+
+	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
+	assert.NoError(err)
+}
+
+func TestDelContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
+	assert := assert.New(t)
+
+	err := DelContainerIDMapping(context.Background(), "")
+	assert.Error(err)
+}
+
+func TestDelContainerIDMappingSuccess(t *testing.T) {
+	assert := assert.New(t)
+
+	path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
+	assert.NoError(err)
+	defer os.RemoveAll(path)
+
+	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
+	assert.NoError(err)
+
+	err = DelContainerIDMapping(context.Background(), testContainerID)
+	assert.NoError(err)
+
+	_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
+	assert.True(os.IsNotExist(err))
+}

pkg/katautils/tracing.go

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-package main
+package katautils
 
 import (
 	"context"
@@ -22,14 +22,15 @@ type traceLogger struct {
 var tracerCloser io.Closer
 
 func (t traceLogger) Error(msg string) {
-	kataLog.Error(msg)
+	kataUtilsLogger.Error(msg)
 }
 
 func (t traceLogger) Infof(msg string, args ...interface{}) {
-	kataLog.Infof(msg, args...)
+	kataUtilsLogger.Infof(msg, args...)
 }
 
-func createTracer(name string) (opentracing.Tracer, error) {
+// CreateTracer create a tracer
+func CreateTracer(name string) (opentracing.Tracer, error) {
 	cfg := &config.Configuration{
 		ServiceName: name,
 
@@ -61,8 +62,8 @@ func createTracer(name string) (opentracing.Tracer, error) {
 	return tracer, nil
 }
 
-// stopTracing() ends all tracing, reporting the spans to the collector.
-func stopTracing(ctx context.Context) {
+// StopTracing ends all tracing, reporting the spans to the collector.
+func StopTracing(ctx context.Context) {
 	if !tracing {
 		return
 	}
@@ -74,12 +75,14 @@ func stopTracing(ctx context.Context) {
 	}
 
 	// report all possible spans to the collector
-	tracerCloser.Close()
+	if tracerCloser != nil {
+		tracerCloser.Close()
+	}
 }
 
-// trace creates a new tracing span based on the specified name and parent
+// Trace creates a new tracing span based on the specified name and parent
 // context.
-func trace(parent context.Context, name string) (opentracing.Span, context.Context) {
+func Trace(parent context.Context, name string) (opentracing.Span, context.Context) {
 	span, ctx := opentracing.StartSpanFromContext(parent, name)
 
 	span.SetTag("source", "runtime")

pkg/katautils/utils.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
@@ -10,10 +10,45 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
+const (
+	k8sEmptyDir = "kubernetes.io~empty-dir"
+)
+
+// FileExists test is a file exiting or not
+func FileExists(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+
+	return true
+}
+
+// IsEphemeralStorage returns true if the given path
+// to the storage belongs to kubernetes ephemeral storage
+//
+// This method depends on a specific path used by k8s
+// to detect if it's of type ephemeral. As of now,
+// this is a very k8s specific solution that works
+// but in future there should be a better way for this
+// method to determine if the path is for ephemeral
+// volume type
+func IsEphemeralStorage(path string) bool {
+	splitSourceSlice := strings.Split(path, "/")
+	if len(splitSourceSlice) > 1 {
+		storageType := splitSourceSlice[len(splitSourceSlice)-2]
+		if storageType == k8sEmptyDir {
+			return true
+		}
+	}
+	return false
+}
+
 // ResolvePath returns the fully resolved and expanded value of the
 // specified path.
 func ResolvePath(path string) (string, error) {
@@ -75,3 +110,27 @@ func GetFileContents(file string) (string, error) {
 
 	return string(bytes), nil
 }
+
+// RunCommandFull returns the commands space-trimmed standard output and
+// error on success. Note that if the command fails, the requested output will
+// still be returned, along with an error.
+func RunCommandFull(args []string, includeStderr bool) (string, error) {
+	cmd := exec.Command(args[0], args[1:]...)
+	var err error
+	var bytes []byte
+
+	if includeStderr {
+		bytes, err = cmd.CombinedOutput()
+	} else {
+		bytes, err = cmd.Output()
+	}
+
+	trimmed := strings.TrimSpace(string(bytes))
+
+	return trimmed, err
+}
+
+// RunCommand returns the commands space-trimmed standard output on success
+func RunCommand(args []string) (string, error) {
+	return RunCommandFull(args, false)
+}

pkg/katautils/utils_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 // Copyright (c) 2018 HyperHQ Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
@@ -7,14 +7,18 @@
 package katautils
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -22,7 +26,16 @@ const (
 	testDirMode  = os.FileMode(0750)
 	testFileMode = os.FileMode(0640)
 
+	testDisabledNeedRoot    = "Test disabled as requires root user"
 	testDisabledNeedNonRoot = "Test disabled as requires non-root user"
+
+	// small docker image used to create root filesystems from
+	testDockerImage = "busybox"
+
+	testSandboxID   = "99999999-9999-9999-99999999999999999"
+	testContainerID = "1"
+	testBundle      = "bundle"
+	specConfig      = "config.json"
 )
 
 var testDir = ""
@@ -37,6 +50,148 @@ func init() {
 	}
 
 	fmt.Printf("INFO: test directory is %v\n", testDir)
+
+	testBundleDir = filepath.Join(testDir, testBundle)
+	err = os.MkdirAll(testBundleDir, testDirMode)
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
+	}
+
+	fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
+	err = realMakeOCIBundle(testBundleDir)
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
+	}
+}
+
+// createOCIConfig creates an OCI configuration (spec) file in
+// the bundle directory specified (which must exist).
+func createOCIConfig(bundleDir string) error {
+	if bundleDir == "" {
+		return errors.New("BUG: Need bundle directory")
+	}
+
+	if !FileExists(bundleDir) {
+		return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
+	}
+
+	var configCmd string
+
+	// Search for a suitable version of runc to use to generate
+	// the OCI config file.
+	for _, cmd := range []string{"docker-runc", "runc"} {
+		fullPath, err := exec.LookPath(cmd)
+		if err == nil {
+			configCmd = fullPath
+			break
+		}
+	}
+
+	if configCmd == "" {
+		return errors.New("Cannot find command to generate OCI config file")
+	}
+
+	_, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	if err != nil {
+		return err
+	}
+
+	specFile := filepath.Join(bundleDir, specConfig)
+	if !FileExists(specFile) {
+		return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
+	}
+
+	return nil
+}
+
+// realMakeOCIBundle will create an OCI bundle (including the "config.json"
+// config file) in the directory specified (which must already exist).
+//
+// XXX: Note that tests should *NOT* call this function - they should
+// XXX: instead call makeOCIBundle().
+func realMakeOCIBundle(bundleDir string) error {
+	if bundleDir == "" {
+		return errors.New("BUG: Need bundle directory")
+	}
+
+	if !FileExists(bundleDir) {
+		return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
+	}
+
+	err := createOCIConfig(bundleDir)
+	if err != nil {
+		return err
+	}
+
+	// Note the unusual parameter (a directory, not the config
+	// file to parse!)
+	spec, err := oci.ParseConfigJSON(bundleDir)
+	if err != nil {
+		return err
+	}
+
+	// Determine the rootfs directory name the OCI config refers to
+	ociRootPath := spec.Root.Path
+
+	rootfsDir := filepath.Join(bundleDir, ociRootPath)
+
+	if strings.HasPrefix(ociRootPath, "/") {
+		return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
+	}
+
+	err = createRootfs(rootfsDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// createRootfs creates a minimal root filesystem below the specified
+// directory.
+func createRootfs(dir string) error {
+	err := os.MkdirAll(dir, testDirMode)
+	if err != nil {
+		return err
+	}
+
+	container, err := RunCommand([]string{"docker", "create", testDockerImage})
+	if err != nil {
+		return err
+	}
+
+	cmd1 := exec.Command("docker", "export", container)
+	cmd2 := exec.Command("tar", "-C", dir, "-xvf", "-")
+
+	cmd1Stdout, err := cmd1.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	cmd2.Stdin = cmd1Stdout
+
+	err = cmd2.Start()
+	if err != nil {
+		return err
+	}
+
+	err = cmd1.Run()
+	if err != nil {
+		return err
+	}
+
+	err = cmd2.Wait()
+	if err != nil {
+		return err
+	}
+
+	// Clean up
+	_, err = RunCommand([]string{"docker", "rm", container})
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func createFile(file, contents string) error {
@@ -211,3 +366,17 @@ func TestGetFileContents(t *testing.T) {
 		assert.Equal(t, contents, d.contents)
 	}
 }
+
+func TestIsEphemeralStorage(t *testing.T) {
+	sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume"
+	isEphe := IsEphemeralStorage(sampleEphePath)
+	if !isEphe {
+		t.Fatalf("Unable to correctly determine volume type")
+	}
+
+	sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume"
+	isEphe = IsEphemeralStorage(sampleEphePath)
+	if isEphe {
+		t.Fatalf("Unable to correctly determine volume type")
+	}
+}

vendor/github.com/Microsoft/go-winio/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/github.com/Microsoft/go-winio/archive/tar/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/Microsoft/go-winio/backup.go

@@ -0,0 +1,280 @@
+// +build windows
+
+package winio
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"syscall"
+	"unicode/utf16"
+)
+
+//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
+//sys backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite
+
+const (
+	BackupData = uint32(iota + 1)
+	BackupEaData
+	BackupSecurity
+	BackupAlternateData
+	BackupLink
+	BackupPropertyData
+	BackupObjectId
+	BackupReparseData
+	BackupSparseBlock
+	BackupTxfsData
+)
+
+const (
+	StreamSparseAttributes = uint32(8)
+)
+
+const (
+	WRITE_DAC              = 0x40000
+	WRITE_OWNER            = 0x80000
+	ACCESS_SYSTEM_SECURITY = 0x1000000
+)
+
+// BackupHeader represents a backup stream of a file.
+type BackupHeader struct {
+	Id         uint32 // The backup stream ID
+	Attributes uint32 // Stream attributes
+	Size       int64  // The size of the stream in bytes
+	Name       string // The name of the stream (for BackupAlternateData only).
+	Offset     int64  // The offset of the stream in the file (for BackupSparseBlock only).
+}
+
+type win32StreamId struct {
+	StreamId   uint32
+	Attributes uint32
+	Size       uint64
+	NameSize   uint32
+}
+
+// BackupStreamReader reads from a stream produced by the BackupRead Win32 API and produces a series
+// of BackupHeader values.
+type BackupStreamReader struct {
+	r         io.Reader
+	bytesLeft int64
+}
+
+// NewBackupStreamReader produces a BackupStreamReader from any io.Reader.
+func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
+	return &BackupStreamReader{r, 0}
+}
+
+// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
+// it was not completely read.
+func (r *BackupStreamReader) Next() (*BackupHeader, error) {
+	if r.bytesLeft > 0 {
+		if s, ok := r.r.(io.Seeker); ok {
+			// Make sure Seek on io.SeekCurrent sometimes succeeds
+			// before trying the actual seek.
+			if _, err := s.Seek(0, io.SeekCurrent); err == nil {
+				if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil {
+					return nil, err
+				}
+				r.bytesLeft = 0
+			}
+		}
+		if _, err := io.Copy(ioutil.Discard, r); err != nil {
+			return nil, err
+		}
+	}
+	var wsi win32StreamId
+	if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
+		return nil, err
+	}
+	hdr := &BackupHeader{
+		Id:         wsi.StreamId,
+		Attributes: wsi.Attributes,
+		Size:       int64(wsi.Size),
+	}
+	if wsi.NameSize != 0 {
+		name := make([]uint16, int(wsi.NameSize/2))
+		if err := binary.Read(r.r, binary.LittleEndian, name); err != nil {
+			return nil, err
+		}
+		hdr.Name = syscall.UTF16ToString(name)
+	}
+	if wsi.StreamId == BackupSparseBlock {
+		if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
+			return nil, err
+		}
+		hdr.Size -= 8
+	}
+	r.bytesLeft = hdr.Size
+	return hdr, nil
+}
+
+// Read reads from the current backup stream.
+func (r *BackupStreamReader) Read(b []byte) (int, error) {
+	if r.bytesLeft == 0 {
+		return 0, io.EOF
+	}
+	if int64(len(b)) > r.bytesLeft {
+		b = b[:r.bytesLeft]
+	}
+	n, err := r.r.Read(b)
+	r.bytesLeft -= int64(n)
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	} else if r.bytesLeft == 0 && err == nil {
+		err = io.EOF
+	}
+	return n, err
+}
+
+// BackupStreamWriter writes a stream compatible with the BackupWrite Win32 API.
+type BackupStreamWriter struct {
+	w         io.Writer
+	bytesLeft int64
+}
+
+// NewBackupStreamWriter produces a BackupStreamWriter on top of an io.Writer.
+func NewBackupStreamWriter(w io.Writer) *BackupStreamWriter {
+	return &BackupStreamWriter{w, 0}
+}
+
+// WriteHeader writes the next backup stream header and prepares for calls to Write().
+func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
+	if w.bytesLeft != 0 {
+		return fmt.Errorf("missing %d bytes", w.bytesLeft)
+	}
+	name := utf16.Encode([]rune(hdr.Name))
+	wsi := win32StreamId{
+		StreamId:   hdr.Id,
+		Attributes: hdr.Attributes,
+		Size:       uint64(hdr.Size),
+		NameSize:   uint32(len(name) * 2),
+	}
+	if hdr.Id == BackupSparseBlock {
+		// Include space for the int64 block offset
+		wsi.Size += 8
+	}
+	if err := binary.Write(w.w, binary.LittleEndian, &wsi); err != nil {
+		return err
+	}
+	if len(name) != 0 {
+		if err := binary.Write(w.w, binary.LittleEndian, name); err != nil {
+			return err
+		}
+	}
+	if hdr.Id == BackupSparseBlock {
+		if err := binary.Write(w.w, binary.LittleEndian, hdr.Offset); err != nil {
+			return err
+		}
+	}
+	w.bytesLeft = hdr.Size
+	return nil
+}
+
+// Write writes to the current backup stream.
+func (w *BackupStreamWriter) Write(b []byte) (int, error) {
+	if w.bytesLeft < int64(len(b)) {
+		return 0, fmt.Errorf("too many bytes by %d", int64(len(b))-w.bytesLeft)
+	}
+	n, err := w.w.Write(b)
+	w.bytesLeft -= int64(n)
+	return n, err
+}
+
+// BackupFileReader provides an io.ReadCloser interface on top of the BackupRead Win32 API.
+type BackupFileReader struct {
+	f               *os.File
+	includeSecurity bool
+	ctx             uintptr
+}
+
+// NewBackupFileReader returns a new BackupFileReader from a file handle. If includeSecurity is true,
+// Read will attempt to read the security descriptor of the file.
+func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader {
+	r := &BackupFileReader{f, includeSecurity, 0}
+	return r
+}
+
+// Read reads a backup stream from the file by calling the Win32 API BackupRead().
+func (r *BackupFileReader) Read(b []byte) (int, error) {
+	var bytesRead uint32
+	err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
+	if err != nil {
+		return 0, &os.PathError{"BackupRead", r.f.Name(), err}
+	}
+	runtime.KeepAlive(r.f)
+	if bytesRead == 0 {
+		return 0, io.EOF
+	}
+	return int(bytesRead), nil
+}
+
+// Close frees Win32 resources associated with the BackupFileReader. It does not close
+// the underlying file.
+func (r *BackupFileReader) Close() error {
+	if r.ctx != 0 {
+		backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
+		runtime.KeepAlive(r.f)
+		r.ctx = 0
+	}
+	return nil
+}
+
+// BackupFileWriter provides an io.WriteCloser interface on top of the BackupWrite Win32 API.
+type BackupFileWriter struct {
+	f               *os.File
+	includeSecurity bool
+	ctx             uintptr
+}
+
+// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true,
+// Write() will attempt to restore the security descriptor from the stream.
+func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter {
+	w := &BackupFileWriter{f, includeSecurity, 0}
+	return w
+}
+
+// Write restores a portion of the file using the provided backup stream.
+func (w *BackupFileWriter) Write(b []byte) (int, error) {
+	var bytesWritten uint32
+	err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
+	if err != nil {
+		return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
+	}
+	runtime.KeepAlive(w.f)
+	if int(bytesWritten) != len(b) {
+		return int(bytesWritten), errors.New("not all bytes could be written")
+	}
+	return len(b), nil
+}
+
+// Close frees Win32 resources associated with the BackupFileWriter. It does not
+// close the underlying file.
+func (w *BackupFileWriter) Close() error {
+	if w.ctx != 0 {
+		backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
+		runtime.KeepAlive(w.f)
+		w.ctx = 0
+	}
+	return nil
+}
+
+// OpenForBackup opens a file or directory, potentially skipping access checks if the backup
+// or restore privileges have been acquired.
+//
+// If the file opened was a directory, it cannot be used with Readdir().
+func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) {
+	winPath, err := syscall.UTF16FromString(path)
+	if err != nil {
+		return nil, err
+	}
+	h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
+	if err != nil {
+		err = &os.PathError{Op: "open", Path: path, Err: err}
+		return nil, err
+	}
+	return os.NewFile(uintptr(h), path), nil
+}

vendor/github.com/Microsoft/go-winio/ea.go

@@ -0,0 +1,137 @@
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+)
+
+type fileFullEaInformation struct {
+	NextEntryOffset uint32
+	Flags           uint8
+	NameLength      uint8
+	ValueLength     uint16
+}
+
+var (
+	fileFullEaInformationSize = binary.Size(&fileFullEaInformation{})
+
+	errInvalidEaBuffer = errors.New("invalid extended attribute buffer")
+	errEaNameTooLarge  = errors.New("extended attribute name too large")
+	errEaValueTooLarge = errors.New("extended attribute value too large")
+)
+
+// ExtendedAttribute represents a single Windows EA.
+type ExtendedAttribute struct {
+	Name  string
+	Value []byte
+	Flags uint8
+}
+
+func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
+	var info fileFullEaInformation
+	err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
+	if err != nil {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	nameOffset := fileFullEaInformationSize
+	nameLen := int(info.NameLength)
+	valueOffset := nameOffset + int(info.NameLength) + 1
+	valueLen := int(info.ValueLength)
+	nextOffset := int(info.NextEntryOffset)
+	if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	ea.Name = string(b[nameOffset : nameOffset+nameLen])
+	ea.Value = b[valueOffset : valueOffset+valueLen]
+	ea.Flags = info.Flags
+	if info.NextEntryOffset != 0 {
+		nb = b[info.NextEntryOffset:]
+	}
+	return
+}
+
+// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
+// buffer retrieved from BackupRead, ZwQueryEaFile, etc.
+func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
+	for len(b) != 0 {
+		ea, nb, err := parseEa(b)
+		if err != nil {
+			return nil, err
+		}
+
+		eas = append(eas, ea)
+		b = nb
+	}
+	return
+}
+
+func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
+	if int(uint8(len(ea.Name))) != len(ea.Name) {
+		return errEaNameTooLarge
+	}
+	if int(uint16(len(ea.Value))) != len(ea.Value) {
+		return errEaValueTooLarge
+	}
+	entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value))
+	withPadding := (entrySize + 3) &^ 3
+	nextOffset := uint32(0)
+	if !last {
+		nextOffset = withPadding
+	}
+	info := fileFullEaInformation{
+		NextEntryOffset: nextOffset,
+		Flags:           ea.Flags,
+		NameLength:      uint8(len(ea.Name)),
+		ValueLength:     uint16(len(ea.Value)),
+	}
+
+	err := binary.Write(buf, binary.LittleEndian, &info)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte(ea.Name))
+	if err != nil {
+		return err
+	}
+
+	err = buf.WriteByte(0)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write(ea.Value)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION
+// buffer for use with BackupWrite, ZwSetEaFile, etc.
+func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) {
+	var buf bytes.Buffer
+	for i := range eas {
+		last := false
+		if i == len(eas)-1 {
+			last = true
+		}
+
+		err := writeEa(&buf, &eas[i], last)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}

vendor/github.com/Microsoft/go-winio/file.go

@@ -0,0 +1,307 @@
+// +build windows
+
+package winio
+
+import (
+	"errors"
+	"io"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+)
+
+//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
+//sys createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) = CreateIoCompletionPort
+//sys getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
+//sys setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
+
+type atomicBool int32
+
+func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
+func (b *atomicBool) setFalse()   { atomic.StoreInt32((*int32)(b), 0) }
+func (b *atomicBool) setTrue()    { atomic.StoreInt32((*int32)(b), 1) }
+func (b *atomicBool) swap(new bool) bool {
+	var newInt int32
+	if new {
+		newInt = 1
+	}
+	return atomic.SwapInt32((*int32)(b), newInt) == 1
+}
+
+const (
+	cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
+	cFILE_SKIP_SET_EVENT_ON_HANDLE        = 2
+)
+
+var (
+	ErrFileClosed = errors.New("file has already been closed")
+	ErrTimeout    = &timeoutError{}
+)
+
+type timeoutError struct{}
+
+func (e *timeoutError) Error() string   { return "i/o timeout" }
+func (e *timeoutError) Timeout() bool   { return true }
+func (e *timeoutError) Temporary() bool { return true }
+
+type timeoutChan chan struct{}
+
+var ioInitOnce sync.Once
+var ioCompletionPort syscall.Handle
+
+// ioResult contains the result of an asynchronous IO operation
+type ioResult struct {
+	bytes uint32
+	err   error
+}
+
+// ioOperation represents an outstanding asynchronous Win32 IO
+type ioOperation struct {
+	o  syscall.Overlapped
+	ch chan ioResult
+}
+
+func initIo() {
+	h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
+	if err != nil {
+		panic(err)
+	}
+	ioCompletionPort = h
+	go ioCompletionProcessor(h)
+}
+
+// win32File implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall.
+// It takes ownership of this handle and will close it if it is garbage collected.
+type win32File struct {
+	handle        syscall.Handle
+	wg            sync.WaitGroup
+	wgLock        sync.RWMutex
+	closing       atomicBool
+	readDeadline  deadlineHandler
+	writeDeadline deadlineHandler
+}
+
+type deadlineHandler struct {
+	setLock     sync.Mutex
+	channel     timeoutChan
+	channelLock sync.RWMutex
+	timer       *time.Timer
+	timedout    atomicBool
+}
+
+// makeWin32File makes a new win32File from an existing file handle
+func makeWin32File(h syscall.Handle) (*win32File, error) {
+	f := &win32File{handle: h}
+	ioInitOnce.Do(initIo)
+	_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
+	if err != nil {
+		return nil, err
+	}
+	err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
+	if err != nil {
+		return nil, err
+	}
+	f.readDeadline.channel = make(timeoutChan)
+	f.writeDeadline.channel = make(timeoutChan)
+	return f, nil
+}
+
+func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
+	return makeWin32File(h)
+}
+
+// closeHandle closes the resources associated with a Win32 handle
+func (f *win32File) closeHandle() {
+	f.wgLock.Lock()
+	// Atomically set that we are closing, releasing the resources only once.
+	if !f.closing.swap(true) {
+		f.wgLock.Unlock()
+		// cancel all IO and wait for it to complete
+		cancelIoEx(f.handle, nil)
+		f.wg.Wait()
+		// at this point, no new IO can start
+		syscall.Close(f.handle)
+		f.handle = 0
+	} else {
+		f.wgLock.Unlock()
+	}
+}
+
+// Close closes a win32File.
+func (f *win32File) Close() error {
+	f.closeHandle()
+	return nil
+}
+
+// prepareIo prepares for a new IO operation.
+// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
+func (f *win32File) prepareIo() (*ioOperation, error) {
+	f.wgLock.RLock()
+	if f.closing.isSet() {
+		f.wgLock.RUnlock()
+		return nil, ErrFileClosed
+	}
+	f.wg.Add(1)
+	f.wgLock.RUnlock()
+	c := &ioOperation{}
+	c.ch = make(chan ioResult)
+	return c, nil
+}
+
+// ioCompletionProcessor processes completed async IOs forever
+func ioCompletionProcessor(h syscall.Handle) {
+	for {
+		var bytes uint32
+		var key uintptr
+		var op *ioOperation
+		err := getQueuedCompletionStatus(h, &bytes, &key, &op, syscall.INFINITE)
+		if op == nil {
+			panic(err)
+		}
+		op.ch <- ioResult{bytes, err}
+	}
+}
+
+// asyncIo processes the return value from ReadFile or WriteFile, blocking until
+// the operation has actually completed.
+func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
+	if err != syscall.ERROR_IO_PENDING {
+		return int(bytes), err
+	}
+
+	if f.closing.isSet() {
+		cancelIoEx(f.handle, &c.o)
+	}
+
+	var timeout timeoutChan
+	if d != nil {
+		d.channelLock.Lock()
+		timeout = d.channel
+		d.channelLock.Unlock()
+	}
+
+	var r ioResult
+	select {
+	case r = <-c.ch:
+		err = r.err
+		if err == syscall.ERROR_OPERATION_ABORTED {
+			if f.closing.isSet() {
+				err = ErrFileClosed
+			}
+		}
+	case <-timeout:
+		cancelIoEx(f.handle, &c.o)
+		r = <-c.ch
+		err = r.err
+		if err == syscall.ERROR_OPERATION_ABORTED {
+			err = ErrTimeout
+		}
+	}
+
+	// runtime.KeepAlive is needed, as c is passed via native
+	// code to ioCompletionProcessor, c must remain alive
+	// until the channel read is complete.
+	runtime.KeepAlive(c)
+	return int(r.bytes), err
+}
+
+// Read reads from a file handle.
+func (f *win32File) Read(b []byte) (int, error) {
+	c, err := f.prepareIo()
+	if err != nil {
+		return 0, err
+	}
+	defer f.wg.Done()
+
+	if f.readDeadline.timedout.isSet() {
+		return 0, ErrTimeout
+	}
+
+	var bytes uint32
+	err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
+	n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
+	runtime.KeepAlive(b)
+
+	// Handle EOF conditions.
+	if err == nil && n == 0 && len(b) != 0 {
+		return 0, io.EOF
+	} else if err == syscall.ERROR_BROKEN_PIPE {
+		return 0, io.EOF
+	} else {
+		return n, err
+	}
+}
+
+// Write writes to a file handle.
+func (f *win32File) Write(b []byte) (int, error) {
+	c, err := f.prepareIo()
+	if err != nil {
+		return 0, err
+	}
+	defer f.wg.Done()
+
+	if f.writeDeadline.timedout.isSet() {
+		return 0, ErrTimeout
+	}
+
+	var bytes uint32
+	err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
+	n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
+	runtime.KeepAlive(b)
+	return n, err
+}
+
+func (f *win32File) SetReadDeadline(deadline time.Time) error {
+	return f.readDeadline.set(deadline)
+}
+
+func (f *win32File) SetWriteDeadline(deadline time.Time) error {
+	return f.writeDeadline.set(deadline)
+}
+
+func (f *win32File) Flush() error {
+	return syscall.FlushFileBuffers(f.handle)
+}
+
+func (d *deadlineHandler) set(deadline time.Time) error {
+	d.setLock.Lock()
+	defer d.setLock.Unlock()
+
+	if d.timer != nil {
+		if !d.timer.Stop() {
+			<-d.channel
+		}
+		d.timer = nil
+	}
+	d.timedout.setFalse()
+
+	select {
+	case <-d.channel:
+		d.channelLock.Lock()
+		d.channel = make(chan struct{})
+		d.channelLock.Unlock()
+	default:
+	}
+
+	if deadline.IsZero() {
+		return nil
+	}
+
+	timeoutIO := func() {
+		d.timedout.setTrue()
+		close(d.channel)
+	}
+
+	now := time.Now()
+	duration := deadline.Sub(now)
+	if deadline.After(now) {
+		// Deadline is in the future, set a timer to wait
+		d.timer = time.AfterFunc(duration, timeoutIO)
+	} else {
+		// Deadline is in the past. Cancel all pending IO now.
+		timeoutIO()
+	}
+	return nil
+}

vendor/github.com/Microsoft/go-winio/fileinfo.go

@@ -0,0 +1,61 @@
+// +build windows
+
+package winio
+
+import (
+	"os"
+	"runtime"
+	"syscall"
+	"unsafe"
+)
+
+//sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
+//sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
+
+const (
+	fileBasicInfo = 0
+	fileIDInfo    = 0x12
+)
+
+// FileBasicInfo contains file access time and file attributes information.
+type FileBasicInfo struct {
+	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
+	FileAttributes                                          uint32
+	pad                                                     uint32 // padding
+}
+
+// GetFileBasicInfo retrieves times and attributes for a file.
+func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
+	bi := &FileBasicInfo{}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return bi, nil
+}
+
+// SetFileBasicInfo sets times and attributes for a file.
+func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
+	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return nil
+}
+
+// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
+// unique on a system.
+type FileIDInfo struct {
+	VolumeSerialNumber uint64
+	FileID             [16]byte
+}
+
+// GetFileID retrieves the unique (volume, file ID) pair for a file.
+func GetFileID(f *os.File) (*FileIDInfo, error) {
+	fileID := &FileIDInfo{}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return fileID, nil
+}

vendor/github.com/Microsoft/go-winio/pipe.go

@@ -0,0 +1,421 @@
+// +build windows
+
+package winio
+
+import (
+	"errors"
+	"io"
+	"net"
+	"os"
+	"syscall"
+	"time"
+	"unsafe"
+)
+
+//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
+//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error)  [failretval==syscall.InvalidHandle] = CreateNamedPipeW
+//sys createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateFileW
+//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
+//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
+//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
+
+const (
+	cERROR_PIPE_BUSY      = syscall.Errno(231)
+	cERROR_NO_DATA        = syscall.Errno(232)
+	cERROR_PIPE_CONNECTED = syscall.Errno(535)
+	cERROR_SEM_TIMEOUT    = syscall.Errno(121)
+
+	cPIPE_ACCESS_DUPLEX            = 0x3
+	cFILE_FLAG_FIRST_PIPE_INSTANCE = 0x80000
+	cSECURITY_SQOS_PRESENT         = 0x100000
+	cSECURITY_ANONYMOUS            = 0
+
+	cPIPE_REJECT_REMOTE_CLIENTS = 0x8
+
+	cPIPE_UNLIMITED_INSTANCES = 255
+
+	cNMPWAIT_USE_DEFAULT_WAIT = 0
+	cNMPWAIT_NOWAIT           = 1
+
+	cPIPE_TYPE_MESSAGE = 4
+
+	cPIPE_READMODE_MESSAGE = 2
+)
+
+var (
+	// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
+	// This error should match net.errClosing since docker takes a dependency on its text.
+	ErrPipeListenerClosed = errors.New("use of closed network connection")
+
+	errPipeWriteClosed = errors.New("pipe has been closed for write")
+)
+
+type win32Pipe struct {
+	*win32File
+	path string
+}
+
+type win32MessageBytePipe struct {
+	win32Pipe
+	writeClosed bool
+	readEOF     bool
+}
+
+type pipeAddress string
+
+func (f *win32Pipe) LocalAddr() net.Addr {
+	return pipeAddress(f.path)
+}
+
+func (f *win32Pipe) RemoteAddr() net.Addr {
+	return pipeAddress(f.path)
+}
+
+func (f *win32Pipe) SetDeadline(t time.Time) error {
+	f.SetReadDeadline(t)
+	f.SetWriteDeadline(t)
+	return nil
+}
+
+// CloseWrite closes the write side of a message pipe in byte mode.
+func (f *win32MessageBytePipe) CloseWrite() error {
+	if f.writeClosed {
+		return errPipeWriteClosed
+	}
+	err := f.win32File.Flush()
+	if err != nil {
+		return err
+	}
+	_, err = f.win32File.Write(nil)
+	if err != nil {
+		return err
+	}
+	f.writeClosed = true
+	return nil
+}
+
+// Write writes bytes to a message pipe in byte mode. Zero-byte writes are ignored, since
+// they are used to implement CloseWrite().
+func (f *win32MessageBytePipe) Write(b []byte) (int, error) {
+	if f.writeClosed {
+		return 0, errPipeWriteClosed
+	}
+	if len(b) == 0 {
+		return 0, nil
+	}
+	return f.win32File.Write(b)
+}
+
+// Read reads bytes from a message pipe in byte mode. A read of a zero-byte message on a message
+// mode pipe will return io.EOF, as will all subsequent reads.
+func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
+	if f.readEOF {
+		return 0, io.EOF
+	}
+	n, err := f.win32File.Read(b)
+	if err == io.EOF {
+		// If this was the result of a zero-byte read, then
+		// it is possible that the read was due to a zero-size
+		// message. Since we are simulating CloseWrite with a
+		// zero-byte message, ensure that all future Read() calls
+		// also return EOF.
+		f.readEOF = true
+	} else if err == syscall.ERROR_MORE_DATA {
+		// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
+		// and the message still has more bytes. Treat this as a success, since
+		// this package presents all named pipes as byte streams.
+		err = nil
+	}
+	return n, err
+}
+
+func (s pipeAddress) Network() string {
+	return "pipe"
+}
+
+func (s pipeAddress) String() string {
+	return string(s)
+}
+
+// DialPipe connects to a named pipe by path, timing out if the connection
+// takes longer than the specified duration. If timeout is nil, then we use
+// a default timeout of 5 seconds.  (We do not use WaitNamedPipe.)
+func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
+	var absTimeout time.Time
+	if timeout != nil {
+		absTimeout = time.Now().Add(*timeout)
+	} else {
+		absTimeout = time.Now().Add(time.Second * 2)
+	}
+	var err error
+	var h syscall.Handle
+	for {
+		h, err = createFile(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
+		if err != cERROR_PIPE_BUSY {
+			break
+		}
+		if time.Now().After(absTimeout) {
+			return nil, ErrTimeout
+		}
+
+		// Wait 10 msec and try again. This is a rather simplistic
+		// view, as we always try each 10 milliseconds.
+		time.Sleep(time.Millisecond * 10)
+	}
+	if err != nil {
+		return nil, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+
+	var flags uint32
+	err = getNamedPipeInfo(h, &flags, nil, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := makeWin32File(h)
+	if err != nil {
+		syscall.Close(h)
+		return nil, err
+	}
+
+	// If the pipe is in message mode, return a message byte pipe, which
+	// supports CloseWrite().
+	if flags&cPIPE_TYPE_MESSAGE != 0 {
+		return &win32MessageBytePipe{
+			win32Pipe: win32Pipe{win32File: f, path: path},
+		}, nil
+	}
+	return &win32Pipe{win32File: f, path: path}, nil
+}
+
+type acceptResponse struct {
+	f   *win32File
+	err error
+}
+
+type win32PipeListener struct {
+	firstHandle        syscall.Handle
+	path               string
+	securityDescriptor []byte
+	config             PipeConfig
+	acceptCh           chan (chan acceptResponse)
+	closeCh            chan int
+	doneCh             chan int
+}
+
+func makeServerPipeHandle(path string, securityDescriptor []byte, c *PipeConfig, first bool) (syscall.Handle, error) {
+	var flags uint32 = cPIPE_ACCESS_DUPLEX | syscall.FILE_FLAG_OVERLAPPED
+	if first {
+		flags |= cFILE_FLAG_FIRST_PIPE_INSTANCE
+	}
+
+	var mode uint32 = cPIPE_REJECT_REMOTE_CLIENTS
+	if c.MessageMode {
+		mode |= cPIPE_TYPE_MESSAGE
+	}
+
+	sa := &syscall.SecurityAttributes{}
+	sa.Length = uint32(unsafe.Sizeof(*sa))
+	if securityDescriptor != nil {
+		len := uint32(len(securityDescriptor))
+		sa.SecurityDescriptor = localAlloc(0, len)
+		defer localFree(sa.SecurityDescriptor)
+		copy((*[0xffff]byte)(unsafe.Pointer(sa.SecurityDescriptor))[:], securityDescriptor)
+	}
+	h, err := createNamedPipe(path, flags, mode, cPIPE_UNLIMITED_INSTANCES, uint32(c.OutputBufferSize), uint32(c.InputBufferSize), 0, sa)
+	if err != nil {
+		return 0, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+	return h, nil
+}
+
+func (l *win32PipeListener) makeServerPipe() (*win32File, error) {
+	h, err := makeServerPipeHandle(l.path, l.securityDescriptor, &l.config, false)
+	if err != nil {
+		return nil, err
+	}
+	f, err := makeWin32File(h)
+	if err != nil {
+		syscall.Close(h)
+		return nil, err
+	}
+	return f, nil
+}
+
+func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
+	p, err := l.makeServerPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	// Wait for the client to connect.
+	ch := make(chan error)
+	go func(p *win32File) {
+		ch <- connectPipe(p)
+	}(p)
+
+	select {
+	case err = <-ch:
+		if err != nil {
+			p.Close()
+			p = nil
+		}
+	case <-l.closeCh:
+		// Abort the connect request by closing the handle.
+		p.Close()
+		p = nil
+		err = <-ch
+		if err == nil || err == ErrFileClosed {
+			err = ErrPipeListenerClosed
+		}
+	}
+	return p, err
+}
+
+func (l *win32PipeListener) listenerRoutine() {
+	closed := false
+	for !closed {
+		select {
+		case <-l.closeCh:
+			closed = true
+		case responseCh := <-l.acceptCh:
+			var (
+				p   *win32File
+				err error
+			)
+			for {
+				p, err = l.makeConnectedServerPipe()
+				// If the connection was immediately closed by the client, try
+				// again.
+				if err != cERROR_NO_DATA {
+					break
+				}
+			}
+			responseCh <- acceptResponse{p, err}
+			closed = err == ErrPipeListenerClosed
+		}
+	}
+	syscall.Close(l.firstHandle)
+	l.firstHandle = 0
+	// Notify Close() and Accept() callers that the handle has been closed.
+	close(l.doneCh)
+}
+
+// PipeConfig contain configuration for the pipe listener.
+type PipeConfig struct {
+	// SecurityDescriptor contains a Windows security descriptor in SDDL format.
+	SecurityDescriptor string
+
+	// MessageMode determines whether the pipe is in byte or message mode. In either
+	// case the pipe is read in byte mode by default. The only practical difference in
+	// this implementation is that CloseWrite() is only supported for message mode pipes;
+	// CloseWrite() is implemented as a zero-byte write, but zero-byte writes are only
+	// transferred to the reader (and returned as io.EOF in this implementation)
+	// when the pipe is in message mode.
+	MessageMode bool
+
+	// InputBufferSize specifies the size the input buffer, in bytes.
+	InputBufferSize int32
+
+	// OutputBufferSize specifies the size the input buffer, in bytes.
+	OutputBufferSize int32
+}
+
+// ListenPipe creates a listener on a Windows named pipe path, e.g. \\.\pipe\mypipe.
+// The pipe must not already exist.
+func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
+	var (
+		sd  []byte
+		err error
+	)
+	if c == nil {
+		c = &PipeConfig{}
+	}
+	if c.SecurityDescriptor != "" {
+		sd, err = SddlToSecurityDescriptor(c.SecurityDescriptor)
+		if err != nil {
+			return nil, err
+		}
+	}
+	h, err := makeServerPipeHandle(path, sd, c, true)
+	if err != nil {
+		return nil, err
+	}
+	// Create a client handle and connect it.  This results in the pipe
+	// instance always existing, so that clients see ERROR_PIPE_BUSY
+	// rather than ERROR_FILE_NOT_FOUND.  This ties the first instance
+	// up so that no other instances can be used.  This would have been
+	// cleaner if the Win32 API matched CreateFile with ConnectNamedPipe
+	// instead of CreateNamedPipe.  (Apparently created named pipes are
+	// considered to be in listening state regardless of whether any
+	// active calls to ConnectNamedPipe are outstanding.)
+	h2, err := createFile(path, 0, 0, nil, syscall.OPEN_EXISTING, cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
+	if err != nil {
+		syscall.Close(h)
+		return nil, err
+	}
+	// Close the client handle. The server side of the instance will
+	// still be busy, leading to ERROR_PIPE_BUSY instead of
+	// ERROR_NOT_FOUND, as long as we don't close the server handle,
+	// or disconnect the client with DisconnectNamedPipe.
+	syscall.Close(h2)
+	l := &win32PipeListener{
+		firstHandle:        h,
+		path:               path,
+		securityDescriptor: sd,
+		config:             *c,
+		acceptCh:           make(chan (chan acceptResponse)),
+		closeCh:            make(chan int),
+		doneCh:             make(chan int),
+	}
+	go l.listenerRoutine()
+	return l, nil
+}
+
+func connectPipe(p *win32File) error {
+	c, err := p.prepareIo()
+	if err != nil {
+		return err
+	}
+	defer p.wg.Done()
+
+	err = connectNamedPipe(p.handle, &c.o)
+	_, err = p.asyncIo(c, nil, 0, err)
+	if err != nil && err != cERROR_PIPE_CONNECTED {
+		return err
+	}
+	return nil
+}
+
+func (l *win32PipeListener) Accept() (net.Conn, error) {
+	ch := make(chan acceptResponse)
+	select {
+	case l.acceptCh <- ch:
+		response := <-ch
+		err := response.err
+		if err != nil {
+			return nil, err
+		}
+		if l.config.MessageMode {
+			return &win32MessageBytePipe{
+				win32Pipe: win32Pipe{win32File: response.f, path: l.path},
+			}, nil
+		}
+		return &win32Pipe{win32File: response.f, path: l.path}, nil
+	case <-l.doneCh:
+		return nil, ErrPipeListenerClosed
+	}
+}
+
+func (l *win32PipeListener) Close() error {
+	select {
+	case l.closeCh <- 1:
+		<-l.doneCh
+	case <-l.doneCh:
+	}
+	return nil
+}
+
+func (l *win32PipeListener) Addr() net.Addr {
+	return pipeAddress(l.path)
+}

vendor/github.com/Microsoft/go-winio/privilege.go

@@ -0,0 +1,202 @@
+// +build windows
+
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"runtime"
+	"sync"
+	"syscall"
+	"unicode/utf16"
+
+	"golang.org/x/sys/windows"
+)
+
+//sys adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) [true] = advapi32.AdjustTokenPrivileges
+//sys impersonateSelf(level uint32) (err error) = advapi32.ImpersonateSelf
+//sys revertToSelf() (err error) = advapi32.RevertToSelf
+//sys openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken
+//sys getCurrentThread() (h syscall.Handle) = GetCurrentThread
+//sys lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) = advapi32.LookupPrivilegeValueW
+//sys lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) = advapi32.LookupPrivilegeNameW
+//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
+
+const (
+	SE_PRIVILEGE_ENABLED = 2
+
+	ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
+
+	SeBackupPrivilege  = "SeBackupPrivilege"
+	SeRestorePrivilege = "SeRestorePrivilege"
+)
+
+const (
+	securityAnonymous = iota
+	securityIdentification
+	securityImpersonation
+	securityDelegation
+)
+
+var (
+	privNames     = make(map[string]uint64)
+	privNameMutex sync.Mutex
+)
+
+// PrivilegeError represents an error enabling privileges.
+type PrivilegeError struct {
+	privileges []uint64
+}
+
+func (e *PrivilegeError) Error() string {
+	s := ""
+	if len(e.privileges) > 1 {
+		s = "Could not enable privileges "
+	} else {
+		s = "Could not enable privilege "
+	}
+	for i, p := range e.privileges {
+		if i != 0 {
+			s += ", "
+		}
+		s += `"`
+		s += getPrivilegeName(p)
+		s += `"`
+	}
+	return s
+}
+
+// RunWithPrivilege enables a single privilege for a function call.
+func RunWithPrivilege(name string, fn func() error) error {
+	return RunWithPrivileges([]string{name}, fn)
+}
+
+// RunWithPrivileges enables privileges for a function call.
+func RunWithPrivileges(names []string, fn func() error) error {
+	privileges, err := mapPrivileges(names)
+	if err != nil {
+		return err
+	}
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	token, err := newThreadToken()
+	if err != nil {
+		return err
+	}
+	defer releaseThreadToken(token)
+	err = adjustPrivileges(token, privileges, SE_PRIVILEGE_ENABLED)
+	if err != nil {
+		return err
+	}
+	return fn()
+}
+
+func mapPrivileges(names []string) ([]uint64, error) {
+	var privileges []uint64
+	privNameMutex.Lock()
+	defer privNameMutex.Unlock()
+	for _, name := range names {
+		p, ok := privNames[name]
+		if !ok {
+			err := lookupPrivilegeValue("", name, &p)
+			if err != nil {
+				return nil, err
+			}
+			privNames[name] = p
+		}
+		privileges = append(privileges, p)
+	}
+	return privileges, nil
+}
+
+// EnableProcessPrivileges enables privileges globally for the process.
+func EnableProcessPrivileges(names []string) error {
+	return enableDisableProcessPrivilege(names, SE_PRIVILEGE_ENABLED)
+}
+
+// DisableProcessPrivileges disables privileges globally for the process.
+func DisableProcessPrivileges(names []string) error {
+	return enableDisableProcessPrivilege(names, 0)
+}
+
+func enableDisableProcessPrivilege(names []string, action uint32) error {
+	privileges, err := mapPrivileges(names)
+	if err != nil {
+		return err
+	}
+
+	p, _ := windows.GetCurrentProcess()
+	var token windows.Token
+	err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
+	if err != nil {
+		return err
+	}
+
+	defer token.Close()
+	return adjustPrivileges(token, privileges, action)
+}
+
+func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
+	var b bytes.Buffer
+	binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
+	for _, p := range privileges {
+		binary.Write(&b, binary.LittleEndian, p)
+		binary.Write(&b, binary.LittleEndian, action)
+	}
+	prevState := make([]byte, b.Len())
+	reqSize := uint32(0)
+	success, err := adjustTokenPrivileges(token, false, &b.Bytes()[0], uint32(len(prevState)), &prevState[0], &reqSize)
+	if !success {
+		return err
+	}
+	if err == ERROR_NOT_ALL_ASSIGNED {
+		return &PrivilegeError{privileges}
+	}
+	return nil
+}
+
+func getPrivilegeName(luid uint64) string {
+	var nameBuffer [256]uint16
+	bufSize := uint32(len(nameBuffer))
+	err := lookupPrivilegeName("", &luid, &nameBuffer[0], &bufSize)
+	if err != nil {
+		return fmt.Sprintf("<unknown privilege %d>", luid)
+	}
+
+	var displayNameBuffer [256]uint16
+	displayBufSize := uint32(len(displayNameBuffer))
+	var langID uint32
+	err = lookupPrivilegeDisplayName("", &nameBuffer[0], &displayNameBuffer[0], &displayBufSize, &langID)
+	if err != nil {
+		return fmt.Sprintf("<unknown privilege %s>", string(utf16.Decode(nameBuffer[:bufSize])))
+	}
+
+	return string(utf16.Decode(displayNameBuffer[:displayBufSize]))
+}
+
+func newThreadToken() (windows.Token, error) {
+	err := impersonateSelf(securityImpersonation)
+	if err != nil {
+		return 0, err
+	}
+
+	var token windows.Token
+	err = openThreadToken(getCurrentThread(), syscall.TOKEN_ADJUST_PRIVILEGES|syscall.TOKEN_QUERY, false, &token)
+	if err != nil {
+		rerr := revertToSelf()
+		if rerr != nil {
+			panic(rerr)
+		}
+		return 0, err
+	}
+	return token, nil
+}
+
+func releaseThreadToken(h windows.Token) {
+	err := revertToSelf()
+	if err != nil {
+		panic(err)
+	}
+	h.Close()
+}

vendor/github.com/Microsoft/go-winio/reparse.go

@@ -0,0 +1,128 @@
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"strings"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	reparseTagMountPoint = 0xA0000003
+	reparseTagSymlink    = 0xA000000C
+)
+
+type reparseDataBuffer struct {
+	ReparseTag           uint32
+	ReparseDataLength    uint16
+	Reserved             uint16
+	SubstituteNameOffset uint16
+	SubstituteNameLength uint16
+	PrintNameOffset      uint16
+	PrintNameLength      uint16
+}
+
+// ReparsePoint describes a Win32 symlink or mount point.
+type ReparsePoint struct {
+	Target       string
+	IsMountPoint bool
+}
+
+// UnsupportedReparsePointError is returned when trying to decode a non-symlink or
+// mount point reparse point.
+type UnsupportedReparsePointError struct {
+	Tag uint32
+}
+
+func (e *UnsupportedReparsePointError) Error() string {
+	return fmt.Sprintf("unsupported reparse point %x", e.Tag)
+}
+
+// DecodeReparsePoint decodes a Win32 REPARSE_DATA_BUFFER structure containing either a symlink
+// or a mount point.
+func DecodeReparsePoint(b []byte) (*ReparsePoint, error) {
+	tag := binary.LittleEndian.Uint32(b[0:4])
+	return DecodeReparsePointData(tag, b[8:])
+}
+
+func DecodeReparsePointData(tag uint32, b []byte) (*ReparsePoint, error) {
+	isMountPoint := false
+	switch tag {
+	case reparseTagMountPoint:
+		isMountPoint = true
+	case reparseTagSymlink:
+	default:
+		return nil, &UnsupportedReparsePointError{tag}
+	}
+	nameOffset := 8 + binary.LittleEndian.Uint16(b[4:6])
+	if !isMountPoint {
+		nameOffset += 4
+	}
+	nameLength := binary.LittleEndian.Uint16(b[6:8])
+	name := make([]uint16, nameLength/2)
+	err := binary.Read(bytes.NewReader(b[nameOffset:nameOffset+nameLength]), binary.LittleEndian, &name)
+	if err != nil {
+		return nil, err
+	}
+	return &ReparsePoint{string(utf16.Decode(name)), isMountPoint}, nil
+}
+
+func isDriveLetter(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+// EncodeReparsePoint encodes a Win32 REPARSE_DATA_BUFFER structure describing a symlink or
+// mount point.
+func EncodeReparsePoint(rp *ReparsePoint) []byte {
+	// Generate an NT path and determine if this is a relative path.
+	var ntTarget string
+	relative := false
+	if strings.HasPrefix(rp.Target, `\\?\`) {
+		ntTarget = `\??\` + rp.Target[4:]
+	} else if strings.HasPrefix(rp.Target, `\\`) {
+		ntTarget = `\??\UNC\` + rp.Target[2:]
+	} else if len(rp.Target) >= 2 && isDriveLetter(rp.Target[0]) && rp.Target[1] == ':' {
+		ntTarget = `\??\` + rp.Target
+	} else {
+		ntTarget = rp.Target
+		relative = true
+	}
+
+	// The paths must be NUL-terminated even though they are counted strings.
+	target16 := utf16.Encode([]rune(rp.Target + "\x00"))
+	ntTarget16 := utf16.Encode([]rune(ntTarget + "\x00"))
+
+	size := int(unsafe.Sizeof(reparseDataBuffer{})) - 8
+	size += len(ntTarget16)*2 + len(target16)*2
+
+	tag := uint32(reparseTagMountPoint)
+	if !rp.IsMountPoint {
+		tag = reparseTagSymlink
+		size += 4 // Add room for symlink flags
+	}
+
+	data := reparseDataBuffer{
+		ReparseTag:           tag,
+		ReparseDataLength:    uint16(size),
+		SubstituteNameOffset: 0,
+		SubstituteNameLength: uint16((len(ntTarget16) - 1) * 2),
+		PrintNameOffset:      uint16(len(ntTarget16) * 2),
+		PrintNameLength:      uint16((len(target16) - 1) * 2),
+	}
+
+	var b bytes.Buffer
+	binary.Write(&b, binary.LittleEndian, &data)
+	if !rp.IsMountPoint {
+		flags := uint32(0)
+		if relative {
+			flags |= 1
+		}
+		binary.Write(&b, binary.LittleEndian, flags)
+	}
+
+	binary.Write(&b, binary.LittleEndian, ntTarget16)
+	binary.Write(&b, binary.LittleEndian, target16)
+	return b.Bytes()
+}

vendor/github.com/Microsoft/go-winio/sd.go

@@ -0,0 +1,98 @@
+// +build windows
+
+package winio
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+//sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW
+//sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW
+//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW
+//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW
+//sys localFree(mem uintptr) = LocalFree
+//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength
+
+const (
+	cERROR_NONE_MAPPED = syscall.Errno(1332)
+)
+
+type AccountLookupError struct {
+	Name string
+	Err  error
+}
+
+func (e *AccountLookupError) Error() string {
+	if e.Name == "" {
+		return "lookup account: empty account name specified"
+	}
+	var s string
+	switch e.Err {
+	case cERROR_NONE_MAPPED:
+		s = "not found"
+	default:
+		s = e.Err.Error()
+	}
+	return "lookup account " + e.Name + ": " + s
+}
+
+type SddlConversionError struct {
+	Sddl string
+	Err  error
+}
+
+func (e *SddlConversionError) Error() string {
+	return "convert " + e.Sddl + ": " + e.Err.Error()
+}
+
+// LookupSidByName looks up the SID of an account by name
+func LookupSidByName(name string) (sid string, err error) {
+	if name == "" {
+		return "", &AccountLookupError{name, cERROR_NONE_MAPPED}
+	}
+
+	var sidSize, sidNameUse, refDomainSize uint32
+	err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse)
+	if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER {
+		return "", &AccountLookupError{name, err}
+	}
+	sidBuffer := make([]byte, sidSize)
+	refDomainBuffer := make([]uint16, refDomainSize)
+	err = lookupAccountName(nil, name, &sidBuffer[0], &sidSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
+	if err != nil {
+		return "", &AccountLookupError{name, err}
+	}
+	var strBuffer *uint16
+	err = convertSidToStringSid(&sidBuffer[0], &strBuffer)
+	if err != nil {
+		return "", &AccountLookupError{name, err}
+	}
+	sid = syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(strBuffer))[:])
+	localFree(uintptr(unsafe.Pointer(strBuffer)))
+	return sid, nil
+}
+
+func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
+	var sdBuffer uintptr
+	err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil)
+	if err != nil {
+		return nil, &SddlConversionError{sddl, err}
+	}
+	defer localFree(sdBuffer)
+	sd := make([]byte, getSecurityDescriptorLength(sdBuffer))
+	copy(sd, (*[0xffff]byte)(unsafe.Pointer(sdBuffer))[:len(sd)])
+	return sd, nil
+}
+
+func SecurityDescriptorToSddl(sd []byte) (string, error) {
+	var sddl *uint16
+	// The returned string length seems to including an aribtrary number of terminating NULs.
+	// Don't use it.
+	err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil)
+	if err != nil {
+		return "", err
+	}
+	defer localFree(uintptr(unsafe.Pointer(sddl)))
+	return syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(sddl))[:]), nil
+}

vendor/github.com/Microsoft/go-winio/syscall.go

@@ -0,0 +1,3 @@
+package winio
+
+//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go

vendor/github.com/Microsoft/go-winio/zsyscall_windows.go

@@ -0,0 +1,520 @@
+// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT
+
+package winio
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return nil
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
+	modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
+
+	procCancelIoEx                                           = modkernel32.NewProc("CancelIoEx")
+	procCreateIoCompletionPort                               = modkernel32.NewProc("CreateIoCompletionPort")
+	procGetQueuedCompletionStatus                            = modkernel32.NewProc("GetQueuedCompletionStatus")
+	procSetFileCompletionNotificationModes                   = modkernel32.NewProc("SetFileCompletionNotificationModes")
+	procConnectNamedPipe                                     = modkernel32.NewProc("ConnectNamedPipe")
+	procCreateNamedPipeW                                     = modkernel32.NewProc("CreateNamedPipeW")
+	procCreateFileW                                          = modkernel32.NewProc("CreateFileW")
+	procWaitNamedPipeW                                       = modkernel32.NewProc("WaitNamedPipeW")
+	procGetNamedPipeInfo                                     = modkernel32.NewProc("GetNamedPipeInfo")
+	procGetNamedPipeHandleStateW                             = modkernel32.NewProc("GetNamedPipeHandleStateW")
+	procLocalAlloc                                           = modkernel32.NewProc("LocalAlloc")
+	procLookupAccountNameW                                   = modadvapi32.NewProc("LookupAccountNameW")
+	procConvertSidToStringSidW                               = modadvapi32.NewProc("ConvertSidToStringSidW")
+	procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
+	procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW")
+	procLocalFree                                            = modkernel32.NewProc("LocalFree")
+	procGetSecurityDescriptorLength                          = modadvapi32.NewProc("GetSecurityDescriptorLength")
+	procGetFileInformationByHandleEx                         = modkernel32.NewProc("GetFileInformationByHandleEx")
+	procSetFileInformationByHandle                           = modkernel32.NewProc("SetFileInformationByHandle")
+	procAdjustTokenPrivileges                                = modadvapi32.NewProc("AdjustTokenPrivileges")
+	procImpersonateSelf                                      = modadvapi32.NewProc("ImpersonateSelf")
+	procRevertToSelf                                         = modadvapi32.NewProc("RevertToSelf")
+	procOpenThreadToken                                      = modadvapi32.NewProc("OpenThreadToken")
+	procGetCurrentThread                                     = modkernel32.NewProc("GetCurrentThread")
+	procLookupPrivilegeValueW                                = modadvapi32.NewProc("LookupPrivilegeValueW")
+	procLookupPrivilegeNameW                                 = modadvapi32.NewProc("LookupPrivilegeNameW")
+	procLookupPrivilegeDisplayNameW                          = modadvapi32.NewProc("LookupPrivilegeDisplayNameW")
+	procBackupRead                                           = modkernel32.NewProc("BackupRead")
+	procBackupWrite                                          = modkernel32.NewProc("BackupWrite")
+)
+
+func cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall(procCancelIoEx.Addr(), 2, uintptr(file), uintptr(unsafe.Pointer(o)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) {
+	r0, _, e1 := syscall.Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount), 0, 0)
+	newport = syscall.Handle(r0)
+	if newport == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) {
+	r1, _, e1 := syscall.Syscall(procSetFileCompletionNotificationModes.Addr(), 2, uintptr(h), uintptr(flags), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall(procConnectNamedPipe.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(o)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return
+	}
+	return _createNamedPipe(_p0, flags, pipeMode, maxInstances, outSize, inSize, defaultTimeout, sa)
+}
+
+func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
+	r0, _, e1 := syscall.Syscall9(procCreateNamedPipeW.Addr(), 8, uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa)), 0)
+	handle = syscall.Handle(r0)
+	if handle == syscall.InvalidHandle {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return
+	}
+	return _createFile(_p0, access, mode, sa, createmode, attrs, templatefile)
+}
+
+func _createFile(name *uint16, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
+	r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0)
+	handle = syscall.Handle(r0)
+	if handle == syscall.InvalidHandle {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func waitNamedPipe(name string, timeout uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return
+	}
+	return _waitNamedPipe(_p0, timeout)
+}
+
+func _waitNamedPipe(name *uint16, timeout uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procWaitNamedPipeW.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(timeout), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procGetNamedPipeInfo.Addr(), 5, uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
+	r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func localAlloc(uFlags uint32, length uint32) (ptr uintptr) {
+	r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(uFlags), uintptr(length), 0)
+	ptr = uintptr(r0)
+	return
+}
+
+func lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(accountName)
+	if err != nil {
+		return
+	}
+	return _lookupAccountName(systemName, _p0, sid, sidSize, refDomain, refDomainSize, sidNameUse)
+}
+
+func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall9(procLookupAccountNameW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func convertSidToStringSid(sid *byte, str **uint16) (err error) {
+	r1, _, e1 := syscall.Syscall(procConvertSidToStringSidW.Addr(), 2, uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(str)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(str)
+	if err != nil {
+		return
+	}
+	return _convertStringSecurityDescriptorToSecurityDescriptor(_p0, revision, sd, size)
+}
+
+func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision uint32, sd *uintptr, size *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procConvertStringSecurityDescriptorToSecurityDescriptorW.Addr(), 4, uintptr(unsafe.Pointer(str)), uintptr(revision), uintptr(unsafe.Pointer(sd)), uintptr(unsafe.Pointer(size)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procConvertSecurityDescriptorToStringSecurityDescriptorW.Addr(), 5, uintptr(unsafe.Pointer(sd)), uintptr(revision), uintptr(secInfo), uintptr(unsafe.Pointer(sddl)), uintptr(unsafe.Pointer(sddlSize)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func localFree(mem uintptr) {
+	syscall.Syscall(procLocalFree.Addr(), 1, uintptr(mem), 0, 0)
+	return
+}
+
+func getSecurityDescriptorLength(sd uintptr) (len uint32) {
+	r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0)
+	len = uint32(r0)
+	return
+}
+
+func getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
+	var _p0 uint32
+	if releaseAll {
+		_p0 = 1
+	} else {
+		_p0 = 0
+	}
+	r0, _, e1 := syscall.Syscall6(procAdjustTokenPrivileges.Addr(), 6, uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize)))
+	success = r0 != 0
+	if true {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func impersonateSelf(level uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procImpersonateSelf.Addr(), 1, uintptr(level), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func revertToSelf() (err error) {
+	r1, _, e1 := syscall.Syscall(procRevertToSelf.Addr(), 0, 0, 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) {
+	var _p0 uint32
+	if openAsSelf {
+		_p0 = 1
+	} else {
+		_p0 = 0
+	}
+	r1, _, e1 := syscall.Syscall6(procOpenThreadToken.Addr(), 4, uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func getCurrentThread() (h syscall.Handle) {
+	r0, _, _ := syscall.Syscall(procGetCurrentThread.Addr(), 0, 0, 0, 0)
+	h = syscall.Handle(r0)
+	return
+}
+
+func lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(systemName)
+	if err != nil {
+		return
+	}
+	var _p1 *uint16
+	_p1, err = syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return
+	}
+	return _lookupPrivilegeValue(_p0, _p1, luid)
+}
+
+func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err error) {
+	r1, _, e1 := syscall.Syscall(procLookupPrivilegeValueW.Addr(), 3, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid)))
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(systemName)
+	if err != nil {
+		return
+	}
+	return _lookupPrivilegeName(_p0, luid, buffer, size)
+}
+
+func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procLookupPrivilegeNameW.Addr(), 4, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(systemName)
+	if err != nil {
+		return
+	}
+	return _lookupPrivilegeDisplayName(_p0, name, buffer, size, languageId)
+}
+
+func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procLookupPrivilegeDisplayNameW.Addr(), 5, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) {
+	var _p0 *byte
+	if len(b) > 0 {
+		_p0 = &b[0]
+	}
+	var _p1 uint32
+	if abort {
+		_p1 = 1
+	} else {
+		_p1 = 0
+	}
+	var _p2 uint32
+	if processSecurity {
+		_p2 = 1
+	} else {
+		_p2 = 0
+	}
+	r1, _, e1 := syscall.Syscall9(procBackupRead.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesRead)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) {
+	var _p0 *byte
+	if len(b) > 0 {
+		_p0 = &b[0]
+	}
+	var _p1 uint32
+	if abort {
+		_p1 = 1
+	} else {
+		_p1 = 0
+	}
+	var _p2 uint32
+	if processSecurity {
+		_p2 = 1
+	} else {
+		_p2 = 0
+	}
+	r1, _, e1 := syscall.Syscall9(procBackupWrite.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesWritten)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}

vendor/github.com/Microsoft/hcsshim/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/Microsoft/hcsshim/cmd/runhcs/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2014 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/Microsoft/hcsshim/cmd/runhcs/NOTICE

@@ -0,0 +1,22 @@
+runhcs is a fork of runc.
+
+The following is runc's legal notice.
+
+---
+
+runc
+
+Copyright 2012-2015 Docker, Inc.
+
+This product includes software developed at Docker, Inc. (http://www.docker.com).
+
+The following is courtesy of our legal counsel:
+
+Use and transfer of Docker may be subject to certain restrictions by the
+United States and other governments.
+It is your responsibility to ensure that your use and/or transfer does not
+violate applicable laws.
+
+For more information, please see http://www.bis.doc.gov
+
+See also http://www.apache.org/dev/crypto.html and/or seek legal counsel.

vendor/github.com/Microsoft/hcsshim/container.go

@@ -0,0 +1,192 @@
+package hcsshim
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/mergemaps"
+	"github.com/Microsoft/hcsshim/internal/schema1"
+)
+
+// ContainerProperties holds the properties for a container and the processes running in that container
+type ContainerProperties = schema1.ContainerProperties
+
+// MemoryStats holds the memory statistics for a container
+type MemoryStats = schema1.MemoryStats
+
+// ProcessorStats holds the processor statistics for a container
+type ProcessorStats = schema1.ProcessorStats
+
+// StorageStats holds the storage statistics for a container
+type StorageStats = schema1.StorageStats
+
+// NetworkStats holds the network statistics for a container
+type NetworkStats = schema1.NetworkStats
+
+// Statistics is the structure returned by a statistics call on a container
+type Statistics = schema1.Statistics
+
+// ProcessList is the structure of an item returned by a ProcessList call on a container
+type ProcessListItem = schema1.ProcessListItem
+
+// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container
+type MappedVirtualDiskController = schema1.MappedVirtualDiskController
+
+// Type of Request Support in ModifySystem
+type RequestType = schema1.RequestType
+
+// Type of Resource Support in ModifySystem
+type ResourceType = schema1.ResourceType
+
+// RequestType const
+const (
+	Add     = schema1.Add
+	Remove  = schema1.Remove
+	Network = schema1.Network
+)
+
+// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system
+// Supported resource types are Network and Request Types are Add/Remove
+type ResourceModificationRequestResponse = schema1.ResourceModificationRequestResponse
+
+type container struct {
+	system *hcs.System
+}
+
+// createComputeSystemAdditionalJSON is read from the environment at initialisation
+// time. It allows an environment variable to define additional JSON which
+// is merged in the CreateComputeSystem call to HCS.
+var createContainerAdditionalJSON []byte
+
+func init() {
+	createContainerAdditionalJSON = ([]byte)(os.Getenv("HCSSHIM_CREATECONTAINER_ADDITIONALJSON"))
+}
+
+// CreateContainer creates a new container with the given configuration but does not start it.
+func CreateContainer(id string, c *ContainerConfig) (Container, error) {
+	fullConfig, err := mergemaps.MergeJSON(c, createContainerAdditionalJSON)
+	if err != nil {
+		return nil, fmt.Errorf("failed to merge additional JSON '%s': %s", createContainerAdditionalJSON, err)
+	}
+
+	system, err := hcs.CreateComputeSystem(id, fullConfig)
+	if err != nil {
+		return nil, err
+	}
+	return &container{system}, err
+}
+
+// OpenContainer opens an existing container by ID.
+func OpenContainer(id string) (Container, error) {
+	system, err := hcs.OpenComputeSystem(id)
+	if err != nil {
+		return nil, err
+	}
+	return &container{system}, err
+}
+
+// GetContainers gets a list of the containers on the system that match the query
+func GetContainers(q ComputeSystemQuery) ([]ContainerProperties, error) {
+	return hcs.GetComputeSystems(q)
+}
+
+// Start synchronously starts the container.
+func (container *container) Start() error {
+	return convertSystemError(container.system.Start(), container)
+}
+
+// Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds.
+func (container *container) Shutdown() error {
+	return convertSystemError(container.system.Shutdown(), container)
+}
+
+// Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds.
+func (container *container) Terminate() error {
+	return convertSystemError(container.system.Terminate(), container)
+}
+
+// Waits synchronously waits for the container to shutdown or terminate.
+func (container *container) Wait() error {
+	return convertSystemError(container.system.Wait(), container)
+}
+
+// WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It
+// returns false if timeout occurs.
+func (container *container) WaitTimeout(t time.Duration) error {
+	return convertSystemError(container.system.WaitTimeout(t), container)
+}
+
+// Pause pauses the execution of a container.
+func (container *container) Pause() error {
+	return convertSystemError(container.system.Pause(), container)
+}
+
+// Resume resumes the execution of a container.
+func (container *container) Resume() error {
+	return convertSystemError(container.system.Resume(), container)
+}
+
+// HasPendingUpdates returns true if the container has updates pending to install
+func (container *container) HasPendingUpdates() (bool, error) {
+	return false, nil
+}
+
+// Statistics returns statistics for the container. This is a legacy v1 call
+func (container *container) Statistics() (Statistics, error) {
+	properties, err := container.system.Properties(schema1.PropertyTypeStatistics)
+	if err != nil {
+		return Statistics{}, convertSystemError(err, container)
+	}
+
+	return properties.Statistics, nil
+}
+
+// ProcessList returns an array of ProcessListItems for the container. This is a legacy v1 call
+func (container *container) ProcessList() ([]ProcessListItem, error) {
+	properties, err := container.system.Properties(schema1.PropertyTypeProcessList)
+	if err != nil {
+		return nil, convertSystemError(err, container)
+	}
+
+	return properties.ProcessList, nil
+}
+
+// This is a legacy v1 call
+func (container *container) MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) {
+	properties, err := container.system.Properties(schema1.PropertyTypeMappedVirtualDisk)
+	if err != nil {
+		return nil, convertSystemError(err, container)
+	}
+
+	return properties.MappedVirtualDiskControllers, nil
+}
+
+// CreateProcess launches a new process within the container.
+func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {
+	p, err := container.system.CreateProcess(c)
+	if err != nil {
+		return nil, convertSystemError(err, container)
+	}
+	return &process{p}, nil
+}
+
+// OpenProcess gets an interface to an existing process within the container.
+func (container *container) OpenProcess(pid int) (Process, error) {
+	p, err := container.system.OpenProcess(pid)
+	if err != nil {
+		return nil, convertSystemError(err, container)
+	}
+	return &process{p}, nil
+}
+
+// Close cleans up any state associated with the container but does not terminate or wait for it.
+func (container *container) Close() error {
+	return convertSystemError(container.system.Close(), container)
+}
+
+// Modify the System
+func (container *container) Modify(config *ResourceModificationRequestResponse) error {
+	return convertSystemError(container.system.Modify(config), container)
+}

vendor/github.com/Microsoft/hcsshim/errors.go

@@ -0,0 +1,257 @@
+package hcsshim
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/Microsoft/hcsshim/internal/hns"
+
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/hcserror"
+)
+
+var (
+	// ErrComputeSystemDoesNotExist is an error encountered when the container being operated on no longer exists = hcs.exist
+	ErrComputeSystemDoesNotExist = hcs.ErrComputeSystemDoesNotExist
+
+	// ErrElementNotFound is an error encountered when the object being referenced does not exist
+	ErrElementNotFound = hcs.ErrElementNotFound
+
+	// ErrElementNotFound is an error encountered when the object being referenced does not exist
+	ErrNotSupported = hcs.ErrNotSupported
+
+	// ErrInvalidData is an error encountered when the request being sent to hcs is invalid/unsupported
+	// decimal -2147024883 / hex 0x8007000d
+	ErrInvalidData = hcs.ErrInvalidData
+
+	// ErrHandleClose is an error encountered when the handle generating the notification being waited on has been closed
+	ErrHandleClose = hcs.ErrHandleClose
+
+	// ErrAlreadyClosed is an error encountered when using a handle that has been closed by the Close method
+	ErrAlreadyClosed = hcs.ErrAlreadyClosed
+
+	// ErrInvalidNotificationType is an error encountered when an invalid notification type is used
+	ErrInvalidNotificationType = hcs.ErrInvalidNotificationType
+
+	// ErrInvalidProcessState is an error encountered when the process is not in a valid state for the requested operation
+	ErrInvalidProcessState = hcs.ErrInvalidProcessState
+
+	// ErrTimeout is an error encountered when waiting on a notification times out
+	ErrTimeout = hcs.ErrTimeout
+
+	// ErrUnexpectedContainerExit is the error encountered when a container exits while waiting for
+	// a different expected notification
+	ErrUnexpectedContainerExit = hcs.ErrUnexpectedContainerExit
+
+	// ErrUnexpectedProcessAbort is the error encountered when communication with the compute service
+	// is lost while waiting for a notification
+	ErrUnexpectedProcessAbort = hcs.ErrUnexpectedProcessAbort
+
+	// ErrUnexpectedValue is an error encountered when hcs returns an invalid value
+	ErrUnexpectedValue = hcs.ErrUnexpectedValue
+
+	// ErrVmcomputeAlreadyStopped is an error encountered when a shutdown or terminate request is made on a stopped container
+	ErrVmcomputeAlreadyStopped = hcs.ErrVmcomputeAlreadyStopped
+
+	// ErrVmcomputeOperationPending is an error encountered when the operation is being completed asynchronously
+	ErrVmcomputeOperationPending = hcs.ErrVmcomputeOperationPending
+
+	// ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation
+	ErrVmcomputeOperationInvalidState = hcs.ErrVmcomputeOperationInvalidState
+
+	// ErrProcNotFound is an error encountered when the the process cannot be found
+	ErrProcNotFound = hcs.ErrProcNotFound
+
+	// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
+	// builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3.
+	ErrVmcomputeOperationAccessIsDenied = hcs.ErrVmcomputeOperationAccessIsDenied
+
+	// ErrVmcomputeInvalidJSON is an error encountered when the compute system does not support/understand the messages sent by management
+	ErrVmcomputeInvalidJSON = hcs.ErrVmcomputeInvalidJSON
+
+	// ErrVmcomputeUnknownMessage is an error encountered guest compute system doesn't support the message
+	ErrVmcomputeUnknownMessage = hcs.ErrVmcomputeUnknownMessage
+
+	// ErrNotSupported is an error encountered when hcs doesn't support the request
+	ErrPlatformNotSupported = hcs.ErrPlatformNotSupported
+)
+
+type EndpointNotFoundError = hns.EndpointNotFoundError
+type NetworkNotFoundError = hns.NetworkNotFoundError
+
+// ProcessError is an error encountered in HCS during an operation on a Process object
+type ProcessError struct {
+	Process   *process
+	Operation string
+	ExtraInfo string
+	Err       error
+	Events    []hcs.ErrorEvent
+}
+
+// ContainerError is an error encountered in HCS during an operation on a Container object
+type ContainerError struct {
+	Container *container
+	Operation string
+	ExtraInfo string
+	Err       error
+	Events    []hcs.ErrorEvent
+}
+
+func (e *ContainerError) Error() string {
+	if e == nil {
+		return "<nil>"
+	}
+
+	if e.Container == nil {
+		return "unexpected nil container for error: " + e.Err.Error()
+	}
+
+	s := "container " + e.Container.system.ID()
+
+	if e.Operation != "" {
+		s += " encountered an error during " + e.Operation
+	}
+
+	switch e.Err.(type) {
+	case nil:
+		break
+	case syscall.Errno:
+		s += fmt.Sprintf(": failure in a Windows system call: %s (0x%x)", e.Err, hcserror.Win32FromError(e.Err))
+	default:
+		s += fmt.Sprintf(": %s", e.Err.Error())
+	}
+
+	for _, ev := range e.Events {
+		s += "\n" + ev.String()
+	}
+
+	if e.ExtraInfo != "" {
+		s += " extra info: " + e.ExtraInfo
+	}
+
+	return s
+}
+
+func makeContainerError(container *container, operation string, extraInfo string, err error) error {
+	// Don't double wrap errors
+	if _, ok := err.(*ContainerError); ok {
+		return err
+	}
+	containerError := &ContainerError{Container: container, Operation: operation, ExtraInfo: extraInfo, Err: err}
+	return containerError
+}
+
+func (e *ProcessError) Error() string {
+	if e == nil {
+		return "<nil>"
+	}
+
+	if e.Process == nil {
+		return "Unexpected nil process for error: " + e.Err.Error()
+	}
+
+	s := fmt.Sprintf("process %d in container %s", e.Process.p.Pid(), e.Process.p.SystemID())
+	if e.Operation != "" {
+		s += " encountered an error during " + e.Operation
+	}
+
+	switch e.Err.(type) {
+	case nil:
+		break
+	case syscall.Errno:
+		s += fmt.Sprintf(": failure in a Windows system call: %s (0x%x)", e.Err, hcserror.Win32FromError(e.Err))
+	default:
+		s += fmt.Sprintf(": %s", e.Err.Error())
+	}
+
+	for _, ev := range e.Events {
+		s += "\n" + ev.String()
+	}
+
+	return s
+}
+
+func makeProcessError(process *process, operation string, extraInfo string, err error) error {
+	// Don't double wrap errors
+	if _, ok := err.(*ProcessError); ok {
+		return err
+	}
+	processError := &ProcessError{Process: process, Operation: operation, ExtraInfo: extraInfo, Err: err}
+	return processError
+}
+
+// IsNotExist checks if an error is caused by the Container or Process not existing.
+// Note: Currently, ErrElementNotFound can mean that a Process has either
+// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
+// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+func IsNotExist(err error) bool {
+	if _, ok := err.(EndpointNotFoundError); ok {
+		return true
+	}
+	if _, ok := err.(NetworkNotFoundError); ok {
+		return true
+	}
+	return hcs.IsNotExist(getInnerError(err))
+}
+
+// IsAlreadyClosed checks if an error is caused by the Container or Process having been
+// already closed by a call to the Close() method.
+func IsAlreadyClosed(err error) bool {
+	return hcs.IsAlreadyClosed(getInnerError(err))
+}
+
+// IsPending returns a boolean indicating whether the error is that
+// the requested operation is being completed in the background.
+func IsPending(err error) bool {
+	return hcs.IsPending(getInnerError(err))
+}
+
+// IsTimeout returns a boolean indicating whether the error is caused by
+// a timeout waiting for the operation to complete.
+func IsTimeout(err error) bool {
+	return hcs.IsTimeout(getInnerError(err))
+}
+
+// IsAlreadyStopped returns a boolean indicating whether the error is caused by
+// a Container or Process being already stopped.
+// Note: Currently, ErrElementNotFound can mean that a Process has either
+// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
+// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+func IsAlreadyStopped(err error) bool {
+	return hcs.IsAlreadyStopped(getInnerError(err))
+}
+
+// IsNotSupported returns a boolean indicating whether the error is caused by
+// unsupported platform requests
+// Note: Currently Unsupported platform requests can be mean either
+// ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage
+// is thrown from the Platform
+func IsNotSupported(err error) bool {
+	return hcs.IsNotSupported(getInnerError(err))
+}
+
+func getInnerError(err error) error {
+	switch pe := err.(type) {
+	case nil:
+		return nil
+	case *ContainerError:
+		err = pe.Err
+	case *ProcessError:
+		err = pe.Err
+	}
+	return err
+}
+
+func convertSystemError(err error, c *container) error {
+	if serr, ok := err.(*hcs.SystemError); ok {
+		return &ContainerError{Container: c, Operation: serr.Op, ExtraInfo: serr.Extra, Err: serr.Err, Events: serr.Events}
+	}
+	return err
+}
+
+func convertProcessError(err error, p *process) error {
+	if perr, ok := err.(*hcs.ProcessError); ok {
+		return &ProcessError{Process: p, Operation: perr.Op, Err: perr.Err, Events: perr.Events}
+	}
+	return err
+}

vendor/github.com/Microsoft/hcsshim/hcsshim.go

@@ -0,0 +1,28 @@
+// Shim for the Host Compute Service (HCS) to manage Windows Server
+// containers and Hyper-V containers.
+
+package hcsshim
+
+import (
+	"syscall"
+
+	"github.com/Microsoft/hcsshim/internal/hcserror"
+)
+
+//go:generate go run mksyscall_windows.go -output zsyscall_windows.go hcsshim.go
+
+//sys SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) = iphlpapi.SetCurrentThreadCompartmentId
+
+const (
+	// Specific user-visible exit codes
+	WaitErrExecFailed = 32767
+
+	ERROR_GEN_FAILURE          = hcserror.ERROR_GEN_FAILURE
+	ERROR_SHUTDOWN_IN_PROGRESS = syscall.Errno(1115)
+	WSAEINVAL                  = syscall.Errno(10022)
+
+	// Timeout on wait calls
+	TimeoutInfinite = 0xFFFFFFFF
+)
+
+type HcsError = hcserror.HcsError

vendor/github.com/Microsoft/hcsshim/hnsendpoint.go

@@ -0,0 +1,94 @@
+package hcsshim
+
+import (
+	"github.com/Microsoft/hcsshim/internal/hns"
+)
+
+// HNSEndpoint represents a network endpoint in HNS
+type HNSEndpoint = hns.HNSEndpoint
+
+// Namespace represents a Compartment.
+type Namespace = hns.Namespace
+
+//SystemType represents the type of the system on which actions are done
+type SystemType string
+
+// SystemType const
+const (
+	ContainerType      SystemType = "Container"
+	VirtualMachineType SystemType = "VirtualMachine"
+	HostType           SystemType = "Host"
+)
+
+// EndpointAttachDetachRequest is the structure used to send request to the container to modify the system
+// Supported resource types are Network and Request Types are Add/Remove
+type EndpointAttachDetachRequest = hns.EndpointAttachDetachRequest
+
+// EndpointResquestResponse is object to get the endpoint request response
+type EndpointResquestResponse = hns.EndpointResquestResponse
+
+// HNSEndpointRequest makes a HNS call to modify/query a network endpoint
+func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) {
+	return hns.HNSEndpointRequest(method, path, request)
+}
+
+// HNSListEndpointRequest makes a HNS call to query the list of available endpoints
+func HNSListEndpointRequest() ([]HNSEndpoint, error) {
+	return hns.HNSListEndpointRequest()
+}
+
+// HotAttachEndpoint makes a HCS Call to attach the endpoint to the container
+func HotAttachEndpoint(containerID string, endpointID string) error {
+	return modifyNetworkEndpoint(containerID, endpointID, Add)
+}
+
+// HotDetachEndpoint makes a HCS Call to detach the endpoint from the container
+func HotDetachEndpoint(containerID string, endpointID string) error {
+	return modifyNetworkEndpoint(containerID, endpointID, Remove)
+}
+
+// ModifyContainer corresponding to the container id, by sending a request
+func modifyContainer(id string, request *ResourceModificationRequestResponse) error {
+	container, err := OpenContainer(id)
+	if err != nil {
+		if IsNotExist(err) {
+			return ErrComputeSystemDoesNotExist
+		}
+		return getInnerError(err)
+	}
+	defer container.Close()
+	err = container.Modify(request)
+	if err != nil {
+		if IsNotSupported(err) {
+			return ErrPlatformNotSupported
+		}
+		return getInnerError(err)
+	}
+
+	return nil
+}
+
+func modifyNetworkEndpoint(containerID string, endpointID string, request RequestType) error {
+	requestMessage := &ResourceModificationRequestResponse{
+		Resource: Network,
+		Request:  request,
+		Data:     endpointID,
+	}
+	err := modifyContainer(containerID, requestMessage)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// GetHNSEndpointByID get the Endpoint by ID
+func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) {
+	return hns.GetHNSEndpointByID(endpointID)
+}
+
+// GetHNSEndpointByName gets the endpoint filtered by Name
+func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) {
+	return hns.GetHNSEndpointByName(endpointName)
+}

vendor/github.com/Microsoft/hcsshim/hnsglobals.go

@@ -0,0 +1,16 @@
+package hcsshim
+
+import (
+	"github.com/Microsoft/hcsshim/internal/hns"
+)
+
+type HNSGlobals = hns.HNSGlobals
+type HNSVersion = hns.HNSVersion
+
+var (
+	HNSVersion1803 = hns.HNSVersion1803
+)
+
+func GetHNSGlobals() (*HNSGlobals, error) {
+	return hns.GetHNSGlobals()
+}

vendor/github.com/Microsoft/hcsshim/hnsnetwork.go

@@ -0,0 +1,36 @@
+package hcsshim
+
+import (
+	"github.com/Microsoft/hcsshim/internal/hns"
+)
+
+// Subnet is assoicated with a network and represents a list
+// of subnets available to the network
+type Subnet = hns.Subnet
+
+// MacPool is assoicated with a network and represents a list
+// of macaddresses available to the network
+type MacPool = hns.MacPool
+
+// HNSNetwork represents a network in HNS
+type HNSNetwork = hns.HNSNetwork
+
+// HNSNetworkRequest makes a call into HNS to update/query a single network
+func HNSNetworkRequest(method, path, request string) (*HNSNetwork, error) {
+	return hns.HNSNetworkRequest(method, path, request)
+}
+
+// HNSListNetworkRequest makes a HNS call to query the list of available networks
+func HNSListNetworkRequest(method, path, request string) ([]HNSNetwork, error) {
+	return hns.HNSListNetworkRequest(method, path, request)
+}
+
+// GetHNSNetworkByID
+func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) {
+	return hns.GetHNSNetworkByID(networkID)
+}
+
+// GetHNSNetworkName filtered by Name
+func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) {
+	return hns.GetHNSNetworkByName(networkName)
+}

vendor/github.com/Microsoft/hcsshim/hnspolicy.go

@@ -0,0 +1,57 @@
+package hcsshim
+
+import (
+	"github.com/Microsoft/hcsshim/internal/hns"
+)
+
+// Type of Request Support in ModifySystem
+type PolicyType = hns.PolicyType
+
+// RequestType const
+const (
+	Nat                  = hns.Nat
+	ACL                  = hns.ACL
+	PA                   = hns.PA
+	VLAN                 = hns.VLAN
+	VSID                 = hns.VSID
+	VNet                 = hns.VNet
+	L2Driver             = hns.L2Driver
+	Isolation            = hns.Isolation
+	QOS                  = hns.QOS
+	OutboundNat          = hns.OutboundNat
+	ExternalLoadBalancer = hns.ExternalLoadBalancer
+	Route                = hns.Route
+)
+
+type NatPolicy = hns.NatPolicy
+
+type QosPolicy = hns.QosPolicy
+
+type IsolationPolicy = hns.IsolationPolicy
+
+type VlanPolicy = hns.VlanPolicy
+
+type VsidPolicy = hns.VsidPolicy
+
+type PaPolicy = hns.PaPolicy
+
+type OutboundNatPolicy = hns.OutboundNatPolicy
+
+type ActionType = hns.ActionType
+type DirectionType = hns.DirectionType
+type RuleType = hns.RuleType
+
+const (
+	Allow = hns.Allow
+	Block = hns.Block
+
+	In  = hns.In
+	Out = hns.Out
+
+	Host   = hns.Host
+	Switch = hns.Switch
+)
+
+type ACLPolicy = hns.ACLPolicy
+
+type Policy = hns.Policy

vendor/github.com/Microsoft/hcsshim/hnspolicylist.go

@@ -0,0 +1,47 @@
+package hcsshim
+
+import (
+	"github.com/Microsoft/hcsshim/internal/hns"
+)
+
+// RoutePolicy is a structure defining schema for Route based Policy
+type RoutePolicy = hns.RoutePolicy
+
+// ELBPolicy is a structure defining schema for ELB LoadBalancing based Policy
+type ELBPolicy = hns.ELBPolicy
+
+// LBPolicy is a structure defining schema for LoadBalancing based Policy
+type LBPolicy = hns.LBPolicy
+
+// PolicyList is a structure defining schema for Policy list request
+type PolicyList = hns.PolicyList
+
+// HNSPolicyListRequest makes a call into HNS to update/query a single network
+func HNSPolicyListRequest(method, path, request string) (*PolicyList, error) {
+	return hns.HNSPolicyListRequest(method, path, request)
+}
+
+// HNSListPolicyListRequest gets all the policy list
+func HNSListPolicyListRequest() ([]PolicyList, error) {
+	return hns.HNSListPolicyListRequest()
+}
+
+// PolicyListRequest makes a HNS call to modify/query a network policy list
+func PolicyListRequest(method, path, request string) (*PolicyList, error) {
+	return hns.PolicyListRequest(method, path, request)
+}
+
+// GetPolicyListByID get the policy list by ID
+func GetPolicyListByID(policyListID string) (*PolicyList, error) {
+	return hns.GetPolicyListByID(policyListID)
+}
+
+// AddLoadBalancer policy list for the specified endpoints
+func AddLoadBalancer(endpoints []HNSEndpoint, isILB bool, isDSR bool, sourceVIP, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*PolicyList, error) {
+	return hns.AddLoadBalancer(endpoints, isILB, isDSR, sourceVIP, vip, protocol, internalPort, externalPort)
+}
+
+// AddRoute adds route policy list for the specified endpoints
+func AddRoute(endpoints []HNSEndpoint, destinationPrefix string, nextHop string, encapEnabled bool) (*PolicyList, error) {
+	return hns.AddRoute(endpoints, destinationPrefix, nextHop, encapEnabled)
+}

vendor/github.com/Microsoft/hcsshim/hnssupport.go

@@ -0,0 +1,13 @@
+package hcsshim
+
+import (
+	"github.com/Microsoft/hcsshim/internal/hns"
+)
+
+type HNSSupportedFeatures = hns.HNSSupportedFeatures
+
+type HNSAclFeatures = hns.HNSAclFeatures
+
+func GetHNSSupportedFeatures() HNSSupportedFeatures {
+	return hns.GetHNSSupportedFeatures()
+}

vendor/github.com/Microsoft/hcsshim/interface.go

@@ -0,0 +1,114 @@
+package hcsshim
+
+import (
+	"io"
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/schema1"
+)
+
+// ProcessConfig is used as both the input of Container.CreateProcess
+// and to convert the parameters to JSON for passing onto the HCS
+type ProcessConfig = schema1.ProcessConfig
+
+type Layer = schema1.Layer
+type MappedDir = schema1.MappedDir
+type MappedPipe = schema1.MappedPipe
+type HvRuntime = schema1.HvRuntime
+type MappedVirtualDisk = schema1.MappedVirtualDisk
+
+// AssignedDevice represents a device that has been directly assigned to a container
+//
+// NOTE: Support added in RS5
+type AssignedDevice = schema1.AssignedDevice
+
+// ContainerConfig is used as both the input of CreateContainer
+// and to convert the parameters to JSON for passing onto the HCS
+type ContainerConfig = schema1.ContainerConfig
+
+type ComputeSystemQuery = schema1.ComputeSystemQuery
+
+// Container represents a created (but not necessarily running) container.
+type Container interface {
+	// Start synchronously starts the container.
+	Start() error
+
+	// Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds.
+	Shutdown() error
+
+	// Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds.
+	Terminate() error
+
+	// Waits synchronously waits for the container to shutdown or terminate.
+	Wait() error
+
+	// WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It
+	// returns false if timeout occurs.
+	WaitTimeout(time.Duration) error
+
+	// Pause pauses the execution of a container.
+	Pause() error
+
+	// Resume resumes the execution of a container.
+	Resume() error
+
+	// HasPendingUpdates returns true if the container has updates pending to install.
+	HasPendingUpdates() (bool, error)
+
+	// Statistics returns statistics for a container.
+	Statistics() (Statistics, error)
+
+	// ProcessList returns details for the processes in a container.
+	ProcessList() ([]ProcessListItem, error)
+
+	// MappedVirtualDisks returns virtual disks mapped to a utility VM, indexed by controller
+	MappedVirtualDisks() (map[int]MappedVirtualDiskController, error)
+
+	// CreateProcess launches a new process within the container.
+	CreateProcess(c *ProcessConfig) (Process, error)
+
+	// OpenProcess gets an interface to an existing process within the container.
+	OpenProcess(pid int) (Process, error)
+
+	// Close cleans up any state associated with the container but does not terminate or wait for it.
+	Close() error
+
+	// Modify the System
+	Modify(config *ResourceModificationRequestResponse) error
+}
+
+// Process represents a running or exited process.
+type Process interface {
+	// Pid returns the process ID of the process within the container.
+	Pid() int
+
+	// Kill signals the process to terminate but does not wait for it to finish terminating.
+	Kill() error
+
+	// Wait waits for the process to exit.
+	Wait() error
+
+	// WaitTimeout waits for the process to exit or the duration to elapse. It returns
+	// false if timeout occurs.
+	WaitTimeout(time.Duration) error
+
+	// ExitCode returns the exit code of the process. The process must have
+	// already terminated.
+	ExitCode() (int, error)
+
+	// ResizeConsole resizes the console of the process.
+	ResizeConsole(width, height uint16) error
+
+	// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing
+	// these pipes does not close the underlying pipes; it should be possible to
+	// call this multiple times to get multiple interfaces.
+	Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error)
+
+	// CloseStdin closes the write side of the stdin pipe so that the process is
+	// notified on the read side that there is no more data in stdin.
+	CloseStdin() error
+
+	// Close cleans up any state associated with the process but does not kill
+	// or wait on it.
+	Close() error
+}

vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go

@@ -0,0 +1,85 @@
+package guestrequest
+
+import "github.com/Microsoft/hcsshim/internal/schema2"
+
+// Arguably, many of these (at least CombinedLayers) should have been generated
+// by swagger.
+//
+// This will also change package name due to an inbound breaking change.
+
+// This class is used by a modify request to add or remove a combined layers
+// structure in the guest. For windows, the GCS applies a filter in ContainerRootPath
+// using the specified layers as the parent content. Ignores property ScratchPath
+// since the container path is already the scratch path. For linux, the GCS unions
+// the specified layers and ScratchPath together, placing the resulting union
+// filesystem at ContainerRootPath.
+type CombinedLayers struct {
+	ContainerRootPath string            `json:"ContainerRootPath,omitempty"`
+	Layers            []hcsschema.Layer `json:"Layers,omitempty"`
+	ScratchPath       string            `json:"ScratchPath,omitempty"`
+}
+
+// Defines the schema for hosted settings passed to GCS and/or OpenGCS
+
+// SCSI. Scratch space for remote file-system commands, or R/W layer for containers
+type LCOWMappedVirtualDisk struct {
+	MountPath  string `json:"MountPath,omitempty"` // /tmp/scratch for an LCOW utility VM being used as a service VM
+	Lun        uint8  `json:"Lun,omitempty"`
+	Controller uint8  `json:"Controller,omitempty"`
+	ReadOnly   bool   `json:"ReadOnly,omitempty"`
+}
+
+type WCOWMappedVirtualDisk struct {
+	ContainerPath string `json:"ContainerPath,omitempty"`
+	Lun           int32  `json:"Lun,omitempty"`
+}
+
+type LCOWMappedDirectory struct {
+	MountPath string `json:"MountPath,omitempty"`
+	Port      int32  `json:"Port,omitempty"`
+	ShareName string `json:"ShareName,omitempty"` // If empty not using ANames (not currently supported)
+	ReadOnly  bool   `json:"ReadOnly,omitempty"`
+}
+
+// Read-only layers over VPMem
+type LCOWMappedVPMemDevice struct {
+	DeviceNumber uint32 `json:"DeviceNumber,omitempty"`
+	MountPath    string `json:"MountPath,omitempty"` // /tmp/pN
+}
+
+type ResourceType string
+
+const (
+	// These are constants for v2 schema modify guest requests.
+	ResourceTypeMappedDirectory   ResourceType = "MappedDirectory"
+	ResourceTypeMappedVirtualDisk ResourceType = "MappedVirtualDisk"
+	ResourceTypeNetwork           ResourceType = "Network"
+	ResourceTypeNetworkNamespace  ResourceType = "NetworkNamespace"
+	ResourceTypeCombinedLayers    ResourceType = "CombinedLayers"
+	ResourceTypeVPMemDevice       ResourceType = "VPMemDevice"
+)
+
+// GuestRequest is for modify commands passed to the guest.
+type GuestRequest struct {
+	RequestType  string       `json:"RequestType,omitempty"`
+	ResourceType ResourceType `json:"ResourceType,omitempty"`
+	Settings     interface{}  `json:"Settings,omitempty"`
+}
+
+type NetworkModifyRequest struct {
+	AdapterId   string      `json:"AdapterId,omitempty"`
+	RequestType string      `json:"RequestType,omitempty"`
+	Settings    interface{} `json:"Settings,omitempty"`
+}
+
+type RS4NetworkModifyRequest struct {
+	AdapterInstanceId string      `json:"AdapterInstanceId,omitempty"`
+	RequestType       string      `json:"RequestType,omitempty"`
+	Settings          interface{} `json:"Settings,omitempty"`
+}
+
+// SignalProcessOptions is the options passed to either WCOW or LCOW
+// to signal a given process.
+type SignalProcessOptions struct {
+	Signal int `json:,omitempty`
+}

vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go

@@ -0,0 +1,69 @@
+package guid
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+var _ = (json.Marshaler)(&GUID{})
+var _ = (json.Unmarshaler)(&GUID{})
+
+type GUID [16]byte
+
+func New() GUID {
+	g := GUID{}
+	_, err := io.ReadFull(rand.Reader, g[:])
+	if err != nil {
+		panic(err)
+	}
+	return g
+}
+
+func (g GUID) String() string {
+	return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:])
+}
+
+func FromString(s string) GUID {
+	if len(s) != 36 {
+		panic(fmt.Sprintf("invalid GUID length: %d", len(s)))
+	}
+	if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' {
+		panic("invalid GUID format")
+	}
+	indexOrder := [16]int{
+		0, 2, 4, 6,
+		9, 11,
+		14, 16,
+		19, 21,
+		24, 26, 28, 30, 32, 34,
+	}
+	byteOrder := [16]int{
+		3, 2, 1, 0,
+		5, 4,
+		7, 6,
+		8, 9,
+		10, 11, 12, 13, 14, 15,
+	}
+	var g GUID
+	for i, x := range indexOrder {
+		b, err := strconv.ParseInt(s[x:x+2], 16, 16)
+		if err != nil {
+			panic(err)
+		}
+		g[byteOrder[i]] = byte(b)
+	}
+	return g
+}
+
+func (g GUID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(g.String())
+}
+
+func (g *GUID) UnmarshalJSON(data []byte) error {
+	*g = FromString(strings.Trim(string(data), "\""))
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go

@@ -0,0 +1,81 @@
+package hcs
+
+import (
+	"sync"
+	"syscall"
+
+	"github.com/Microsoft/hcsshim/internal/interop"
+)
+
+var (
+	nextCallback    uintptr
+	callbackMap     = map[uintptr]*notifcationWatcherContext{}
+	callbackMapLock = sync.RWMutex{}
+
+	notificationWatcherCallback = syscall.NewCallback(notificationWatcher)
+
+	// Notifications for HCS_SYSTEM handles
+	hcsNotificationSystemExited          hcsNotification = 0x00000001
+	hcsNotificationSystemCreateCompleted hcsNotification = 0x00000002
+	hcsNotificationSystemStartCompleted  hcsNotification = 0x00000003
+	hcsNotificationSystemPauseCompleted  hcsNotification = 0x00000004
+	hcsNotificationSystemResumeCompleted hcsNotification = 0x00000005
+
+	// Notifications for HCS_PROCESS handles
+	hcsNotificationProcessExited hcsNotification = 0x00010000
+
+	// Common notifications
+	hcsNotificationInvalid           hcsNotification = 0x00000000
+	hcsNotificationServiceDisconnect hcsNotification = 0x01000000
+)
+
+type hcsNotification uint32
+type notificationChannel chan error
+
+type notifcationWatcherContext struct {
+	channels notificationChannels
+	handle   hcsCallback
+}
+
+type notificationChannels map[hcsNotification]notificationChannel
+
+func newChannels() notificationChannels {
+	channels := make(notificationChannels)
+
+	channels[hcsNotificationSystemExited] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemCreateCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemStartCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemPauseCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationSystemResumeCompleted] = make(notificationChannel, 1)
+	channels[hcsNotificationProcessExited] = make(notificationChannel, 1)
+	channels[hcsNotificationServiceDisconnect] = make(notificationChannel, 1)
+	return channels
+}
+func closeChannels(channels notificationChannels) {
+	close(channels[hcsNotificationSystemExited])
+	close(channels[hcsNotificationSystemCreateCompleted])
+	close(channels[hcsNotificationSystemStartCompleted])
+	close(channels[hcsNotificationSystemPauseCompleted])
+	close(channels[hcsNotificationSystemResumeCompleted])
+	close(channels[hcsNotificationProcessExited])
+	close(channels[hcsNotificationServiceDisconnect])
+}
+
+func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr {
+	var result error
+	if int32(notificationStatus) < 0 {
+		result = interop.Win32FromHresult(notificationStatus)
+	}
+
+	callbackMapLock.RLock()
+	context := callbackMap[callbackNumber]
+	callbackMapLock.RUnlock()
+
+	if context == nil {
+		return 0
+	}
+
+	context.channels[notificationType] <- result
+
+	return 0
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/cgo.go

@@ -0,0 +1,7 @@
+package hcs
+
+import "C"
+
+// This import is needed to make the library compile as CGO because HCSSHIM
+// only works with CGO due to callbacks from HCS comming back from a C thread
+// which is not supported without CGO. See https://github.com/golang/go/issues/10973

vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go

@@ -0,0 +1,279 @@
+package hcs
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"syscall"
+
+	"github.com/Microsoft/hcsshim/internal/interop"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	// ErrComputeSystemDoesNotExist is an error encountered when the container being operated on no longer exists
+	ErrComputeSystemDoesNotExist = syscall.Errno(0xc037010e)
+
+	// ErrElementNotFound is an error encountered when the object being referenced does not exist
+	ErrElementNotFound = syscall.Errno(0x490)
+
+	// ErrElementNotFound is an error encountered when the object being referenced does not exist
+	ErrNotSupported = syscall.Errno(0x32)
+
+	// ErrInvalidData is an error encountered when the request being sent to hcs is invalid/unsupported
+	// decimal -2147024883 / hex 0x8007000d
+	ErrInvalidData = syscall.Errno(0xd)
+
+	// ErrHandleClose is an error encountered when the handle generating the notification being waited on has been closed
+	ErrHandleClose = errors.New("hcsshim: the handle generating this notification has been closed")
+
+	// ErrAlreadyClosed is an error encountered when using a handle that has been closed by the Close method
+	ErrAlreadyClosed = errors.New("hcsshim: the handle has already been closed")
+
+	// ErrInvalidNotificationType is an error encountered when an invalid notification type is used
+	ErrInvalidNotificationType = errors.New("hcsshim: invalid notification type")
+
+	// ErrInvalidProcessState is an error encountered when the process is not in a valid state for the requested operation
+	ErrInvalidProcessState = errors.New("the process is in an invalid state for the attempted operation")
+
+	// ErrTimeout is an error encountered when waiting on a notification times out
+	ErrTimeout = errors.New("hcsshim: timeout waiting for notification")
+
+	// ErrUnexpectedContainerExit is the error encountered when a container exits while waiting for
+	// a different expected notification
+	ErrUnexpectedContainerExit = errors.New("unexpected container exit")
+
+	// ErrUnexpectedProcessAbort is the error encountered when communication with the compute service
+	// is lost while waiting for a notification
+	ErrUnexpectedProcessAbort = errors.New("lost communication with compute service")
+
+	// ErrUnexpectedValue is an error encountered when hcs returns an invalid value
+	ErrUnexpectedValue = errors.New("unexpected value returned from hcs")
+
+	// ErrVmcomputeAlreadyStopped is an error encountered when a shutdown or terminate request is made on a stopped container
+	ErrVmcomputeAlreadyStopped = syscall.Errno(0xc0370110)
+
+	// ErrVmcomputeOperationPending is an error encountered when the operation is being completed asynchronously
+	ErrVmcomputeOperationPending = syscall.Errno(0xC0370103)
+
+	// ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation
+	ErrVmcomputeOperationInvalidState = syscall.Errno(0xc0370105)
+
+	// ErrProcNotFound is an error encountered when the the process cannot be found
+	ErrProcNotFound = syscall.Errno(0x7f)
+
+	// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
+	// builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3.
+	ErrVmcomputeOperationAccessIsDenied = syscall.Errno(0x5)
+
+	// ErrVmcomputeInvalidJSON is an error encountered when the compute system does not support/understand the messages sent by management
+	ErrVmcomputeInvalidJSON = syscall.Errno(0xc037010d)
+
+	// ErrVmcomputeUnknownMessage is an error encountered guest compute system doesn't support the message
+	ErrVmcomputeUnknownMessage = syscall.Errno(0xc037010b)
+
+	// ErrNotSupported is an error encountered when hcs doesn't support the request
+	ErrPlatformNotSupported = errors.New("unsupported platform request")
+)
+
+type ErrorEvent struct {
+	Message    string `json:"Message,omitempty"`    // Fully formated error message
+	StackTrace string `json:"StackTrace,omitempty"` // Stack trace in string form
+	Provider   string `json:"Provider,omitempty"`
+	EventID    uint16 `json:"EventId,omitempty"`
+	Flags      uint32 `json:"Flags,omitempty"`
+	Source     string `json:"Source,omitempty"`
+	//Data       []EventData `json:"Data,omitempty"`  // Omit this as HCS doesn't encode this well. It's more confusing to include. It is however logged in debug mode (see processHcsResult function)
+}
+
+type hcsResult struct {
+	Error        int32
+	ErrorMessage string
+	ErrorEvents  []ErrorEvent `json:"ErrorEvents,omitempty"`
+}
+
+func (ev *ErrorEvent) String() string {
+	evs := "[Event Detail: " + ev.Message
+	if ev.StackTrace != "" {
+		evs += " Stack Trace: " + ev.StackTrace
+	}
+	if ev.Provider != "" {
+		evs += " Provider: " + ev.Provider
+	}
+	if ev.EventID != 0 {
+		evs = fmt.Sprintf("%s EventID: %d", evs, ev.EventID)
+	}
+	if ev.Flags != 0 {
+		evs = fmt.Sprintf("%s flags: %d", evs, ev.Flags)
+	}
+	if ev.Source != "" {
+		evs += " Source: " + ev.Source
+	}
+	evs += "]"
+	return evs
+}
+
+func processHcsResult(resultp *uint16) []ErrorEvent {
+	if resultp != nil {
+		resultj := interop.ConvertAndFreeCoTaskMemString(resultp)
+		logrus.Debugf("Result: %s", resultj)
+		result := &hcsResult{}
+		if err := json.Unmarshal([]byte(resultj), result); err != nil {
+			logrus.Warnf("Could not unmarshal HCS result %s: %s", resultj, err)
+			return nil
+		}
+		return result.ErrorEvents
+	}
+	return nil
+}
+
+type HcsError struct {
+	Op     string
+	Err    error
+	Events []ErrorEvent
+}
+
+func (e *HcsError) Error() string {
+	s := e.Op + ": " + e.Err.Error()
+	for _, ev := range e.Events {
+		s += "\n" + ev.String()
+	}
+	return s
+}
+
+// ProcessError is an error encountered in HCS during an operation on a Process object
+type ProcessError struct {
+	SystemID string
+	Pid      int
+	Op       string
+	Err      error
+	Events   []ErrorEvent
+}
+
+// SystemError is an error encountered in HCS during an operation on a Container object
+type SystemError struct {
+	ID     string
+	Op     string
+	Err    error
+	Extra  string
+	Events []ErrorEvent
+}
+
+func (e *SystemError) Error() string {
+	s := e.Op + " " + e.ID + ": " + e.Err.Error()
+	for _, ev := range e.Events {
+		s += "\n" + ev.String()
+	}
+	if e.Extra != "" {
+		s += "\n(extra info: " + e.Extra + ")"
+	}
+	return s
+}
+
+func makeSystemError(system *System, op string, extra string, err error, events []ErrorEvent) error {
+	// Don't double wrap errors
+	if _, ok := err.(*SystemError); ok {
+		return err
+	}
+	return &SystemError{
+		ID:     system.ID(),
+		Op:     op,
+		Extra:  extra,
+		Err:    err,
+		Events: events,
+	}
+}
+
+func (e *ProcessError) Error() string {
+	s := fmt.Sprintf("%s %s:%d: %s", e.Op, e.SystemID, e.Pid, e.Err.Error())
+	for _, ev := range e.Events {
+		s += "\n" + ev.String()
+	}
+	return s
+}
+
+func makeProcessError(process *Process, op string, err error, events []ErrorEvent) error {
+	// Don't double wrap errors
+	if _, ok := err.(*ProcessError); ok {
+		return err
+	}
+	return &ProcessError{
+		Pid:      process.Pid(),
+		SystemID: process.SystemID(),
+		Op:       op,
+		Err:      err,
+		Events:   events,
+	}
+}
+
+// IsNotExist checks if an error is caused by the Container or Process not existing.
+// Note: Currently, ErrElementNotFound can mean that a Process has either
+// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
+// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+func IsNotExist(err error) bool {
+	err = getInnerError(err)
+	return err == ErrComputeSystemDoesNotExist ||
+		err == ErrElementNotFound ||
+		err == ErrProcNotFound
+}
+
+// IsAlreadyClosed checks if an error is caused by the Container or Process having been
+// already closed by a call to the Close() method.
+func IsAlreadyClosed(err error) bool {
+	err = getInnerError(err)
+	return err == ErrAlreadyClosed
+}
+
+// IsPending returns a boolean indicating whether the error is that
+// the requested operation is being completed in the background.
+func IsPending(err error) bool {
+	err = getInnerError(err)
+	return err == ErrVmcomputeOperationPending
+}
+
+// IsTimeout returns a boolean indicating whether the error is caused by
+// a timeout waiting for the operation to complete.
+func IsTimeout(err error) bool {
+	err = getInnerError(err)
+	return err == ErrTimeout
+}
+
+// IsAlreadyStopped returns a boolean indicating whether the error is caused by
+// a Container or Process being already stopped.
+// Note: Currently, ErrElementNotFound can mean that a Process has either
+// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
+// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
+func IsAlreadyStopped(err error) bool {
+	err = getInnerError(err)
+	return err == ErrVmcomputeAlreadyStopped ||
+		err == ErrElementNotFound ||
+		err == ErrProcNotFound
+}
+
+// IsNotSupported returns a boolean indicating whether the error is caused by
+// unsupported platform requests
+// Note: Currently Unsupported platform requests can be mean either
+// ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage
+// is thrown from the Platform
+func IsNotSupported(err error) bool {
+	err = getInnerError(err)
+	// If Platform doesn't recognize or support the request sent, below errors are seen
+	return err == ErrVmcomputeInvalidJSON ||
+		err == ErrInvalidData ||
+		err == ErrNotSupported ||
+		err == ErrVmcomputeUnknownMessage
+}
+
+func getInnerError(err error) error {
+	switch pe := err.(type) {
+	case nil:
+		return nil
+	case *HcsError:
+		err = pe.Err
+	case *SystemError:
+		err = pe.Err
+	case *ProcessError:
+		err = pe.Err
+	}
+	return err
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go

@@ -0,0 +1,48 @@
+// Shim for the Host Compute Service (HCS) to manage Windows Server
+// containers and Hyper-V containers.
+
+package hcs
+
+import (
+	"syscall"
+)
+
+//go:generate go run ../../mksyscall_windows.go -output zsyscall_windows.go hcs.go
+
+//sys hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) = vmcompute.HcsEnumerateComputeSystems?
+//sys hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsCreateComputeSystem?
+//sys hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsOpenComputeSystem?
+//sys hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) = vmcompute.HcsCloseComputeSystem?
+//sys hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsStartComputeSystem?
+//sys hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsShutdownComputeSystem?
+//sys hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsTerminateComputeSystem?
+//sys hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsPauseComputeSystem?
+//sys hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsResumeComputeSystem?
+//sys hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetComputeSystemProperties?
+//sys hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) = vmcompute.HcsModifyComputeSystem?
+//sys hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterComputeSystemCallback?
+//sys hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterComputeSystemCallback?
+
+//sys hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsCreateProcess?
+//sys hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsOpenProcess?
+//sys hcsCloseProcess(process hcsProcess) (hr error) = vmcompute.HcsCloseProcess?
+//sys hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) = vmcompute.HcsTerminateProcess?
+//sys hcsSignalProcess(process hcsProcess, options string, result **uint16) (hr error) = vmcompute.HcsTerminateProcess?
+//sys hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) = vmcompute.HcsGetProcessInfo?
+//sys hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) = vmcompute.HcsGetProcessProperties?
+//sys hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) = vmcompute.HcsModifyProcess?
+//sys hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetServiceProperties?
+//sys hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterProcessCallback?
+//sys hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterProcessCallback?
+
+type hcsSystem syscall.Handle
+type hcsProcess syscall.Handle
+type hcsCallback syscall.Handle
+
+type hcsProcessInformation struct {
+	ProcessId uint32
+	Reserved  uint32
+	StdInput  syscall.Handle
+	StdOutput syscall.Handle
+	StdError  syscall.Handle
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go

@@ -0,0 +1,427 @@
+package hcs
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/guestrequest"
+	"github.com/Microsoft/hcsshim/internal/interop"
+	"github.com/sirupsen/logrus"
+)
+
+// ContainerError is an error encountered in HCS
+type Process struct {
+	handleLock     sync.RWMutex
+	handle         hcsProcess
+	processID      int
+	system         *System
+	cachedPipes    *cachedPipes
+	callbackNumber uintptr
+}
+
+type cachedPipes struct {
+	stdIn  syscall.Handle
+	stdOut syscall.Handle
+	stdErr syscall.Handle
+}
+
+type processModifyRequest struct {
+	Operation   string
+	ConsoleSize *consoleSize `json:",omitempty"`
+	CloseHandle *closeHandle `json:",omitempty"`
+}
+
+type consoleSize struct {
+	Height uint16
+	Width  uint16
+}
+
+type closeHandle struct {
+	Handle string
+}
+
+type ProcessStatus struct {
+	ProcessID      uint32
+	Exited         bool
+	ExitCode       uint32
+	LastWaitResult int32
+}
+
+const (
+	stdIn  string = "StdIn"
+	stdOut string = "StdOut"
+	stdErr string = "StdErr"
+)
+
+const (
+	modifyConsoleSize string = "ConsoleSize"
+	modifyCloseHandle string = "CloseHandle"
+)
+
+// Pid returns the process ID of the process within the container.
+func (process *Process) Pid() int {
+	return process.processID
+}
+
+// SystemID returns the ID of the process's compute system.
+func (process *Process) SystemID() string {
+	return process.system.ID()
+}
+
+// Signal signals the process with `options`.
+func (process *Process) Signal(options guestrequest.SignalProcessOptions) error {
+	process.handleLock.RLock()
+	defer process.handleLock.RUnlock()
+	operation := "Signal"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	if process.handle == 0 {
+		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
+	}
+
+	optionsb, err := json.Marshal(options)
+	if err != nil {
+		return err
+	}
+
+	optionsStr := string(optionsb)
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("SignalProcess %s: %d", process.SystemID(), process.Pid()), &completed)
+	err = hcsSignalProcess(process.handle, optionsStr, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeProcessError(process, operation, err, events)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+// Kill signals the process to terminate but does not wait for it to finish terminating.
+func (process *Process) Kill() error {
+	process.handleLock.RLock()
+	defer process.handleLock.RUnlock()
+	operation := "Kill"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	if process.handle == 0 {
+		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
+	}
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("TerminateProcess %s: %d", process.SystemID(), process.Pid()), &completed)
+	err := hcsTerminateProcess(process.handle, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeProcessError(process, operation, err, events)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+// Wait waits for the process to exit.
+func (process *Process) Wait() error {
+	operation := "Wait"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil)
+	if err != nil {
+		return makeProcessError(process, operation, err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+// WaitTimeout waits for the process to exit or the duration to elapse. It returns
+// false if timeout occurs.
+func (process *Process) WaitTimeout(timeout time.Duration) error {
+	operation := "WaitTimeout"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout)
+	if err != nil {
+		return makeProcessError(process, operation, err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+// ResizeConsole resizes the console of the process.
+func (process *Process) ResizeConsole(width, height uint16) error {
+	process.handleLock.RLock()
+	defer process.handleLock.RUnlock()
+	operation := "ResizeConsole"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	if process.handle == 0 {
+		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
+	}
+
+	modifyRequest := processModifyRequest{
+		Operation: modifyConsoleSize,
+		ConsoleSize: &consoleSize{
+			Height: height,
+			Width:  width,
+		},
+	}
+
+	modifyRequestb, err := json.Marshal(modifyRequest)
+	if err != nil {
+		return err
+	}
+
+	modifyRequestStr := string(modifyRequestb)
+
+	var resultp *uint16
+	err = hcsModifyProcess(process.handle, modifyRequestStr, &resultp)
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeProcessError(process, operation, err, events)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+func (process *Process) Properties() (*ProcessStatus, error) {
+	process.handleLock.RLock()
+	defer process.handleLock.RUnlock()
+	operation := "Properties"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	if process.handle == 0 {
+		return nil, makeProcessError(process, operation, ErrAlreadyClosed, nil)
+	}
+
+	var (
+		resultp     *uint16
+		propertiesp *uint16
+	)
+	completed := false
+	go syscallWatcher(fmt.Sprintf("GetProcessProperties %s: %d", process.SystemID(), process.Pid()), &completed)
+	err := hcsGetProcessProperties(process.handle, &propertiesp, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return nil, makeProcessError(process, operation, err, events)
+	}
+
+	if propertiesp == nil {
+		return nil, ErrUnexpectedValue
+	}
+	propertiesRaw := interop.ConvertAndFreeCoTaskMemBytes(propertiesp)
+
+	properties := &ProcessStatus{}
+	if err := json.Unmarshal(propertiesRaw, properties); err != nil {
+		return nil, makeProcessError(process, operation, err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d, properties=%s", process.processID, propertiesRaw)
+	return properties, nil
+}
+
+// ExitCode returns the exit code of the process. The process must have
+// already terminated.
+func (process *Process) ExitCode() (int, error) {
+	operation := "ExitCode"
+	properties, err := process.Properties()
+	if err != nil {
+		return 0, makeProcessError(process, operation, err, nil)
+	}
+
+	if properties.Exited == false {
+		return 0, makeProcessError(process, operation, ErrInvalidProcessState, nil)
+	}
+
+	if properties.LastWaitResult != 0 {
+		return 0, makeProcessError(process, operation, syscall.Errno(properties.LastWaitResult), nil)
+	}
+
+	return int(properties.ExitCode), nil
+}
+
+// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing
+// these pipes does not close the underlying pipes; it should be possible to
+// call this multiple times to get multiple interfaces.
+func (process *Process) Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) {
+	process.handleLock.RLock()
+	defer process.handleLock.RUnlock()
+	operation := "Stdio"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	if process.handle == 0 {
+		return nil, nil, nil, makeProcessError(process, operation, ErrAlreadyClosed, nil)
+	}
+
+	var stdIn, stdOut, stdErr syscall.Handle
+
+	if process.cachedPipes == nil {
+		var (
+			processInfo hcsProcessInformation
+			resultp     *uint16
+		)
+		err := hcsGetProcessInfo(process.handle, &processInfo, &resultp)
+		events := processHcsResult(resultp)
+		if err != nil {
+			return nil, nil, nil, makeProcessError(process, operation, err, events)
+		}
+
+		stdIn, stdOut, stdErr = processInfo.StdInput, processInfo.StdOutput, processInfo.StdError
+	} else {
+		// Use cached pipes
+		stdIn, stdOut, stdErr = process.cachedPipes.stdIn, process.cachedPipes.stdOut, process.cachedPipes.stdErr
+
+		// Invalidate the cache
+		process.cachedPipes = nil
+	}
+
+	pipes, err := makeOpenFiles([]syscall.Handle{stdIn, stdOut, stdErr})
+	if err != nil {
+		return nil, nil, nil, makeProcessError(process, operation, err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return pipes[0], pipes[1], pipes[2], nil
+}
+
+// CloseStdin closes the write side of the stdin pipe so that the process is
+// notified on the read side that there is no more data in stdin.
+func (process *Process) CloseStdin() error {
+	process.handleLock.RLock()
+	defer process.handleLock.RUnlock()
+	operation := "CloseStdin"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	if process.handle == 0 {
+		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
+	}
+
+	modifyRequest := processModifyRequest{
+		Operation: modifyCloseHandle,
+		CloseHandle: &closeHandle{
+			Handle: stdIn,
+		},
+	}
+
+	modifyRequestb, err := json.Marshal(modifyRequest)
+	if err != nil {
+		return err
+	}
+
+	modifyRequestStr := string(modifyRequestb)
+
+	var resultp *uint16
+	err = hcsModifyProcess(process.handle, modifyRequestStr, &resultp)
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeProcessError(process, operation, err, events)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+// Close cleans up any state associated with the process but does not kill
+// or wait on it.
+func (process *Process) Close() error {
+	process.handleLock.Lock()
+	defer process.handleLock.Unlock()
+	operation := "Close"
+	title := "hcsshim::Process::" + operation
+	logrus.Debugf(title+" processid=%d", process.processID)
+
+	// Don't double free this
+	if process.handle == 0 {
+		return nil
+	}
+
+	if err := process.unregisterCallback(); err != nil {
+		return makeProcessError(process, operation, err, nil)
+	}
+
+	if err := hcsCloseProcess(process.handle); err != nil {
+		return makeProcessError(process, operation, err, nil)
+	}
+
+	process.handle = 0
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return nil
+}
+
+func (process *Process) registerCallback() error {
+	context := &notifcationWatcherContext{
+		channels: newChannels(),
+	}
+
+	callbackMapLock.Lock()
+	callbackNumber := nextCallback
+	nextCallback++
+	callbackMap[callbackNumber] = context
+	callbackMapLock.Unlock()
+
+	var callbackHandle hcsCallback
+	err := hcsRegisterProcessCallback(process.handle, notificationWatcherCallback, callbackNumber, &callbackHandle)
+	if err != nil {
+		return err
+	}
+	context.handle = callbackHandle
+	process.callbackNumber = callbackNumber
+
+	return nil
+}
+
+func (process *Process) unregisterCallback() error {
+	callbackNumber := process.callbackNumber
+
+	callbackMapLock.RLock()
+	context := callbackMap[callbackNumber]
+	callbackMapLock.RUnlock()
+
+	if context == nil {
+		return nil
+	}
+
+	handle := context.handle
+
+	if handle == 0 {
+		return nil
+	}
+
+	// hcsUnregisterProcessCallback has its own syncronization
+	// to wait for all callbacks to complete. We must NOT hold the callbackMapLock.
+	err := hcsUnregisterProcessCallback(handle)
+	if err != nil {
+		return err
+	}
+
+	closeChannels(context.channels)
+
+	callbackMapLock.Lock()
+	callbackMap[callbackNumber] = nil
+	callbackMapLock.Unlock()
+
+	handle = 0
+
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go

@@ -0,0 +1,585 @@
+package hcs
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/interop"
+	"github.com/Microsoft/hcsshim/internal/schema1"
+	"github.com/Microsoft/hcsshim/internal/timeout"
+	"github.com/sirupsen/logrus"
+)
+
+// currentContainerStarts is used to limit the number of concurrent container
+// starts.
+var currentContainerStarts containerStarts
+
+type containerStarts struct {
+	maxParallel int
+	inProgress  int
+	sync.Mutex
+}
+
+func init() {
+	mpsS := os.Getenv("HCSSHIM_MAX_PARALLEL_START")
+	if len(mpsS) > 0 {
+		mpsI, err := strconv.Atoi(mpsS)
+		if err != nil || mpsI < 0 {
+			return
+		}
+		currentContainerStarts.maxParallel = mpsI
+	}
+}
+
+type System struct {
+	handleLock     sync.RWMutex
+	handle         hcsSystem
+	id             string
+	callbackNumber uintptr
+}
+
+// CreateComputeSystem creates a new compute system with the given configuration but does not start it.
+func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System, error) {
+	operation := "CreateComputeSystem"
+	title := "hcsshim::" + operation
+
+	computeSystem := &System{
+		id: id,
+	}
+
+	hcsDocumentB, err := json.Marshal(hcsDocumentInterface)
+	if err != nil {
+		return nil, err
+	}
+
+	hcsDocument := string(hcsDocumentB)
+	logrus.Debugf(title+" ID=%s config=%s", id, hcsDocument)
+
+	var (
+		resultp  *uint16
+		identity syscall.Handle
+	)
+	completed := false
+	go syscallWatcher(fmt.Sprintf("CreateCompleteSystem %s: %s", id, hcsDocument), &completed)
+	createError := hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp)
+	completed = true
+
+	if createError == nil || IsPending(createError) {
+		if err := computeSystem.registerCallback(); err != nil {
+			// Terminate the compute system if it still exists. We're okay to
+			// ignore a failure here.
+			computeSystem.Terminate()
+			return nil, makeSystemError(computeSystem, operation, "", err, nil)
+		}
+	}
+
+	events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.SystemCreate)
+	if err != nil {
+		if err == ErrTimeout {
+			// Terminate the compute system if it still exists. We're okay to
+			// ignore a failure here.
+			computeSystem.Terminate()
+		}
+		return nil, makeSystemError(computeSystem, operation, hcsDocument, err, events)
+	}
+
+	logrus.Debugf(title+" succeeded id=%s handle=%d", id, computeSystem.handle)
+	return computeSystem, nil
+}
+
+// OpenComputeSystem opens an existing compute system by ID.
+func OpenComputeSystem(id string) (*System, error) {
+	operation := "OpenComputeSystem"
+	title := "hcsshim::" + operation
+	logrus.Debugf(title+" ID=%s", id)
+
+	computeSystem := &System{
+		id: id,
+	}
+
+	var (
+		handle  hcsSystem
+		resultp *uint16
+	)
+	err := hcsOpenComputeSystem(id, &handle, &resultp)
+	events := processHcsResult(resultp)
+	if err != nil {
+		return nil, makeSystemError(computeSystem, operation, "", err, events)
+	}
+
+	computeSystem.handle = handle
+
+	if err := computeSystem.registerCallback(); err != nil {
+		return nil, makeSystemError(computeSystem, operation, "", err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded id=%s handle=%d", id, handle)
+	return computeSystem, nil
+}
+
+// GetComputeSystems gets a list of the compute systems on the system that match the query
+func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerProperties, error) {
+	operation := "GetComputeSystems"
+	title := "hcsshim::" + operation
+
+	queryb, err := json.Marshal(q)
+	if err != nil {
+		return nil, err
+	}
+
+	query := string(queryb)
+	logrus.Debugf(title+" query=%s", query)
+
+	var (
+		resultp         *uint16
+		computeSystemsp *uint16
+	)
+	completed := false
+	go syscallWatcher(fmt.Sprintf("GetComputeSystems %s:", query), &completed)
+	err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return nil, &HcsError{Op: operation, Err: err, Events: events}
+	}
+
+	if computeSystemsp == nil {
+		return nil, ErrUnexpectedValue
+	}
+	computeSystemsRaw := interop.ConvertAndFreeCoTaskMemBytes(computeSystemsp)
+	computeSystems := []schema1.ContainerProperties{}
+	if err := json.Unmarshal(computeSystemsRaw, &computeSystems); err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return computeSystems, nil
+}
+
+// Start synchronously starts the computeSystem.
+func (computeSystem *System) Start() error {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::Start ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, "Start", "", ErrAlreadyClosed, nil)
+	}
+
+	// This is a very simple backoff-retry loop to limit the number
+	// of parallel container starts if environment variable
+	// HCSSHIM_MAX_PARALLEL_START is set to a positive integer.
+	// It should generally only be used as a workaround to various
+	// platform issues that exist between RS1 and RS4 as of Aug 2018
+	if currentContainerStarts.maxParallel > 0 {
+		for {
+			currentContainerStarts.Lock()
+			if currentContainerStarts.inProgress < currentContainerStarts.maxParallel {
+				currentContainerStarts.inProgress++
+				currentContainerStarts.Unlock()
+				break
+			}
+			if currentContainerStarts.inProgress == currentContainerStarts.maxParallel {
+				currentContainerStarts.Unlock()
+				time.Sleep(100 * time.Millisecond)
+			}
+		}
+		// Make sure we decrement the count when we are done.
+		defer func() {
+			currentContainerStarts.Lock()
+			currentContainerStarts.inProgress--
+			currentContainerStarts.Unlock()
+		}()
+	}
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("StartComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsStartComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
+	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart)
+	if err != nil {
+		return makeSystemError(computeSystem, "Start", "", err, events)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+// ID returns the compute system's identifier.
+func (computeSystem *System) ID() string {
+	return computeSystem.id
+}
+
+// Shutdown requests a compute system shutdown, if IsPending() on the error returned is true,
+// it may not actually be shut down until Wait() succeeds.
+func (computeSystem *System) Shutdown() error {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::Shutdown"
+	logrus.Debugf(title)
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, "Shutdown", "", ErrAlreadyClosed, nil)
+	}
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("ShutdownComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsShutdownComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeSystemError(computeSystem, "Shutdown", "", err, events)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+// Terminate requests a compute system terminate, if IsPending() on the error returned is true,
+// it may not actually be shut down until Wait() succeeds.
+func (computeSystem *System) Terminate() error {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::Terminate ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, "Terminate", "", ErrAlreadyClosed, nil)
+	}
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("TerminateComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsTerminateComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeSystemError(computeSystem, "Terminate", "", err, events)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+// Wait synchronously waits for the compute system to shutdown or terminate.
+func (computeSystem *System) Wait() error {
+	title := "hcsshim::ComputeSystem::Wait ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	err := waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil)
+	if err != nil {
+		return makeSystemError(computeSystem, "Wait", "", err, nil)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+// WaitTimeout synchronously waits for the compute system to terminate or the duration to elapse.
+// If the timeout expires, IsTimeout(err) == true
+func (computeSystem *System) WaitTimeout(timeout time.Duration) error {
+	title := "hcsshim::ComputeSystem::WaitTimeout ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	err := waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, &timeout)
+	if err != nil {
+		return makeSystemError(computeSystem, "WaitTimeout", "", err, nil)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+func (computeSystem *System) Properties(types ...schema1.PropertyType) (*schema1.ContainerProperties, error) {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+
+	queryj, err := json.Marshal(schema1.PropertyQuery{types})
+	if err != nil {
+		return nil, makeSystemError(computeSystem, "Properties", "", err, nil)
+	}
+
+	var resultp, propertiesp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("GetComputeSystemProperties %s:", computeSystem.ID()), &completed)
+	err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return nil, makeSystemError(computeSystem, "Properties", "", err, events)
+	}
+
+	if propertiesp == nil {
+		return nil, ErrUnexpectedValue
+	}
+	propertiesRaw := interop.ConvertAndFreeCoTaskMemBytes(propertiesp)
+	properties := &schema1.ContainerProperties{}
+	if err := json.Unmarshal(propertiesRaw, properties); err != nil {
+		return nil, makeSystemError(computeSystem, "Properties", "", err, nil)
+	}
+	return properties, nil
+}
+
+// Pause pauses the execution of the computeSystem. This feature is not enabled in TP5.
+func (computeSystem *System) Pause() error {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::Pause ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, "Pause", "", ErrAlreadyClosed, nil)
+	}
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("PauseComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsPauseComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
+	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause)
+	if err != nil {
+		return makeSystemError(computeSystem, "Pause", "", err, events)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+// Resume resumes the execution of the computeSystem. This feature is not enabled in TP5.
+func (computeSystem *System) Resume() error {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::Resume ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, "Resume", "", ErrAlreadyClosed, nil)
+	}
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("ResumeComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsResumeComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
+	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume)
+	if err != nil {
+		return makeSystemError(computeSystem, "Resume", "", err, events)
+	}
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+// CreateProcess launches a new process within the computeSystem.
+func (computeSystem *System) CreateProcess(c interface{}) (*Process, error) {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::CreateProcess ID=" + computeSystem.ID()
+	var (
+		processInfo   hcsProcessInformation
+		processHandle hcsProcess
+		resultp       *uint16
+	)
+
+	if computeSystem.handle == 0 {
+		return nil, makeSystemError(computeSystem, "CreateProcess", "", ErrAlreadyClosed, nil)
+	}
+
+	configurationb, err := json.Marshal(c)
+	if err != nil {
+		return nil, makeSystemError(computeSystem, "CreateProcess", "", err, nil)
+	}
+
+	configuration := string(configurationb)
+	logrus.Debugf(title+" config=%s", configuration)
+
+	completed := false
+	go syscallWatcher(fmt.Sprintf("CreateProcess %s: %s", computeSystem.ID(), configuration), &completed)
+	err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events)
+	}
+
+	process := &Process{
+		handle:    processHandle,
+		processID: int(processInfo.ProcessId),
+		system:    computeSystem,
+		cachedPipes: &cachedPipes{
+			stdIn:  processInfo.StdInput,
+			stdOut: processInfo.StdOutput,
+			stdErr: processInfo.StdError,
+		},
+	}
+
+	if err := process.registerCallback(); err != nil {
+		return nil, makeSystemError(computeSystem, "CreateProcess", "", err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%d", process.processID)
+	return process, nil
+}
+
+// OpenProcess gets an interface to an existing process within the computeSystem.
+func (computeSystem *System) OpenProcess(pid int) (*Process, error) {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::ComputeSystem::OpenProcess ID=" + computeSystem.ID()
+	logrus.Debugf(title+" processid=%d", pid)
+	var (
+		processHandle hcsProcess
+		resultp       *uint16
+	)
+
+	if computeSystem.handle == 0 {
+		return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil)
+	}
+
+	completed := false
+	go syscallWatcher(fmt.Sprintf("OpenProcess %s: %d", computeSystem.ID(), pid), &completed)
+	err := hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events)
+	}
+
+	process := &Process{
+		handle:    processHandle,
+		processID: pid,
+		system:    computeSystem,
+	}
+
+	if err := process.registerCallback(); err != nil {
+		return nil, makeSystemError(computeSystem, "OpenProcess", "", err, nil)
+	}
+
+	logrus.Debugf(title+" succeeded processid=%s", process.processID)
+	return process, nil
+}
+
+// Close cleans up any state associated with the compute system but does not terminate or wait for it.
+func (computeSystem *System) Close() error {
+	computeSystem.handleLock.Lock()
+	defer computeSystem.handleLock.Unlock()
+	title := "hcsshim::ComputeSystem::Close ID=" + computeSystem.ID()
+	logrus.Debugf(title)
+
+	// Don't double free this
+	if computeSystem.handle == 0 {
+		return nil
+	}
+
+	if err := computeSystem.unregisterCallback(); err != nil {
+		return makeSystemError(computeSystem, "Close", "", err, nil)
+	}
+
+	completed := false
+	go syscallWatcher(fmt.Sprintf("CloseComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsCloseComputeSystem(computeSystem.handle)
+	completed = true
+	if err != nil {
+		return makeSystemError(computeSystem, "Close", "", err, nil)
+	}
+
+	computeSystem.handle = 0
+
+	logrus.Debugf(title + " succeeded")
+	return nil
+}
+
+func (computeSystem *System) registerCallback() error {
+	context := &notifcationWatcherContext{
+		channels: newChannels(),
+	}
+
+	callbackMapLock.Lock()
+	callbackNumber := nextCallback
+	nextCallback++
+	callbackMap[callbackNumber] = context
+	callbackMapLock.Unlock()
+
+	var callbackHandle hcsCallback
+	err := hcsRegisterComputeSystemCallback(computeSystem.handle, notificationWatcherCallback, callbackNumber, &callbackHandle)
+	if err != nil {
+		return err
+	}
+	context.handle = callbackHandle
+	computeSystem.callbackNumber = callbackNumber
+
+	return nil
+}
+
+func (computeSystem *System) unregisterCallback() error {
+	callbackNumber := computeSystem.callbackNumber
+
+	callbackMapLock.RLock()
+	context := callbackMap[callbackNumber]
+	callbackMapLock.RUnlock()
+
+	if context == nil {
+		return nil
+	}
+
+	handle := context.handle
+
+	if handle == 0 {
+		return nil
+	}
+
+	// hcsUnregisterComputeSystemCallback has its own syncronization
+	// to wait for all callbacks to complete. We must NOT hold the callbackMapLock.
+	err := hcsUnregisterComputeSystemCallback(handle)
+	if err != nil {
+		return err
+	}
+
+	closeChannels(context.channels)
+
+	callbackMapLock.Lock()
+	callbackMap[callbackNumber] = nil
+	callbackMapLock.Unlock()
+
+	handle = 0
+
+	return nil
+}
+
+// Modifies the System by sending a request to HCS
+func (computeSystem *System) Modify(config interface{}) error {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+	title := "hcsshim::Modify ID=" + computeSystem.id
+
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, "Modify", "", ErrAlreadyClosed, nil)
+	}
+
+	requestJSON, err := json.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	requestString := string(requestJSON)
+	logrus.Debugf(title + " " + requestString)
+
+	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("ModifyComputeSystem %s: %s", computeSystem.ID(), requestString), &completed)
+	err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp)
+	completed = true
+	events := processHcsResult(resultp)
+	if err != nil {
+		return makeSystemError(computeSystem, "Modify", requestString, err, events)
+	}
+	logrus.Debugf(title + " succeeded ")
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/utils.go

@@ -0,0 +1,33 @@
+package hcs
+
+import (
+	"io"
+	"syscall"
+
+	"github.com/Microsoft/go-winio"
+)
+
+// makeOpenFiles calls winio.MakeOpenFile for each handle in a slice but closes all the handles
+// if there is an error.
+func makeOpenFiles(hs []syscall.Handle) (_ []io.ReadWriteCloser, err error) {
+	fs := make([]io.ReadWriteCloser, len(hs))
+	for i, h := range hs {
+		if h != syscall.Handle(0) {
+			if err == nil {
+				fs[i], err = winio.MakeOpenFile(h)
+			}
+			if err != nil {
+				syscall.Close(h)
+			}
+		}
+	}
+	if err != nil {
+		for _, f := range fs {
+			if f != nil {
+				f.Close()
+			}
+		}
+		return nil, err
+	}
+	return fs, nil
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go

@@ -0,0 +1,63 @@
+package hcs
+
+import (
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) ([]ErrorEvent, error) {
+	events := processHcsResult(resultp)
+	if IsPending(err) {
+		return nil, waitForNotification(callbackNumber, expectedNotification, timeout)
+	}
+
+	return events, err
+}
+
+func waitForNotification(callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error {
+	callbackMapLock.RLock()
+	channels := callbackMap[callbackNumber].channels
+	callbackMapLock.RUnlock()
+
+	expectedChannel := channels[expectedNotification]
+	if expectedChannel == nil {
+		logrus.Errorf("unknown notification type in waitForNotification %x", expectedNotification)
+		return ErrInvalidNotificationType
+	}
+
+	var c <-chan time.Time
+	if timeout != nil {
+		timer := time.NewTimer(*timeout)
+		c = timer.C
+		defer timer.Stop()
+	}
+
+	select {
+	case err, ok := <-expectedChannel:
+		if !ok {
+			return ErrHandleClose
+		}
+		return err
+	case err, ok := <-channels[hcsNotificationSystemExited]:
+		if !ok {
+			return ErrHandleClose
+		}
+		// If the expected notification is hcsNotificationSystemExited which of the two selects
+		// chosen is random. Return the raw error if hcsNotificationSystemExited is expected
+		if channels[hcsNotificationSystemExited] == expectedChannel {
+			return err
+		}
+		return ErrUnexpectedContainerExit
+	case _, ok := <-channels[hcsNotificationServiceDisconnect]:
+		if !ok {
+			return ErrHandleClose
+		}
+		// hcsNotificationServiceDisconnect should never be an expected notification
+		// it does not need the same handling as hcsNotificationSystemExited
+		return ErrUnexpectedProcessAbort
+	case <-c:
+		return ErrTimeout
+	}
+	return nil
+}