Merge pull request #572 from hyperhq/shimv2

Implement containerd shim v2 API for Kata Containers
This commit is contained in:
Sebastien Boeuf
2018-11-28 16:37:10 +00:00
committed by GitHub
358 changed files with 44605 additions and 1207 deletions

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
@@ -15,6 +15,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
var defaultFirmwarePath = ""
var defaultMachineAccelerators = ""
var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
var systemdUnitName = "kata-containers.target"
const defaultKernelParams = ""
const defaultMachineType = "pc"

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
@@ -28,6 +28,9 @@ const (
var (
defaultProxy = vc.KataProxyType
defaultShim = vc.KataShimType
// if true, enable opentracing support.
tracing = false
)
// The TOML configuration file contains a number of sections (or
@@ -531,7 +534,7 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
return nil
}
func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) {
func initConfig() (config oci.RuntimeConfig, err error) {
var defaultAgentConfig interface{}
defaultHypervisorConfig := vc.HypervisorConfig{
@@ -565,13 +568,6 @@ func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) {
defaultAgentConfig = vc.HyperConfig{}
if builtIn {
defaultProxy = vc.KataBuiltInProxyType
defaultShim = vc.KataBuiltInShimType
defaultAgentConfig = vc.KataAgentConfig{LongLiveConn: true}
}
config = oci.RuntimeConfig{
HypervisorType: defaultHypervisor,
HypervisorConfig: defaultHypervisorConfig,
@@ -592,12 +588,12 @@ func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) {
//
// All paths are resolved fully meaning if this function does not return an
// error, all paths are valid at the time of the call.
func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, tracing bool, err error) {
func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
var resolved string
config, err = initConfig(builtIn)
config, err = initConfig()
if err != nil {
return "", oci.RuntimeConfig{}, tracing, err
return "", oci.RuntimeConfig{}, err
}
if configPath == "" {
@@ -607,18 +603,18 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
}
if err != nil {
return "", config, tracing, fmt.Errorf("Cannot find usable config file (%v)", err)
return "", config, fmt.Errorf("Cannot find usable config file (%v)", err)
}
configData, err := ioutil.ReadFile(resolved)
if err != nil {
return "", config, tracing, err
return "", config, err
}
var tomlConf tomlConfig
_, err = toml.Decode(string(configData), &tomlConf)
if err != nil {
return "", config, tracing, err
return "", config, err
}
config.Debug = tomlConf.Runtime.Debug
@@ -634,14 +630,14 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
if tomlConf.Runtime.InterNetworkModel != "" {
err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel)
if err != nil {
return "", config, tracing, err
return "", config, err
}
}
if !ignoreLogging {
err := handleSystemLog("", "")
if err != nil {
return "", config, tracing, err
return "", config, err
}
kataUtilsLogger.WithFields(
@@ -651,13 +647,13 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
}).Info("loaded configuration")
}
if err := updateRuntimeConfig(resolved, tomlConf, &config); err != nil {
return "", config, tracing, err
if err := updateConfig(resolved, tomlConf, &config, builtIn); err != nil {
return "", config, err
}
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
if err := checkNetNsConfig(config); err != nil {
return "", config, tracing, err
return "", config, err
}
// use no proxy if HypervisorConfig.UseVSock is true
@@ -668,10 +664,26 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
}
if err := checkHypervisorConfig(config.HypervisorConfig); err != nil {
return "", config, tracing, err
return "", config, err
}
return resolved, config, tracing, nil
return resolved, config, nil
}
func updateConfig(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error {
if err := updateRuntimeConfig(configPath, tomlConf, config); err != nil {
return err
}
if builtIn {
config.ProxyType = vc.KataBuiltInProxyType
config.ShimType = vc.KataBuiltInShimType
config.AgentType = vc.KataContainersAgent
config.AgentConfig = vc.KataAgentConfig{LongLiveConn: true}
}
return nil
}
// checkNetNsConfig performs sanity checks on disable_new_netns config.

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
@@ -249,7 +249,7 @@ func testLoadConfiguration(t *testing.T, dir string,
assert.NoError(t, err)
}
resolvedConfigPath, config, _, err := LoadConfiguration(file, ignoreLogging, false)
resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
if expectFail {
assert.Error(t, err)
@@ -558,7 +558,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
t.Fatal(err)
}
_, config, _, err := LoadConfiguration(configPath, false, false)
_, config, err := LoadConfiguration(configPath, false, false)
if err == nil {
t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
}
@@ -583,7 +583,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
t.Error(err)
}
_, config, _, err = LoadConfiguration(configPath, false, false)
_, config, err = LoadConfiguration(configPath, false, false)
if err != nil {
t.Fatal(err)
}
@@ -712,7 +712,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
t.Fatal(err)
}
_, config, _, err := LoadConfiguration(configPath, false, false)
_, config, err := LoadConfiguration(configPath, false, false)
if err != nil {
t.Fatal(err)
}

248
pkg/katautils/create.go Normal file
View File

@@ -0,0 +1,248 @@
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"context"
"fmt"
"strings"
vc "github.com/kata-containers/runtime/virtcontainers"
vf "github.com/kata-containers/runtime/virtcontainers/factory"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
)
// GetKernelParamsFunc use a variable to allow tests to modify its value
var GetKernelParamsFunc = getKernelParams
var systemdKernelParam = []vc.Param{
{
Key: "init",
Value: "/usr/lib/systemd/systemd",
},
{
Key: "systemd.unit",
Value: systemdUnitName,
},
{
Key: "systemd.mask",
Value: "systemd-networkd.service",
},
{
Key: "systemd.mask",
Value: "systemd-networkd.socket",
},
}
func getKernelParams(needSystemd bool) []vc.Param {
p := []vc.Param{}
if needSystemd {
p = append(p, systemdKernelParam...)
}
return p
}
func needSystemd(config vc.HypervisorConfig) bool {
return config.ImagePath != ""
}
// HandleFactory set the factory
func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) {
if !runtimeConfig.FactoryConfig.Template {
return
}
factoryConfig := vf.Config{
Template: true,
VMConfig: vc.VMConfig{
HypervisorType: runtimeConfig.HypervisorType,
HypervisorConfig: runtimeConfig.HypervisorConfig,
AgentType: runtimeConfig.AgentType,
AgentConfig: runtimeConfig.AgentConfig,
},
}
kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory")
f, err := vf.NewFactory(ctx, factoryConfig, true)
if err != nil {
kataUtilsLogger.WithError(err).Warn("load vm factory failed, about to create new one")
f, err = vf.NewFactory(ctx, factoryConfig, false)
if err != nil {
kataUtilsLogger.WithError(err).Warn("create vm factory failed")
return
}
}
vci.SetFactory(ctx, f)
}
// SetKernelParams adds the user-specified kernel parameters (from the
// configuration file) to the defaults so that the former take priority.
func SetKernelParams(containerID string, runtimeConfig *oci.RuntimeConfig) error {
defaultKernelParams := GetKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig))
if runtimeConfig.HypervisorConfig.Debug {
strParams := vc.SerializeParams(defaultKernelParams, "=")
formatted := strings.Join(strParams, " ")
kataUtilsLogger.WithField("default-kernel-parameters", formatted).Debug()
}
// retrieve the parameters specified in the config file
userKernelParams := runtimeConfig.HypervisorConfig.KernelParams
// reset
runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{}
// first, add default values
for _, p := range defaultKernelParams {
if err := (runtimeConfig).AddKernelParam(p); err != nil {
return err
}
}
// now re-add the user-specified values so that they take priority.
for _, p := range userKernelParams {
if err := (runtimeConfig).AddKernelParam(p); err != nil {
return err
}
}
return nil
}
// SetEphemeralStorageType sets the mount type to 'ephemeral'
// if the mount source path is provisioned by k8s for ephemeral storage.
// For the given pod ephemeral volume is created only once
// backed by tmpfs inside the VM. For successive containers
// of the same pod the already existing volume is reused.
func SetEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec {
for idx, mnt := range ociSpec.Mounts {
if IsEphemeralStorage(mnt.Source) {
ociSpec.Mounts[idx].Type = "ephemeral"
}
}
return ociSpec
}
// CreateSandbox create a sandbox container
func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig,
containerID, bundlePath, console string, disableOutput, systemdCgroup, builtIn bool) (vc.VCSandbox, vc.Process, error) {
span, ctx := Trace(ctx, "createSandbox")
defer span.Finish()
err := SetKernelParams(containerID, &runtimeConfig)
if err != nil {
return nil, vc.Process{}, err
}
sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
if err != nil {
return nil, vc.Process{}, err
}
if builtIn {
sandboxConfig.Stateful = true
}
// Important to create the network namespace before the sandbox is
// created, because it is not responsible for the creation of the
// netns if it does not exist.
if err := SetupNetworkNamespace(&sandboxConfig.NetworkConfig); err != nil {
return nil, vc.Process{}, err
}
// Run pre-start OCI hooks.
err = EnterNetNS(sandboxConfig.NetworkConfig.NetNSPath, func() error {
return PreStartHooks(ctx, ociSpec, containerID, bundlePath)
})
if err != nil {
return nil, vc.Process{}, err
}
sandbox, err := vci.CreateSandbox(ctx, sandboxConfig)
if err != nil {
return nil, vc.Process{}, err
}
sid := sandbox.ID()
kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
span.SetTag("sandbox", sid)
containers := sandbox.GetAllContainers()
if len(containers) != 1 {
return nil, vc.Process{}, fmt.Errorf("BUG: Container list from sandbox is wrong, expecting only one container, found %d containers", len(containers))
}
if !builtIn {
err = AddContainerIDMapping(ctx, containerID, sandbox.ID())
if err != nil {
return nil, vc.Process{}, err
}
}
return sandbox, containers[0].Process(), nil
}
// CreateContainer create a container
func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) {
var c vc.VCContainer
span, ctx := Trace(ctx, "createContainer")
defer span.Finish()
ociSpec = SetEphemeralStorageType(ociSpec)
contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput)
if err != nil {
return vc.Process{}, err
}
sandboxID, err := ociSpec.SandboxID()
if err != nil {
return vc.Process{}, err
}
span.SetTag("sandbox", sandboxID)
if builtIn {
c, err = sandbox.CreateContainer(contConfig)
if err != nil {
return vc.Process{}, err
}
} else {
kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sandboxID)
sandbox, c, err = vci.CreateContainer(ctx, sandboxID, contConfig)
if err != nil {
return vc.Process{}, err
}
if err := AddContainerIDMapping(ctx, containerID, sandboxID); err != nil {
return vc.Process{}, err
}
kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sandboxID)
if err := AddContainerIDMapping(ctx, containerID, sandboxID); err != nil {
return vc.Process{}, err
}
}
// Run pre-start OCI hooks.
err = EnterNetNS(sandbox.GetNetNs(), func() error {
return PreStartHooks(ctx, ociSpec, containerID, bundlePath)
})
if err != nil {
return vc.Process{}, err
}
return c.Process(), nil
}

View File

@@ -0,0 +1,455 @@
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"testing"
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
)
const (
testConsole = "/dev/pts/999"
testContainerTypeAnnotation = "io.kubernetes.cri-o.ContainerType"
testSandboxIDAnnotation = "io.kubernetes.cri-o.SandboxID"
testContainerTypeContainer = "container"
)
var (
testBundleDir = ""
// testingImpl is a concrete mock RVC implementation used for testing
testingImpl = &vcmock.VCMock{}
)
// readOCIConfig returns an OCI spec.
func readOCIConfigFile(configPath string) (oci.CompatOCISpec, error) {
if configPath == "" {
return oci.CompatOCISpec{}, errors.New("BUG: need config file path")
}
data, err := ioutil.ReadFile(configPath)
if err != nil {
return oci.CompatOCISpec{}, err
}
var ociSpec oci.CompatOCISpec
if err := json.Unmarshal(data, &ociSpec); err != nil {
return oci.CompatOCISpec{}, err
}
caps, err := oci.ContainerCapabilities(ociSpec)
if err != nil {
return oci.CompatOCISpec{}, err
}
ociSpec.Process.Capabilities = caps
return ociSpec, nil
}
func writeOCIConfigFile(spec oci.CompatOCISpec, configPath string) error {
if configPath == "" {
return errors.New("BUG: need config file path")
}
bytes, err := json.MarshalIndent(spec, "", "\t")
if err != nil {
return err
}
return ioutil.WriteFile(configPath, bytes, testFileMode)
}
// Create an OCI bundle in the specified directory.
//
// Note that the directory will be created, but it's parent is expected to exist.
//
// This function works by copying the already-created test bundle. Ideally,
// the bundle would be recreated for each test, but createRootfs() uses
// docker which on some systems is too slow, resulting in the tests timing
// out.
func makeOCIBundle(bundleDir string) error {
from := testBundleDir
to := bundleDir
// only the basename of bundleDir needs to exist as bundleDir
// will get created by cp(1).
base := filepath.Dir(bundleDir)
for _, dir := range []string{from, base} {
if !FileExists(dir) {
return fmt.Errorf("BUG: directory %v should exist", dir)
}
}
output, err := RunCommandFull([]string{"cp", "-a", from, to}, true)
if err != nil {
return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
}
return nil
}
// newTestRuntimeConfig creates a new RuntimeConfig
func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) {
if dir == "" {
return oci.RuntimeConfig{}, errors.New("BUG: need directory")
}
hypervisorConfig, err := newTestHypervisorConfig(dir, create)
if err != nil {
return oci.RuntimeConfig{}, err
}
return oci.RuntimeConfig{
HypervisorType: vc.QemuHypervisor,
HypervisorConfig: hypervisorConfig,
AgentType: vc.KataContainersAgent,
ProxyType: vc.CCProxyType,
ShimType: vc.CCShimType,
Console: consolePath,
}, nil
}
// newTestHypervisorConfig creaets a new virtcontainers
// HypervisorConfig, ensuring that the required resources are also
// created.
//
// Note: no parameter validation in case caller wishes to create an invalid
// object.
func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) {
kernelPath := path.Join(dir, "kernel")
imagePath := path.Join(dir, "image")
hypervisorPath := path.Join(dir, "hypervisor")
if create {
for _, file := range []string{kernelPath, imagePath, hypervisorPath} {
err := createEmptyFile(file)
if err != nil {
return vc.HypervisorConfig{}, err
}
}
}
return vc.HypervisorConfig{
KernelPath: kernelPath,
ImagePath: imagePath,
HypervisorPath: hypervisorPath,
HypervisorMachineType: "pc-lite",
}, nil
}
// return the value of the *last* param with the specified key
func findLastParam(key string, params []vc.Param) (string, error) {
if key == "" {
return "", errors.New("ERROR: need non-nil key")
}
l := len(params)
if l == 0 {
return "", errors.New("ERROR: no params")
}
for i := l - 1; i >= 0; i-- {
p := params[i]
if key == p.Key {
return p.Value, nil
}
}
return "", fmt.Errorf("no param called %q found", name)
}
func TestSetEphemeralStorageType(t *testing.T) {
assert := assert.New(t)
ociSpec := oci.CompatOCISpec{}
var ociMounts []specs.Mount
mount := specs.Mount{
Source: "/var/lib/kubelet/pods/366c3a77-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume",
}
ociMounts = append(ociMounts, mount)
ociSpec.Mounts = ociMounts
ociSpec = SetEphemeralStorageType(ociSpec)
mountType := ociSpec.Mounts[0].Type
assert.Equal(mountType, "ephemeral",
"Unexpected mount type, got %s expected ephemeral", mountType)
}
func TestSetKernelParams(t *testing.T) {
assert := assert.New(t)
config := oci.RuntimeConfig{}
assert.Empty(config.HypervisorConfig.KernelParams)
err := SetKernelParams(testContainerID, &config)
assert.NoError(err)
if needSystemd(config.HypervisorConfig) {
assert.NotEmpty(config.HypervisorConfig.KernelParams)
}
}
func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) {
assert := assert.New(t)
initName := "init"
initValue := "/sbin/myinit"
ipName := "ip"
ipValue := "127.0.0.1"
params := []vc.Param{
{Key: initName, Value: initValue},
{Key: ipName, Value: ipValue},
}
hypervisorConfig := vc.HypervisorConfig{
KernelParams: params,
}
// Config containing user-specified kernel parameters
config := oci.RuntimeConfig{
HypervisorConfig: hypervisorConfig,
}
assert.NotEmpty(config.HypervisorConfig.KernelParams)
err := SetKernelParams(testContainerID, &config)
assert.NoError(err)
kernelParams := config.HypervisorConfig.KernelParams
init, err := findLastParam(initName, kernelParams)
assert.NoError(err)
assert.Equal(initValue, init)
ip, err := findLastParam(ipName, kernelParams)
assert.NoError(err)
assert.Equal(ipValue, ip)
}
func TestCreateSandboxConfigFail(t *testing.T) {
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
assert.NoError(err)
bundlePath := filepath.Join(tmpdir, "bundle")
err = makeOCIBundle(bundlePath)
assert.NoError(err)
ociConfigFile := filepath.Join(bundlePath, "config.json")
assert.True(FileExists(ociConfigFile))
spec, err := readOCIConfigFile(ociConfigFile)
assert.NoError(err)
quota := int64(0)
limit := int64(0)
spec.Linux.Resources.Memory = &specs.LinuxMemory{
Limit: &limit,
}
spec.Linux.Resources.CPU = &specs.LinuxCPU{
// specify an invalid value
Quota: &quota,
}
_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false)
assert.Error(err)
}
func TestCreateSandboxFail(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true)
assert.NoError(err)
bundlePath := filepath.Join(tmpdir, "bundle")
err = makeOCIBundle(bundlePath)
assert.NoError(err)
ociConfigFile := filepath.Join(bundlePath, "config.json")
assert.True(FileExists(ociConfigFile))
spec, err := readOCIConfigFile(ociConfigFile)
assert.NoError(err)
_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false)
assert.Error(err)
assert.True(vcmock.IsMockError(err))
}
func TestCreateContainerContainerConfigFail(t *testing.T) {
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
bundlePath := filepath.Join(tmpdir, "bundle")
err = makeOCIBundle(bundlePath)
assert.NoError(err)
ociConfigFile := filepath.Join(bundlePath, "config.json")
assert.True(FileExists(ociConfigFile))
spec, err := readOCIConfigFile(ociConfigFile)
assert.NoError(err)
// Set invalid container type
containerType := "你好,世界"
spec.Annotations = make(map[string]string)
spec.Annotations[testContainerTypeAnnotation] = containerType
// rewrite file
err = writeOCIConfigFile(spec, ociConfigFile)
assert.NoError(err)
for _, disableOutput := range []bool{true, false} {
_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
assert.Error(err)
assert.False(vcmock.IsMockError(err))
assert.True(strings.Contains(err.Error(), containerType))
os.RemoveAll(path)
}
}
func TestCreateContainerFail(t *testing.T) {
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
bundlePath := filepath.Join(tmpdir, "bundle")
err = makeOCIBundle(bundlePath)
assert.NoError(err)
ociConfigFile := filepath.Join(bundlePath, "config.json")
assert.True(FileExists(ociConfigFile))
spec, err := readOCIConfigFile(ociConfigFile)
assert.NoError(err)
// set expected container type and sandboxID
spec.Annotations = make(map[string]string)
spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
spec.Annotations[testSandboxIDAnnotation] = testSandboxID
// rewrite file
err = writeOCIConfigFile(spec, ociConfigFile)
assert.NoError(err)
for _, disableOutput := range []bool{true, false} {
_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
assert.Error(err)
assert.True(vcmock.IsMockError(err))
os.RemoveAll(path)
}
}
func TestCreateContainer(t *testing.T) {
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) {
return &vcmock.Sandbox{}, &vcmock.Container{}, nil
}
defer func() {
testingImpl.CreateContainerFunc = nil
}()
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
bundlePath := filepath.Join(tmpdir, "bundle")
err = makeOCIBundle(bundlePath)
assert.NoError(err)
ociConfigFile := filepath.Join(bundlePath, "config.json")
assert.True(FileExists(ociConfigFile))
spec, err := readOCIConfigFile(ociConfigFile)
assert.NoError(err)
// set expected container type and sandboxID
spec.Annotations = make(map[string]string)
spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer
spec.Annotations[testSandboxIDAnnotation] = testSandboxID
// rewrite file
err = writeOCIConfigFile(spec, ociConfigFile)
assert.NoError(err)
for _, disableOutput := range []bool{true, false} {
_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
assert.NoError(err)
os.RemoveAll(path)
}
}

142
pkg/katautils/hook.go Normal file
View File

@@ -0,0 +1,142 @@
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
"syscall"
"time"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/opentracing/opentracing-go/log"
"github.com/sirupsen/logrus"
)
// Logger returns a logrus logger appropriate for logging hook messages
func hookLogger() *logrus.Entry {
return kataUtilsLogger.WithField("subsystem", "hook")
}
func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
span, _ := Trace(ctx, "hook")
defer span.Finish()
span.SetTag("subsystem", "runHook")
span.LogFields(
log.String("hook-name", hook.Path),
log.String("hook-args", strings.Join(hook.Args, " ")))
state := specs.State{
Pid: os.Getpid(),
Bundle: bundlePath,
ID: cid,
}
stateJSON, err := json.Marshal(state)
if err != nil {
return err
}
var stdout, stderr bytes.Buffer
cmd := &exec.Cmd{
Path: hook.Path,
Args: hook.Args,
Env: hook.Env,
Stdin: bytes.NewReader(stateJSON),
Stdout: &stdout,
Stderr: &stderr,
}
if err := cmd.Start(); err != nil {
return err
}
if hook.Timeout == nil {
if err := cmd.Wait(); err != nil {
return fmt.Errorf("%s: stdout: %s, stderr: %s", err, stdout.String(), stderr.String())
}
} else {
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
close(done)
}()
select {
case err := <-done:
if err != nil {
return fmt.Errorf("%s: stdout: %s, stderr: %s", err, stdout.String(), stderr.String())
}
case <-time.After(time.Duration(*hook.Timeout) * time.Second):
if err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL); err != nil {
return err
}
return fmt.Errorf("Hook timeout")
}
}
return nil
}
func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
span, _ := Trace(ctx, "hooks")
defer span.Finish()
span.SetTag("subsystem", hookType)
for _, hook := range hooks {
if err := runHook(ctx, hook, cid, bundlePath); err != nil {
hookLogger().WithFields(logrus.Fields{
"hook-type": hookType,
"error": err,
}).Error("hook error")
return err
}
}
return nil
}
// PreStartHooks run the hooks before start container
func PreStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
// If no hook available, nothing needs to be done.
if spec.Hooks == nil {
return nil
}
return runHooks(ctx, spec.Hooks.Prestart, cid, bundlePath, "pre-start")
}
// PostStartHooks run the hooks just after start container
func PostStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
// If no hook available, nothing needs to be done.
if spec.Hooks == nil {
return nil
}
return runHooks(ctx, spec.Hooks.Poststart, cid, bundlePath, "post-start")
}
// PostStopHooks run the hooks after stop container
func PostStopHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error {
// If no hook available, nothing needs to be done.
if spec.Hooks == nil {
return nil
}
return runHooks(ctx, spec.Hooks.Poststop, cid, bundlePath, "post-stop")
}

230
pkg/katautils/hook_test.go Normal file
View File

@@ -0,0 +1,230 @@
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"context"
"os"
"testing"
. "github.com/kata-containers/runtime/virtcontainers/pkg/mock"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
)
// Important to keep these values in sync with hook test binary
var testKeyHook = "test-key"
var testContainerIDHook = "test-container-id"
var testControllerIDHook = "test-controller-id"
var testBinHookPath = "/usr/bin/virtcontainers/bin/test/hook"
var testBundlePath = "/test/bundle"
func getMockHookBinPath() string {
if DefaultMockHookBinPath == "" {
return testBinHookPath
}
return DefaultMockHookBinPath
}
func createHook(timeout int) specs.Hook {
to := &timeout
if timeout == 0 {
to = nil
}
return specs.Hook{
Path: getMockHookBinPath(),
Args: []string{testKeyHook, testContainerIDHook, testControllerIDHook},
Env: os.Environ(),
Timeout: to,
}
}
func createWrongHook() specs.Hook {
return specs.Hook{
Path: getMockHookBinPath(),
Args: []string{"wrong-args"},
Env: os.Environ(),
}
}
func TestRunHook(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
ctx := context.Background()
// Run with timeout 0
hook := createHook(0)
err := runHook(ctx, hook, testSandboxID, testBundlePath)
assert.NoError(err)
// Run with timeout 1
hook = createHook(1)
err = runHook(ctx, hook, testSandboxID, testBundlePath)
assert.NoError(err)
// Run timeout failure
hook = createHook(1)
hook.Args = append(hook.Args, "2")
err = runHook(ctx, hook, testSandboxID, testBundlePath)
assert.Error(err)
// Failure due to wrong hook
hook = createWrongHook()
err = runHook(ctx, hook, testSandboxID, testBundlePath)
assert.Error(err)
}
func TestPreStartHooks(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
ctx := context.Background()
// Hooks field is nil
spec := oci.CompatOCISpec{}
err := PreStartHooks(ctx, spec, "", "")
assert.NoError(err)
// Hooks list is empty
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{},
},
}
err = PreStartHooks(ctx, spec, "", "")
assert.NoError(err)
// Run with timeout 0
hook := createHook(0)
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{hook},
},
},
}
err = PreStartHooks(ctx, spec, testSandboxID, testBundlePath)
assert.NoError(err)
// Failure due to wrong hook
hook = createWrongHook()
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{
Prestart: []specs.Hook{hook},
},
},
}
err = PreStartHooks(ctx, spec, testSandboxID, testBundlePath)
assert.Error(err)
}
func TestPostStartHooks(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
ctx := context.Background()
// Hooks field is nil
spec := oci.CompatOCISpec{}
err := PostStartHooks(ctx, spec, "", "")
assert.NoError(err)
// Hooks list is empty
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{},
},
}
err = PostStartHooks(ctx, spec, "", "")
assert.NoError(err)
// Run with timeout 0
hook := createHook(0)
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{
Poststart: []specs.Hook{hook},
},
},
}
err = PostStartHooks(ctx, spec, testSandboxID, testBundlePath)
assert.NoError(err)
// Failure due to wrong hook
hook = createWrongHook()
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{
Poststart: []specs.Hook{hook},
},
},
}
err = PostStartHooks(ctx, spec, testSandboxID, testBundlePath)
assert.Error(err)
}
func TestPostStopHooks(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
ctx := context.Background()
// Hooks field is nil
spec := oci.CompatOCISpec{}
err := PostStopHooks(ctx, spec, "", "")
assert.NoError(err)
// Hooks list is empty
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{},
},
}
err = PostStopHooks(ctx, spec, "", "")
assert.NoError(err)
// Run with timeout 0
hook := createHook(0)
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{
Poststop: []specs.Hook{hook},
},
},
}
err = PostStopHooks(ctx, spec, testSandboxID, testBundlePath)
assert.NoError(err)
// Failure due to wrong hook
hook = createWrongHook()
spec = oci.CompatOCISpec{
Spec: specs.Spec{
Hooks: &specs.Hooks{
Poststop: []specs.Hook{hook},
},
},
}
err = PostStopHooks(ctx, spec, testSandboxID, testBundlePath)
assert.Error(err)
}

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0

181
pkg/katautils/network.go Normal file
View File

@@ -0,0 +1,181 @@
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"bufio"
"fmt"
"os"
"path/filepath"
goruntime "runtime"
"strings"
"github.com/containernetworking/plugins/pkg/ns"
vc "github.com/kata-containers/runtime/virtcontainers"
"golang.org/x/sys/unix"
)
const procMountInfoFile = "/proc/self/mountinfo"
// EnterNetNS is free from any call to a go routine, and it calls
// into runtime.LockOSThread(), meaning it won't be executed in a
// different thread than the one expected by the caller.
func EnterNetNS(netNSPath string, cb func() error) error {
if netNSPath == "" {
return cb()
}
goruntime.LockOSThread()
defer goruntime.UnlockOSThread()
currentNS, err := ns.GetCurrentNS()
if err != nil {
return err
}
defer currentNS.Close()
targetNS, err := ns.GetNS(netNSPath)
if err != nil {
return err
}
if err := targetNS.Set(); err != nil {
return err
}
defer currentNS.Set()
return cb()
}
// SetupNetworkNamespace create a network namespace
func SetupNetworkNamespace(config *vc.NetworkConfig) error {
if config.DisableNewNetNs {
kataUtilsLogger.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns")
return nil
}
if config.NetNSPath == "" {
n, err := ns.NewNS()
if err != nil {
return err
}
config.NetNSPath = n.Path()
config.NetNsCreated = true
return nil
}
isHostNs, err := hostNetworkingRequested(config.NetNSPath)
if err != nil {
return err
}
if isHostNs {
return fmt.Errorf("Host networking requested, not supported by runtime")
}
return nil
}
// getNetNsFromBindMount returns the network namespace for the bind-mounted path
func getNetNsFromBindMount(nsPath string, procMountFile string) (string, error) {
netNsMountType := "nsfs"
// Resolve all symlinks in the path as the mountinfo file contains
// resolved paths.
nsPath, err := filepath.EvalSymlinks(nsPath)
if err != nil {
return "", err
}
f, err := os.Open(procMountFile)
if err != nil {
return "", err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
text := scanner.Text()
// Scan the mountinfo file to search for the network namespace path
// This file contains mounts in the eg format:
// "711 26 0:3 net:[4026532009] /run/docker/netns/default rw shared:535 - nsfs nsfs rw"
//
// Reference: https://www.kernel.org/doc/Documentation/filesystems/proc.txt
// We are interested in the first 9 fields of this file,
// to check for the correct mount type.
fields := strings.Split(text, " ")
if len(fields) < 9 {
continue
}
// We check here if the mount type is a network namespace mount type, namely "nsfs"
mountTypeFieldIdx := 8
if fields[mountTypeFieldIdx] != netNsMountType {
continue
}
// This is the mount point/destination for the mount
mntDestIdx := 4
if fields[mntDestIdx] != nsPath {
continue
}
// This is the root/source of the mount
return fields[3], nil
}
return "", nil
}
// hostNetworkingRequested checks if the network namespace requested is the
// same as the current process.
func hostNetworkingRequested(configNetNs string) (bool, error) {
var evalNS, nsPath, currentNsPath string
var err error
// Net namespace provided as "/proc/pid/ns/net" or "/proc/<pid>/task/<tid>/ns/net"
if strings.HasPrefix(configNetNs, "/proc") && strings.HasSuffix(configNetNs, "/ns/net") {
if _, err := os.Stat(configNetNs); err != nil {
return false, err
}
// Here we are trying to resolve the path but it fails because
// namespaces links don't really exist. For this reason, the
// call to EvalSymlinks will fail when it will try to stat the
// resolved path found. As we only care about the path, we can
// retrieve it from the PathError structure.
if _, err = filepath.EvalSymlinks(configNetNs); err != nil {
nsPath = err.(*os.PathError).Path
} else {
return false, fmt.Errorf("Net namespace path %s is not a symlink", configNetNs)
}
_, evalNS = filepath.Split(nsPath)
} else {
// Bind-mounted path provided
evalNS, _ = getNetNsFromBindMount(configNetNs, procMountInfoFile)
}
currentNS := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
if _, err = filepath.EvalSymlinks(currentNS); err != nil {
currentNsPath = err.(*os.PathError).Path
} else {
return false, fmt.Errorf("Unexpected: Current network namespace path is not a symlink")
}
_, evalCurrentNS := filepath.Split(currentNsPath)
if evalNS == evalCurrentNS {
return true, nil
}
return false, nil
}

View File

@@ -0,0 +1,150 @@
// Copyright (c) 2018 Huawei Corporation.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"syscall"
"testing"
"github.com/containernetworking/plugins/pkg/ns"
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/stretchr/testify/assert"
"golang.org/x/sys/unix"
)
func TestGetNetNsFromBindMount(t *testing.T) {
assert := assert.New(t)
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
mountFile := filepath.Join(tmpdir, "mountInfo")
nsPath := filepath.Join(tmpdir, "ns123")
// Non-existent namespace path
_, err = getNetNsFromBindMount(nsPath, mountFile)
assert.NotNil(err)
tmpNSPath := filepath.Join(tmpdir, "testNetNs")
f, err := os.Create(tmpNSPath)
assert.NoError(err)
defer f.Close()
type testData struct {
contents string
expectedResult string
}
data := []testData{
{fmt.Sprintf("711 26 0:3 net:[4026532008] %s rw shared:535 - nsfs nsfs rw", tmpNSPath), "net:[4026532008]"},
{"711 26 0:3 net:[4026532008] /run/netns/ns123 rw shared:535 - tmpfs tmpfs rw", ""},
{"a a a a a a a - b c d", ""},
{"", ""},
}
for i, d := range data {
err := ioutil.WriteFile(mountFile, []byte(d.contents), 0640)
assert.NoError(err)
path, err := getNetNsFromBindMount(tmpNSPath, mountFile)
assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", path, d))
assert.Equal(d.expectedResult, path, "Test %d, expected %s, got %s", i, d.expectedResult, path)
}
}
func TestHostNetworkingRequested(t *testing.T) {
assert := assert.New(t)
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedRoot)
}
// Network namespace same as the host
selfNsPath := "/proc/self/ns/net"
isHostNs, err := hostNetworkingRequested(selfNsPath)
assert.NoError(err)
assert.True(isHostNs)
// Non-existent netns path
nsPath := "/proc/123456789/ns/net"
_, err = hostNetworkingRequested(nsPath)
assert.Error(err)
// Bind-mounted Netns
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
// Create a bind mount to the current network namespace.
tmpFile := filepath.Join(tmpdir, "testNetNs")
f, err := os.Create(tmpFile)
assert.NoError(err)
defer f.Close()
err = syscall.Mount(selfNsPath, tmpFile, "bind", syscall.MS_BIND, "")
assert.Nil(err)
isHostNs, err = hostNetworkingRequested(tmpFile)
assert.NoError(err)
assert.True(isHostNs)
syscall.Unmount(tmpFile, 0)
}
func TestSetupNetworkNamespace(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
// Network namespace same as the host
config := &vc.NetworkConfig{
NetNSPath: "/proc/self/ns/net",
}
err := SetupNetworkNamespace(config)
assert.Error(err)
// Non-existent netns path
config = &vc.NetworkConfig{
NetNSPath: "/proc/123456789/ns/net",
}
err = SetupNetworkNamespace(config)
assert.Error(err)
// Existent netns path
n, err := ns.NewNS()
assert.NoError(err)
config = &vc.NetworkConfig{
NetNSPath: n.Path(),
}
err = SetupNetworkNamespace(config)
assert.NoError(err)
n.Close()
// Empty netns path
config = &vc.NetworkConfig{}
err = SetupNetworkNamespace(config)
assert.NoError(err)
n, err = ns.GetNS(config.NetNSPath)
assert.NoError(err)
assert.NotNil(n)
assert.True(config.NetNsCreated)
n.Close()
unix.Unmount(config.NetNSPath, unix.MNT_DETACH)
os.RemoveAll(config.NetNSPath)
// Config with DisableNewNetNs
config = &vc.NetworkConfig{DisableNewNetNs: true}
err = SetupNetworkNamespace(config)
assert.NoError(err)
}

92
pkg/katautils/oci.go Normal file
View File

@@ -0,0 +1,92 @@
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
)
const ctrsMappingDirMode = os.FileMode(0750)
var ctrsMapTreePath = "/var/run/kata-containers/containers-mapping"
// SetCtrsMapTreePath let the testcases change the ctrsMapTreePath to a test dir
func SetCtrsMapTreePath(path string) {
ctrsMapTreePath = path
}
// FetchContainerIDMapping This function assumes it should find only one file inside the container
// ID directory. If there are several files, we could not determine which
// file name corresponds to the sandbox ID associated, and this would throw
// an error.
func FetchContainerIDMapping(containerID string) (string, error) {
if containerID == "" {
return "", fmt.Errorf("Missing container ID")
}
dirPath := filepath.Join(ctrsMapTreePath, containerID)
files, err := ioutil.ReadDir(dirPath)
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
if len(files) != 1 {
return "", fmt.Errorf("Too many files (%d) in %q", len(files), dirPath)
}
return files[0].Name(), nil
}
// AddContainerIDMapping add a container id mapping to sandbox id
func AddContainerIDMapping(ctx context.Context, containerID, sandboxID string) error {
span, _ := Trace(ctx, "addContainerIDMapping")
defer span.Finish()
if containerID == "" {
return fmt.Errorf("Missing container ID")
}
if sandboxID == "" {
return fmt.Errorf("Missing sandbox ID")
}
parentPath := filepath.Join(ctrsMapTreePath, containerID)
if err := os.RemoveAll(parentPath); err != nil {
return err
}
path := filepath.Join(parentPath, sandboxID)
if err := os.MkdirAll(path, ctrsMappingDirMode); err != nil {
return err
}
return nil
}
// DelContainerIDMapping delete container id mapping from a sandbox
func DelContainerIDMapping(ctx context.Context, containerID string) error {
span, _ := Trace(ctx, "delContainerIDMapping")
defer span.Finish()
if containerID == "" {
return fmt.Errorf("Missing container ID")
}
path := filepath.Join(ctrsMapTreePath, containerID)
return os.RemoveAll(path)
}

135
pkg/katautils/oci_test.go Normal file
View File

@@ -0,0 +1,135 @@
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func createTempContainerIDMapping(containerID, sandboxID string) (string, error) {
tmpDir, err := ioutil.TempDir("", "containers-mapping")
if err != nil {
return "", err
}
ctrsMapTreePath = tmpDir
path := filepath.Join(ctrsMapTreePath, containerID, sandboxID)
if err := os.MkdirAll(path, 0750); err != nil {
return "", err
}
return tmpDir, nil
}
func TestFetchContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
assert := assert.New(t)
sandboxID, err := FetchContainerIDMapping("")
assert.Error(err)
assert.Empty(sandboxID)
}
func TestFetchContainerIDMappingEmptyMappingSuccess(t *testing.T) {
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
sandboxID, err := FetchContainerIDMapping(testContainerID)
assert.NoError(err)
assert.Empty(sandboxID)
}
func TestFetchContainerIDMappingTooManyFilesFailure(t *testing.T) {
assert := assert.New(t)
path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
assert.NoError(err)
defer os.RemoveAll(path)
err = os.MkdirAll(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID+"2"), ctrsMappingDirMode)
assert.NoError(err)
sandboxID, err := FetchContainerIDMapping(testContainerID)
assert.Error(err)
assert.Empty(sandboxID)
}
func TestFetchContainerIDMappingSuccess(t *testing.T) {
assert := assert.New(t)
path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
assert.NoError(err)
defer os.RemoveAll(path)
sandboxID, err := FetchContainerIDMapping(testContainerID)
assert.NoError(err)
assert.Equal(sandboxID, testSandboxID)
}
func TestAddContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
assert := assert.New(t)
err := AddContainerIDMapping(context.Background(), "", testSandboxID)
assert.Error(err)
}
func TestAddContainerIDMappingSandboxIDEmptyFailure(t *testing.T) {
assert := assert.New(t)
err := AddContainerIDMapping(context.Background(), testContainerID, "")
assert.Error(err)
}
func TestAddContainerIDMappingSuccess(t *testing.T) {
assert := assert.New(t)
path, err := ioutil.TempDir("", "containers-mapping")
assert.NoError(err)
defer os.RemoveAll(path)
ctrsMapTreePath = path
_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
assert.True(os.IsNotExist(err))
err = AddContainerIDMapping(context.Background(), testContainerID, testSandboxID)
assert.NoError(err)
_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
assert.NoError(err)
}
func TestDelContainerIDMappingContainerIDEmptyFailure(t *testing.T) {
assert := assert.New(t)
err := DelContainerIDMapping(context.Background(), "")
assert.Error(err)
}
func TestDelContainerIDMappingSuccess(t *testing.T) {
assert := assert.New(t)
path, err := createTempContainerIDMapping(testContainerID, testSandboxID)
assert.NoError(err)
defer os.RemoveAll(path)
_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
assert.NoError(err)
err = DelContainerIDMapping(context.Background(), testContainerID)
assert.NoError(err)
_, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID))
assert.True(os.IsNotExist(err))
}

92
pkg/katautils/tracing.go Normal file
View File

@@ -0,0 +1,92 @@
// Copyright (c) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
package katautils
import (
"context"
"io"
opentracing "github.com/opentracing/opentracing-go"
"github.com/uber/jaeger-client-go/config"
)
// Implements jaeger-client-go.Logger interface
type traceLogger struct {
}
// tracerCloser contains a copy of the closer returned by createTracer() which
// is used by stopTracing().
var tracerCloser io.Closer
func (t traceLogger) Error(msg string) {
kataUtilsLogger.Error(msg)
}
func (t traceLogger) Infof(msg string, args ...interface{}) {
kataUtilsLogger.Infof(msg, args...)
}
// CreateTracer create a tracer
func CreateTracer(name string) (opentracing.Tracer, error) {
cfg := &config.Configuration{
ServiceName: name,
// If tracing is disabled, use a NOP trace implementation
Disabled: !tracing,
// Note that span logging reporter option cannot be enabled as
// it pollutes the output stream which causes (atleast) the
// "state" command to fail under Docker.
Sampler: &config.SamplerConfig{
Type: "const",
Param: 1,
},
}
logger := traceLogger{}
tracer, closer, err := cfg.NewTracer(config.Logger(logger))
if err != nil {
return nil, err
}
// save for stopTracing()'s exclusive use
tracerCloser = closer
// Seems to be essential to ensure non-root spans are logged
opentracing.SetGlobalTracer(tracer)
return tracer, nil
}
// StopTracing ends all tracing, reporting the spans to the collector.
func StopTracing(ctx context.Context) {
if !tracing {
return
}
span := opentracing.SpanFromContext(ctx)
if span != nil {
span.Finish()
}
// report all possible spans to the collector
if tracerCloser != nil {
tracerCloser.Close()
}
}
// Trace creates a new tracing span based on the specified name and parent
// context.
func Trace(parent context.Context, name string) (opentracing.Span, context.Context) {
span, ctx := opentracing.StartSpanFromContext(parent, name)
span.SetTag("source", "runtime")
span.SetTag("component", "cli")
return span, ctx
}

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
@@ -10,10 +10,45 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
)
const (
k8sEmptyDir = "kubernetes.io~empty-dir"
)
// FileExists test is a file exiting or not
func FileExists(path string) bool {
if _, err := os.Stat(path); os.IsNotExist(err) {
return false
}
return true
}
// IsEphemeralStorage returns true if the given path
// to the storage belongs to kubernetes ephemeral storage
//
// This method depends on a specific path used by k8s
// to detect if it's of type ephemeral. As of now,
// this is a very k8s specific solution that works
// but in future there should be a better way for this
// method to determine if the path is for ephemeral
// volume type
func IsEphemeralStorage(path string) bool {
splitSourceSlice := strings.Split(path, "/")
if len(splitSourceSlice) > 1 {
storageType := splitSourceSlice[len(splitSourceSlice)-2]
if storageType == k8sEmptyDir {
return true
}
}
return false
}
// ResolvePath returns the fully resolved and expanded value of the
// specified path.
func ResolvePath(path string) (string, error) {
@@ -75,3 +110,27 @@ func GetFileContents(file string) (string, error) {
return string(bytes), nil
}
// RunCommandFull returns the commands space-trimmed standard output and
// error on success. Note that if the command fails, the requested output will
// still be returned, along with an error.
func RunCommandFull(args []string, includeStderr bool) (string, error) {
cmd := exec.Command(args[0], args[1:]...)
var err error
var bytes []byte
if includeStderr {
bytes, err = cmd.CombinedOutput()
} else {
bytes, err = cmd.Output()
}
trimmed := strings.TrimSpace(string(bytes))
return trimmed, err
}
// RunCommand returns the commands space-trimmed standard output on success
func RunCommand(args []string) (string, error) {
return RunCommandFull(args, false)
}

View File

@@ -1,4 +1,4 @@
// Copyright (c) 2017 Intel Corporation
// Copyright (c) 2018 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
@@ -7,14 +7,18 @@
package katautils
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"syscall"
"testing"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/stretchr/testify/assert"
)
@@ -22,7 +26,16 @@ const (
testDirMode = os.FileMode(0750)
testFileMode = os.FileMode(0640)
testDisabledNeedRoot = "Test disabled as requires root user"
testDisabledNeedNonRoot = "Test disabled as requires non-root user"
// small docker image used to create root filesystems from
testDockerImage = "busybox"
testSandboxID = "99999999-9999-9999-99999999999999999"
testContainerID = "1"
testBundle = "bundle"
specConfig = "config.json"
)
var testDir = ""
@@ -37,6 +50,148 @@ func init() {
}
fmt.Printf("INFO: test directory is %v\n", testDir)
testBundleDir = filepath.Join(testDir, testBundle)
err = os.MkdirAll(testBundleDir, testDirMode)
if err != nil {
panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err))
}
fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir)
err = realMakeOCIBundle(testBundleDir)
if err != nil {
panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err))
}
}
// createOCIConfig creates an OCI configuration (spec) file in
// the bundle directory specified (which must exist).
func createOCIConfig(bundleDir string) error {
if bundleDir == "" {
return errors.New("BUG: Need bundle directory")
}
if !FileExists(bundleDir) {
return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir)
}
var configCmd string
// Search for a suitable version of runc to use to generate
// the OCI config file.
for _, cmd := range []string{"docker-runc", "runc"} {
fullPath, err := exec.LookPath(cmd)
if err == nil {
configCmd = fullPath
break
}
}
if configCmd == "" {
return errors.New("Cannot find command to generate OCI config file")
}
_, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
if err != nil {
return err
}
specFile := filepath.Join(bundleDir, specConfig)
if !FileExists(specFile) {
return fmt.Errorf("generated OCI config file does not exist: %v", specFile)
}
return nil
}
// realMakeOCIBundle will create an OCI bundle (including the "config.json"
// config file) in the directory specified (which must already exist).
//
// XXX: Note that tests should *NOT* call this function - they should
// XXX: instead call makeOCIBundle().
func realMakeOCIBundle(bundleDir string) error {
if bundleDir == "" {
return errors.New("BUG: Need bundle directory")
}
if !FileExists(bundleDir) {
return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir)
}
err := createOCIConfig(bundleDir)
if err != nil {
return err
}
// Note the unusual parameter (a directory, not the config
// file to parse!)
spec, err := oci.ParseConfigJSON(bundleDir)
if err != nil {
return err
}
// Determine the rootfs directory name the OCI config refers to
ociRootPath := spec.Root.Path
rootfsDir := filepath.Join(bundleDir, ociRootPath)
if strings.HasPrefix(ociRootPath, "/") {
return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test")
}
err = createRootfs(rootfsDir)
if err != nil {
return err
}
return nil
}
// createRootfs creates a minimal root filesystem below the specified
// directory.
func createRootfs(dir string) error {
err := os.MkdirAll(dir, testDirMode)
if err != nil {
return err
}
container, err := RunCommand([]string{"docker", "create", testDockerImage})
if err != nil {
return err
}
cmd1 := exec.Command("docker", "export", container)
cmd2 := exec.Command("tar", "-C", dir, "-xvf", "-")
cmd1Stdout, err := cmd1.StdoutPipe()
if err != nil {
return err
}
cmd2.Stdin = cmd1Stdout
err = cmd2.Start()
if err != nil {
return err
}
err = cmd1.Run()
if err != nil {
return err
}
err = cmd2.Wait()
if err != nil {
return err
}
// Clean up
_, err = RunCommand([]string{"docker", "rm", container})
if err != nil {
return err
}
return nil
}
func createFile(file, contents string) error {
@@ -211,3 +366,17 @@ func TestGetFileContents(t *testing.T) {
assert.Equal(t, contents, d.contents)
}
}
func TestIsEphemeralStorage(t *testing.T) {
sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume"
isEphe := IsEphemeralStorage(sampleEphePath)
if !isEphe {
t.Fatalf("Unable to correctly determine volume type")
}
sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume"
isEphe = IsEphemeralStorage(sampleEphePath)
if isEphe {
t.Fatalf("Unable to correctly determine volume type")
}
}