aljaz/kata-containers
1
0
Fork 0
mirror of https://github.com/aljazceru/kata-containers.git synced 2025-12-23 01:04:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

virtcontainers: Initial import

Browse Source 
This is a virtcontainers 1.0.8 import into Kata Containers runtime.

virtcontainers is a Go library designed to manage hardware virtualized
pods and containers. It is the core Clear Containers framework and will
become the core Kata Containers framework, as discussed at
https://github.com/kata-containers/runtime/issues/33

Some more more pointers:

virtcontainers README, including some design and architecure notes:
https://github.com/containers/virtcontainers/blob/master/README.md

virtcontainers 1.0 API:
https://github.com/containers/virtcontainers/blob/master/documentation/api/1.0/api.md

Fixes #40

Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>
This commit is contained in:
Samuel Ortiz
2018-02-23 10:35:08 +01:00
parent ad9cef49c6
commit 24eff72d82
854 changed files with 441776 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
virtcontainers/.ci/go-lint.sh Executable file
View File

@@ -0,0 +1,46 @@
# Copyright (c) 2017 Intel Corporation
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#!/bin/bash
if [ ! $(command -v gometalinter) ]
then
go get github.com/alecthomas/gometalinter
gometalinter --install --vendor
fi
linter_args="--tests --vendor"
# When running the linters in a CI environment we need to disable them all
# by default and then explicitly enable the ones we are care about. This is
# necessary since *if* gometalinter adds a new linter, that linter may cause
# the CI build to fail when it really shouldn't. However, when this script is
# run locally, all linters should be run to allow the developer to review any
# failures (and potentially decide whether we need to explicitly enable a new
# linter in the CI).
if [ "$CI" = true ]; then
linter_args+=" --disable-all"
fi
linter_args+=" --enable=misspell"
linter_args+=" --enable=vet"
linter_args+=" --enable=ineffassign"
linter_args+=" --enable=gofmt"
linter_args+=" --enable=gocyclo"
linter_args+=" --cyclo-over=15"
linter_args+=" --enable=golint"
linter_args+=" --deadline=600s"
eval gometalinter "${linter_args}" ./...

31
virtcontainers/.ci/go-test.sh Executable file
View File

@@ -0,0 +1,31 @@
# Copyright (c) 2017 Intel Corporation
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#!/bin/bash
set -e
test_packages=$(go list ./... | grep -v vendor)
test_ldflags="-X github.com/containers/virtcontainers/pkg/mock.DefaultMockCCShimBinPath=$1 \
-X github.com/containers/virtcontainers/pkg/mock.DefaultMockKataShimBinPath=$2 \
-X github.com/containers/virtcontainers/pkg/mock.DefaultMockHookBinPath=$3"
echo "Run go test and generate coverage:"
for pkg in $test_packages; do
if [ "$pkg" = "github.com/containers/virtcontainers" ]; then
sudo env GOPATH=$GOPATH GOROOT=$GOROOT PATH=$PATH go test -ldflags "$test_ldflags" -cover -coverprofile=profile.cov $pkg
else
sudo env GOPATH=$GOPATH GOROOT=$GOROOT PATH=$PATH go test -ldflags "$test_ldflags" -cover $pkg
fi
done

55
virtcontainers/.ci/jenkins_job_build.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
vc_repo="github.com/containers/virtcontainers"
# Export all environment variables needed.
export GOROOT="/usr/local/go"
export GOPATH=${HOME}/go
export PATH=${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:${PATH}
export CI=true
# Download and build goveralls binary in case we need to submit the code
# coverage.
if [ ${COVERALLS_REPO_TOKEN} ]
then
go get github.com/mattn/goveralls
fi
# Get the repository and move HEAD to the appropriate commit.
go get ${vc_repo} || true
cd "${GOPATH}/src/${vc_repo}"
if [ "${ghprbPullId}" ] && [ "${ghprbTargetBranch}" ]
then
git fetch origin "pull/${ghprbPullId}/head" && git checkout master && git reset --hard FETCH_HEAD && git rebase "origin/${ghprbTargetBranch}"
export AUTHOR_REPO_GIT_URL="${ghprbAuthorRepoGitUrl}"
export COMMIT_REVISION="${ghprbActualCommit}"
else
git fetch origin && git checkout master && git reset --hard origin/master
fi
# Setup environment and run the tests
sudo -E PATH=$PATH bash .ci/setup.sh
sudo -E PATH=$PATH bash .ci/run.sh
# Publish the code coverage if needed.
if [ ${COVERALLS_REPO_TOKEN} ]
then
sudo -E PATH=${PATH} bash -c "${GOPATH}/bin/goveralls -repotoken=${COVERALLS_REPO_TOKEN} -coverprofile=profile.cov"
fi

33
virtcontainers/.ci/run.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
export CI=true
tests_repo="github.com/clearcontainers/tests"
make check
sudo -E PATH=$PATH sh -c "go test -bench=."
sudo -E PATH=$PATH sh -c "go test -bench=CreateStartStopDeletePodQemuHypervisorNoopAgentNetworkCNI -benchtime=60s"
sudo -E PATH=$PATH sh -c "go test -bench=CreateStartStopDeletePodQemuHypervisorHyperstartAgentNetworkCNI -benchtime=60s"
pushd "${GOPATH}/src/${tests_repo}"
.ci/run.sh
popd

41
virtcontainers/.ci/setup.sh Executable file
View File

@@ -0,0 +1,41 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
cidir=$(dirname "$0")
tests_repo="github.com/clearcontainers/tests"
# Clone Tests repository.
go get "$tests_repo"
tests_repo_dir="${GOPATH}/src/${tests_repo}"
echo "Update proxy and runtime vendoring"
sudo -E PATH=$PATH bash -c "${cidir}/update-vendoring.sh"
pushd "${tests_repo_dir}"
echo "Setup Clear Containers"
sudo -E PATH=$PATH bash -c ".ci/setup.sh"
popd
echo "Setup virtcontainers environment"
chronic sudo -E PATH=$PATH bash -c "${cidir}/../utils/virtcontainers-setup.sh"
echo "Install virtcontainers"
chronic make
chronic sudo make install

89
virtcontainers/.ci/update-vendoring.sh Executable file
View File

@@ -0,0 +1,89 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is used to update the virtcontainers code in the vendor
# directories of the proxy and runtime repositories.
set -e
proxy_repo="github.com/clearcontainers/proxy"
runtime_repo="github.com/clearcontainers/runtime"
virtcontainers_repo="github.com/containers/virtcontainers"
function apply_depends_on(){
pushd "${GOPATH}/src/${virtcontainers_repo}"
label_lines=$(git log --format=%s%b -n 1 | grep "Depends-on:" || true)
if [ "${label_lines}" == "" ]; then
return 0
fi
nb_lines=$(echo ${label_lines} | wc -l)
for i in `seq 1 ${nb_lines}`
do
label_line=$(echo $label_lines | sed "${i}q;d")
label_str=$(echo "${label_line}" | cut -d' ' -f2)
repo=$(echo "${label_str}" | cut -d'#' -f1)
pr_id=$(echo "${label_str}" | cut -d'#' -f2)
if [ ! -d "${GOPATH}/src/${repo}" ]; then
go get -d "$repo" || true
fi
pushd "${GOPATH}/src/${repo}"
git fetch origin "pull/${pr_id}/head" && git checkout FETCH_HEAD && git rebase origin/master
popd
done
popd
}
function install_dep(){
go get -u github.com/golang/dep/cmd/dep
}
function update_repo(){
if [ "${AUTHOR_REPO_GIT_URL}" ] && [ "${COMMIT_REVISION}" ]
then
repo="$1"
if [ ! -d "${GOPATH}/src/${repo}" ]; then
go get -d "$repo" || true
fi
pushd "${GOPATH}/src/${repo}"
# Update Gopkg.toml
cat >> Gopkg.toml <<EOF
[[override]]
name = "${virtcontainers_repo}"
source = "${AUTHOR_REPO_GIT_URL}"
revision = "${COMMIT_REVISION}"
EOF
# Update the whole vendoring
dep ensure && dep ensure -update "${virtcontainers_repo}" && dep prune
popd
fi
}
apply_depends_on
install_dep
update_repo "${proxy_repo}"
update_repo "${runtime_repo}"

10
virtcontainers/.gitignore vendored Normal file
View File

@@ -0,0 +1,10 @@
*.patch
*.o
*.swp
/hack/virtc/virtc
/hook/mock/hook
/shim/mock/shim
/shim/mock/cc-shim/cc-shim
/shim/mock/kata-shim/kata-shim
/utils/supportfiles
profile.cov

52
virtcontainers/.pullapprove.yml Normal file
View File

@@ -0,0 +1,52 @@
version: 2
requirements:
signed_off_by:
required: true
# Disallow approval of PRs still under development
always_pending:
title_regex: 'WIP'
labels:
- do-not-merge
- wip
explanation: 'Work in progress - do not merge'
group_defaults:
approve_by_comment:
enabled: true
approve_regex: '^(LGTM|lgtm|Approved|\+1|:\+1:)'
reject_regex: '^(Rejected|-1|:-1:)'
reset_on_push:
enabled: false
reset_on_reopened:
enabled: false
author_approval:
ignored: true
groups:
networking:
conditions:
files:
- pkg/cni/*
- cni*
- cnm*
required: 1
users:
- mcastelino
- sboeuf
approvers:
required: 1
users:
- sameo
- sboeuf
- amshinde
- jodh-intel
reviewers:
required: 1
teams:
- virtcontainers-maintainers
users:
- amshinde

74
virtcontainers/CODE_OF_CONDUCT.md Normal file
View File

@@ -0,0 +1,74 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at
clear-containers-code-of-conduct@clearlinux.org. All complaints will be
reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct/
[homepage]: https://www.contributor-covenant.org

127
virtcontainers/CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,127 @@
# Contributing to virtcontainers
virtcontainers is an open source project licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0).
## Coding Style (Go)
The usual Go style, enforced by `gofmt`, should be used. Additionally, the [Go
Code Review](https://github.com/golang/go/wiki/CodeReviewComments) document
contains a few common errors to be mindful of.
## Certificate of Origin
In order to get a clear contribution chain of trust we use the [signed-off-by language](https://01.org/community/signed-process)
used by the Linux kernel project.
## Patch format
Beside the signed-off-by footer, we expect each patch to comply with the following format:
```
Subsystem: Change summary (no longer than 75 characters)
More detailed explanation of your changes: Why and how.
Wrap it to 72 characters.
See:
http://chris.beams.io/posts/git-commit/
for some more good advice, and the Linux Kernel document:
https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/SubmittingPatches
Signed-off-by: <contributor@foo.com>
```
For example:
```
pod: Remove token from Cmd structure
The token and pid data will be hold by the new Process structure and
they are related to a container.
Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
```
Note, that the body of the message should not just be a continuation of the subject line, and is not used to extend the subject line beyond its length limit. They should stand alone as complete sentence and paragraphs.
It is recommended that each of your patches fixes one thing. Smaller patches are easier to review, and are thus more likely to be accepted and merged, and problems are more likely to be picked up during review.
### Breaking compatibility
In case the patch you submit will break virtcontainers CI, because Clear Containers runtime compatibility is tested through the CI, you have to specify which repository and which pull request it depends on.
Using a simple tag `Depends-on:` in your commit message will allow virtcontainers CI to run properly. Notice that this tag is parsed from the latest commit of the pull request.
For example:
```
pod: Remove token from Cmd structure
The token and pid data will be hold by the new Process structure and
they are related to a container.
Depends-on: github.com/clearcontainers/runtime#75
Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
```
In this example, we want our CI scripts to fetch the pull request 75 of the runtime repository.
## Pull requests
We accept [github pull requests](https://github.com/containers/virtcontainers/pulls).
Github has a basic introduction to the process [here](https://help.github.com/articles/using-pull-requests/).
When submitting your Pull Request (PR), treat the Pull Request message the same you would a patch message, including pre-fixing the title with a subsystem name. Github by default seems to copy the message from your first patch, which many times is appropriate, but please ensure your message is accurate and complete for the whole Pull Request, as it ends up in the git log as the merge message.
Your pull request may get some feedback and comments, and require some rework. The recommended procedure for reworking is to rework your branch to a new clean state and 'force push' it to your github. GitHub understands this action, and does sensible things in the online comment history. Do not pile patches on patches to rework your branch. Any relevant information from the github comments section should be re-worked into your patch set, as the ultimate place where your patches are documented is in the git log, and not in the github comments section.
For more information on github 'force push' workflows see [here](http://blog.adamspiers.org/2015/03/24/why-and-how-to-correctly-amend-github-pull-requests/).
It is perfectly fine for your Pull Request to contain more than one patch - use as many patches as you need to implement the Request (see the previously mentioned 'small patch' thoughts). Each Pull Request should only cover one topic - if you mix up different items in your patches or pull requests then you will most likely be asked to rework them.
## Reviews
Before your Pull Requests are merged into the main code base, they will be reviewed. Anybody can review any Pull Request and leave feedback (in fact, it is encouraged).
We use an 'acknowledge' system for people to note if they agree, or disagree, with a Pull Request. We utilise some automated systems that can spot common acknowledge patterns, which include placing any of these at the beginning of a comment line:
- LGTM
- lgtm
- +1
- Approve
### Project maintainers
The virtcontainers maintainers will be the ones accepting or rejecting any pull request. They are listed in the OWNERS files, and there can be one OWNERS file per directory.
The OWNERS files split maintainership into 2 categories: reviewers and approvers. All approvers also belong to the reviewers list and there must be one approval from one member of each list for a pull request to be merged.
Since approvers are also reviewers, they technically can approve a pull request without getting another reviewer's approval. However, it is their due diligence to rely on reviewers and should use their approval power only in very specific cases.
## Issue tracking
To report a bug that is not already documented, please [open an
issue in github](https://github.com/containers/virtcontainers/issues/new) so we all get
visibility on the problem and work toward resolution.
## Closing issues
You can either close issues manually by adding the fixing commit SHA1 to the issue
comments or by adding the `Fixes` keyword to your commit message:
```
pod: Remove token from Cmd structure
The token and pid data will be hold by the new Process structure and
they are related to a container.
Fixes #123
Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
```
Github will then automatically close that issue when parsing the
[commit message](https://help.github.com/articles/closing-issues-via-commit-messages/).

226
virtcontainers/Gopkg.lock generated Normal file
View File

@@ -0,0 +1,226 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/clearcontainers/proxy"
packages = [
"api",
"client"
]
revision = "1d2a6a3ea132a86abd0731408b7dc34f2fc17d55"
[[projects]]
name = "github.com/containerd/cri-containerd"
packages = ["pkg/annotations"]
revision = "3d382e2f5dabe3bae62ceb9ded56bdee847008ee"
[[projects]]
name = "github.com/containernetworking/cni"
packages = [
"libcni",
"pkg/invoke",
"pkg/types",
"pkg/types/020",
"pkg/types/current",
"pkg/version"
]
revision = "384d8c0b5288c25b9f1da901c66ea5155e6c567d"
[[projects]]
name = "github.com/containernetworking/plugins"
packages = ["pkg/ns"]
revision = "7f98c94613021d8b57acfa1a2f0c8d0f6fd7ae5a"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
name = "github.com/go-ini/ini"
packages = ["."]
revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd"
[[projects]]
name = "github.com/gogo/protobuf"
packages = [
"gogoproto",
"proto",
"protoc-gen-gogo/descriptor",
"sortkeys",
"types"
]
revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
[[projects]]
name = "github.com/golang/protobuf"
packages = [
"proto",
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp"
]
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
version = "v1.0.0"
[[projects]]
name = "github.com/intel/govmm"
packages = ["qemu"]
revision = "e87160f8ea39dfe558bf6761f74717d191d82493"
[[projects]]
name = "github.com/kata-containers/agent"
packages = [
"protocols/client",
"protocols/grpc"
]
revision = "33eecb2a445f906811a5bc9713d2dafd10768d18"
[[projects]]
name = "github.com/kubernetes-incubator/cri-o"
packages = ["pkg/annotations"]
revision = "3394b3b2d6af0e41d185bb695c6378be5dd4d61d"
[[projects]]
name = "github.com/mdlayher/vsock"
packages = ["."]
revision = "738c88d6e4cfd60e8124a5344fa10d205fd828b9"
[[projects]]
name = "github.com/mitchellh/mapstructure"
packages = ["."]
revision = "d0303fe809921458f417bcf828397a65db30a7e4"
[[projects]]
name = "github.com/opencontainers/runc"
packages = ["libcontainer/configs"]
revision = "0351df1c5a66838d0c392b4ac4cf9450de844e2d"
[[projects]]
name = "github.com/opencontainers/runtime-spec"
packages = ["specs-go"]
revision = "4e3b9264a330d094b0386c3703c5f379119711e8"
[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
name = "github.com/sirupsen/logrus"
packages = ["."]
revision = "89742aefa4b206dcf400792f3bd35b542998eb3b"
[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert"]
revision = "890a5c3458b43e6104ff5da8dfa139d013d77544"
[[projects]]
name = "github.com/urfave/cli"
packages = ["."]
revision = "ac249472b7de27a9e8990819566d9be95ab5b816"
[[projects]]
name = "github.com/vishvananda/netlink"
packages = [
".",
"nl"
]
revision = "c2a3de3b38bd00f07290c3c5e12b4dbc04ec8666"
[[projects]]
name = "github.com/vishvananda/netns"
packages = ["."]
revision = "86bef332bfc3b59b7624a600bd53009ce91a9829"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
revision = "91a49db82a88618983a78a06c1cbd4e00ab749ab"
[[projects]]
name = "golang.org/x/net"
packages = [
"context",
"http2",
"http2/hpack",
"idna",
"internal/timeseries",
"lex/httplex",
"trace"
]
revision = "a8b9294777976932365dabb6640cf1468d95c70f"
[[projects]]
name = "golang.org/x/sys"
packages = [
"unix",
"windows"
]
revision = "1d2aa6dbdea45adaaebb9905d0666e4537563829"
[[projects]]
name = "golang.org/x/text"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable"
]
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "2c5e7ac708aaa719366570dd82bda44541ca2a63"
[[projects]]
name = "google.golang.org/grpc"
packages = [
".",
"balancer",
"balancer/roundrobin",
"codes",
"connectivity",
"credentials",
"encoding",
"grpclb/grpc_lb_v1/messages",
"grpclog",
"internal",
"keepalive",
"metadata",
"naming",
"peer",
"resolver",
"resolver/dns",
"resolver/passthrough",
"stats",
"status",
"tap",
"transport"
]
revision = "5a9f7b402fe85096d2e1d0383435ee1876e863d0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "590b7d31900883c6782b596fa607426a77d71565516cfe8395379308a6285771"
solver-name = "gps-cdcl"
solver-version = 1

72
virtcontainers/Gopkg.toml Normal file
View File

@@ -0,0 +1,72 @@
[[constraint]]
name = "github.com/clearcontainers/proxy"
revision = "1d2a6a3ea132a86abd0731408b7dc34f2fc17d55"
[[constraint]]
name = "github.com/containernetworking/cni"
revision = "384d8c0b5288c25b9f1da901c66ea5155e6c567d"
[[constraint]]
name = "github.com/containernetworking/plugins"
revision = "7f98c94613021d8b57acfa1a2f0c8d0f6fd7ae5a"
[[constraint]]
name = "github.com/go-ini/ini"
revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd"
[[constraint]]
name = "github.com/kubernetes-incubator/cri-o"
revision = "3394b3b2d6af0e41d185bb695c6378be5dd4d61d"
[[constraint]]
name = "github.com/mitchellh/mapstructure"
revision = "d0303fe809921458f417bcf828397a65db30a7e4"
[[constraint]]
name = "github.com/opencontainers/runc"
revision = "0351df1c5a66838d0c392b4ac4cf9450de844e2d"
[[constraint]]
name = "github.com/opencontainers/runtime-spec"
revision = "4e3b9264a330d094b0386c3703c5f379119711e8"
[[constraint]]
name = "github.com/stretchr/testify"
revision = "890a5c3458b43e6104ff5da8dfa139d013d77544"
[[constraint]]
name = "github.com/urfave/cli"
revision = "ac249472b7de27a9e8990819566d9be95ab5b816"
[[constraint]]
name = "github.com/vishvananda/netlink"
revision = "c2a3de3b38bd00f07290c3c5e12b4dbc04ec8666"
[[constraint]]
name = "github.com/vishvananda/netns"
revision = "86bef332bfc3b59b7624a600bd53009ce91a9829"
[[constraint]]
name = "golang.org/x/sys"
revision = "1d2aa6dbdea45adaaebb9905d0666e4537563829"
[[constraint]]
name = "github.com/sirupsen/logrus"
revision = "89742aefa4b206dcf400792f3bd35b542998eb3b"
[[constraint]]
name = "github.com/intel/govmm"
revision = "e87160f8ea39dfe558bf6761f74717d191d82493"
[[constraint]]
name = "github.com/kata-containers/agent"
revision = "33eecb2a445f906811a5bc9713d2dafd10768d18"
[[constraint]]
name = "github.com/containerd/cri-containerd"
revision = "3d382e2f5dabe3bae62ceb9ded56bdee847008ee"
[prune]
non-go = true
go-tests = true
unused-packages = true

201
virtcontainers/LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

128
virtcontainers/Makefile Normal file
View File

@@ -0,0 +1,128 @@
PREFIX := /usr
BIN_DIR := $(PREFIX)/bin
VC_BIN_DIR := $(BIN_DIR)/virtcontainers/bin
TEST_BIN_DIR := $(VC_BIN_DIR)/test
VIRTC_DIR := hack/virtc
VIRTC_BIN := virtc
HOOK_DIR := hook/mock
HOOK_BIN := hook
CC_SHIM_DIR := shim/mock/cc-shim
CC_SHIM_BIN := cc-shim
KATA_SHIM_DIR := shim/mock/kata-shim
KATA_SHIM_BIN := kata-shim
#
# Pretty printing
#
V = @
Q = $(V:1=)
QUIET_GOBUILD = $(Q:@=@echo ' GOBUILD '$@;)
#
# Build
#
all: build binaries
build:
$(QUIET_GOBUILD)go build $(go list ./... | grep -v /vendor/)
virtc:
$(QUIET_GOBUILD)go build -o $(VIRTC_DIR)/$@ $(VIRTC_DIR)/*.go
hook:
$(QUIET_GOBUILD)go build -o $(HOOK_DIR)/$@ $(HOOK_DIR)/*.go
cc-shim:
$(QUIET_GOBUILD)go build -o $(CC_SHIM_DIR)/$@ $(CC_SHIM_DIR)/*.go
kata-shim:
$(QUIET_GOBUILD)go build -o $(KATA_SHIM_DIR)/$@ $(KATA_SHIM_DIR)/*.go
binaries: virtc hook cc-shim kata-shim
#
# Tests
#
check: check-go-static check-go-test
check-go-static:
bash .ci/go-lint.sh
check-go-test:
bash .ci/go-test.sh \
$(TEST_BIN_DIR)/$(CC_SHIM_BIN) \
$(TEST_BIN_DIR)/$(KATA_SHIM_BIN) \
$(TEST_BIN_DIR)/$(HOOK_BIN)
#
# Install
#
define INSTALL_EXEC
install -D $1 $(VC_BIN_DIR)/ || exit 1;
endef
define INSTALL_TEST_EXEC
install -D $1 $(TEST_BIN_DIR)/ || exit 1;
endef
install:
@mkdir -p $(VC_BIN_DIR)
$(call INSTALL_EXEC,$(VIRTC_DIR)/$(VIRTC_BIN))
@mkdir -p $(TEST_BIN_DIR)
$(call INSTALL_TEST_EXEC,$(HOOK_DIR)/$(HOOK_BIN))
$(call INSTALL_TEST_EXEC,$(CC_SHIM_DIR)/$(CC_SHIM_BIN))
$(call INSTALL_TEST_EXEC,$(KATA_SHIM_DIR)/$(KATA_SHIM_BIN))
#
# Uninstall
#
define UNINSTALL_EXEC
rm -f $(call FILE_SAFE_TO_REMOVE,$(VC_BIN_DIR)/$1) || exit 1;
endef
define UNINSTALL_TEST_EXEC
rm -f $(call FILE_SAFE_TO_REMOVE,$(TEST_BIN_DIR)/$1) || exit 1;
endef
uninstall:
$(call UNINSTALL_EXEC,$(VIRTC_BIN))
$(call UNINSTALL_TEST_EXEC,$(HOOK_BIN))
$(call UNINSTALL_TEST_EXEC,$(CC_SHIM_BIN))
$(call UNINSTALL_TEST_EXEC,$(KATA_SHIM_BIN))
#
# Clean
#
# Input: filename to check.
# Output: filename, assuming the file exists and is safe to delete.
define FILE_SAFE_TO_REMOVE =
$(shell test -e "$(1)" && test "$(1)" != "/" && echo "$(1)")
endef
CLEAN_FILES += $(VIRTC_DIR)/$(VIRTC_BIN)
CLEAN_FILES += $(HOOK_DIR)/$(HOOK_BIN)
CLEAN_FILES += $(SHIM_DIR)/$(CC_SHIM_BIN)
CLEAN_FILES += $(SHIM_DIR)/$(KATA_SHIM_BIN)
clean:
rm -f $(foreach f,$(CLEAN_FILES),$(call FILE_SAFE_TO_REMOVE,$(f)))
.PHONY: \
all \
build \
virtc \
hook \
shim \
binaries \
check \
check-go-static \
check-go-test \
install \
uninstall \
clean

138
virtcontainers/NEWS Normal file
View File

@@ -0,0 +1,138 @@
1.0.8:
Added support for CPU hotplug
Added SCSI support for drives and volumes
Added networking support for Kata Containers
Added block device hotplug support for Kata Containers
Added support for starting shim inside their own PID namespace
1.0.7:
Added ARM64 support
Added Kata Containers support
Added dockershim support
Added govmm support
Added support for the one proxy per VM model
Added 1.0 API documentation
Added hotplug support for QEMU's Q35 machine type
Fixed the agent, proxy and shim hierarchy
Fixed the state storage implementation
1.0.6:
Added SRIOV support
Added Multi-OS support
Removed Travis CI
1.0.5:
Added full CI (unit, functional and integration tests)
Added docker ps command support
Added multi-queue macvtap to the list of networking models
Added --device support for block devices (emulated)
Added pod shared locks for more efficient locking
Removed the ciao uuid package dependency
1.0.4:
Added shim debug output
Added support for custom qemu accelerators
Added structured logging
Added drive hotplug support for all containers
Fixed pod block index accounting
Fixed vendoring model
1.0.3:
Added initial device assignment framework
Added VFIO devices assignment initial support
Fixed pod locking
Fixed unconfigured interfaces handling
1.0.2:
Added James Hunt as maintainer
Added huge pages support
Added option for disabling nesting optimizations
Fixed the rootfs readonly support
1.0.1:
Added QEMU memory locking support
1.0.0:
Added support for memory and cpu reservation
Added stroage hotplug support
Added code of conduct
Added memory pre-allocation, realtime and memory locking support
Fixed nested virtualization support
Fixed log verbosity
Fixed pod cleanup
1.0.0-rc.4:
Added new project approvers
Fixed gometalinter
Increased unit test coverage
Added "pc" machine type for Qemu hypervisor
Fixed "kill" and "stop" container
1.0.0-rc.3:
Added support for assigning the container rootfs underlying block device to the VM (when using devicemapper)
Changed qemu to use virtio-net vhost by default
Changed qemu parameters to not be Clear Containers specific
Fixed MTU passing to hyperstart
1.0.0-rc.2:
Added support for transitioning from Ready to Stopped
Fixed pod container creation
Fixed CRI-O support for sandbox identifiers
1.0.0-rc.1:
Added standard error for invalid pod resources
Fixed docker gateway routing
1.0.0-rc.0:
Added pod pause and resume support
Added bind mounts support
Added CRI-O support
Added QEMU Q35 machine type support
Added readonly rootfs support
Added annotations passing to the OCI config file
Fixed hyperstart users and groups management
Fixed the installation and build scripts
Fixed networking MAC address configuration
0.7.0:
Added Clear Containers 3.0 shim support
Added Clear Containers 3.0 proxy protocol support
Added self contained hyperstart package
Added full build scripts
Added hypervisor and pod debug options
Added command line and function arguments validity checks
Added logrus support, replacing glog
Added OCI runtime vendoring
Added structured and separated error definitions
Fixed networking support and unit testing
Fixed code layering by keeping generic code out of interface implementations
Fixed benchmarks
0.6.0:
Added pullapprove support
Added Slack integration
Added KillContainer API
Added networking documentation
Added a Process structure for describing container processes
Added a more complete pod and container status structure
Fixed hyperstart implementation for stopping containers
0.5.0:
Changed the CreatePod/StartPod logic
Added an OCI config file conversion package
Added initial benchmarks
Added more CNI network unit tests
Moved all non virtcontainers packages under pkg/
Fixed netns race conditions
0.4.0:
Added CNM network model support
Added hyperstart asynchronous events support
Added low level CTL routines
Added OCI compatible hooks support
Added Travis code coverage report
0.3.0:
Added support for the latest hyperstart changes
Added proxy interface and cc-proxy implementation
Added CNI unit tests

7
virtcontainers/OWNERS Normal file
View File

@@ -0,0 +1,7 @@
reviewers:
- virtcontainers-maintainers
approvers:
- sameo
- sboeuf
- jodh-intel

354
virtcontainers/README.md Normal file
View File

@@ -0,0 +1,354 @@
[![Build Status](https://travis-ci.org/containers/virtcontainers.svg?branch=master)](https://travis-ci.org/containers/virtcontainers)
[![Build Status](http://cc-jenkins-ci.westus2.cloudapp.azure.com/job/virtcontainers-ubuntu-16-04-master/badge/icon)](http://cc-jenkins-ci.westus2.cloudapp.azure.com/job/virtcontainers-ubuntu-16-04-master)
[![Build Status](http://cc-jenkins-ci.westus2.cloudapp.azure.com/job/virtcontainers-ubuntu-17-04-master/badge/icon)](http://cc-jenkins-ci.westus2.cloudapp.azure.com/job/virtcontainers-ubuntu-17-04-master)
[![Build Status](http://cc-jenkins-ci.westus2.cloudapp.azure.com/job/virtcontainers-fedora-26-master/badge/icon)](http://cc-jenkins-ci.westus2.cloudapp.azure.com/job/virtcontainers-fedora-26-master)
[![Go Report Card](https://goreportcard.com/badge/github.com/containers/virtcontainers)](https://goreportcard.com/report/github.com/containers/virtcontainers)
[![Coverage Status](https://coveralls.io/repos/github/containers/virtcontainers/badge.svg?branch=master)](https://coveralls.io/github/containers/virtcontainers?branch=master)
[![GoDoc](https://godoc.org/github.com/containers/virtcontainers?status.svg)](https://godoc.org/github.com/containers/virtcontainers)
Table of Contents
=================
* [What is it ?](#what-is-it-)
* [Background](#background)
* [Out of scope](#out-of-scope)
* [virtcontainers and Kubernetes CRI](#virtcontainers-and-kubernetes-cri)
* [Design](#design)
* [Pods](#pods)
* [Hypervisors](#hypervisors)
* [Agents](#agents)
* [Shim](#shim)
* [Proxy](#proxy)
* [API](#api)
* [Pod API](#pod-api)
* [Container API](#container-api)
* [Networking](#networking)
* [CNM](#cnm)
* [CNI](#cni)
* [Storage](#storage)
* [How to check if container uses devicemapper block device as its rootfs](#how-to-check-if-container-uses-devicemapper-block-device-as-its-rootfs)
* [Devices](#devices)
* [How to pass a device using VFIO-passthrough](#how-to-pass-a-device-using-vfio-passthrough)
* [Developers](#developers)
# What is it ?
`virtcontainers` is a Go library that can be used to build hardware-virtualized container
runtimes.
# Background
The few existing VM-based container runtimes (Clear Containers, runv, rkt's
kvm stage 1) all share the same hardware virtualization semantics but use different
code bases to implement them. `virtcontainers`'s goal is to factorize this code into
a common Go library.
Ideally, VM-based container runtime implementations would become translation
layers from the runtime specification they implement (e.g. the [OCI runtime-spec][oci]
or the [Kubernetes CRI][cri]) to the `virtcontainers` API.
`virtcontainers` is [Clear Containers][cc]'s runtime foundational package for their
[runtime][cc-runtime] implementation
[oci]: https://github.com/opencontainers/runtime-spec
[cri]: https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/container-runtime-interface-v1.md
[cc]: https://github.com/clearcontainers/
[cc-runtime]: https://github.com/clearcontainers/runtime/
# Out of scope
Implementing a container runtime is out of scope for this project. Any
tools or executables in this repository are only provided for demonstration or
testing purposes.
## virtcontainers and Kubernetes CRI
`virtcontainers`'s API is loosely inspired by the Kubernetes [CRI][cri] because
we believe it provides the right level of abstractions for containerized pods.
However, despite the API similarities between the two projects, the goal of
`virtcontainers` is _not_ to build a CRI implementation, but instead to provide a
generic, runtime-specification agnostic, hardware-virtualized containers
library that other projects could leverage to implement CRI themselves.
# Design
## Pods
The `virtcontainers` execution unit is a _pod_, i.e. `virtcontainers` users start pods where
containers will be running.
`virtcontainers` creates a pod by starting a virtual machine and setting the pod
up within that environment. Starting a pod means launching all containers with
the VM pod runtime environment.
## Hypervisors
The `virtcontainers` package relies on hypervisors to start and stop virtual machine where
pods will be running. An hypervisor is defined by an Hypervisor interface implementation,
and the default implementation is the QEMU one.
## Agents
During the lifecycle of a container, the runtime running on the host needs to interact with
the virtual machine guest OS in order to start new commands to be executed as part of a given
container workload, set new networking routes or interfaces, fetch a container standard or
error output, and so on.
There are many existing and potential solutions to resolve that problem and `virtcontainers` abstracts
this through the Agent interface.
## Shim
In some cases the runtime will need a translation shim between the higher level container
stack (e.g. Docker) and the virtual machine holding the container workload. This is needed
for container stacks that make strong assumptions on the nature of the container they're
monitoring. In cases where they assume containers are simply regular host processes, a shim
layer is needed to translate host specific semantics into e.g. agent controlled virtual
machine ones.
## Proxy
When hardware virtualized containers have limited I/O multiplexing capabilities,
runtimes may decide to rely on an external host proxy to support cases where several
runtime instances are talking to the same container.
# API
The high level `virtcontainers` API is the following one:
## Pod API
* `CreatePod(podConfig PodConfig)` creates a Pod.
The virtual machine is started and the Pod is prepared.
* `DeletePod(podID string)` deletes a Pod.
The virtual machine is shut down and all information related to the Pod are removed.
The function will fail if the Pod is running. In that case `StopPod()` has to be called first.
* `StartPod(podID string)` starts an already created Pod.
The Pod and all its containers are started.
* `RunPod(podConfig PodConfig)` creates and starts a Pod.
This performs `CreatePod()` + `StartPod()`.
* `StopPod(podID string)` stops an already running Pod.
The Pod and all its containers are stopped.
* `PausePod(podID string)` pauses an existing Pod.
* `ResumePod(podID string)` resume a paused Pod.
* `StatusPod(podID string)` returns a detailed Pod status.
* `ListPod()` lists all Pods on the host.
It returns a detailed status for every Pod.
## Container API
* `CreateContainer(podID string, containerConfig ContainerConfig)` creates a Container on an existing Pod.
* `DeleteContainer(podID, containerID string)` deletes a Container from a Pod.
If the Container is running it has to be stopped first.
* `StartContainer(podID, containerID string)` starts an already created Container.
The Pod has to be running.
* `StopContainer(podID, containerID string)` stops an already running Container.
* `EnterContainer(podID, containerID string, cmd Cmd)` enters an already running Container and runs a given command.
* `StatusContainer(podID, containerID string)` returns a detailed Container status.
* `KillContainer(podID, containerID string, signal syscall.Signal, all bool)` sends a signal to all or one container inside a Pod.
An example tool using the `virtcontainers` API is provided in the `hack/virtc` package.
# Networking
`virtcontainers` supports the 2 major container networking models: the [Container Network Model (CNM)][cnm] and the [Container Network Interface (CNI)][cni].
Typically the former is the Docker default networking model while the later is used on Kubernetes deployments.
`virtcontainers` callers can select one or the other, on a per pod basis, by setting their `PodConfig`'s `NetworkModel` field properly.
[cnm]: https://github.com/docker/libnetwork/blob/master/docs/design.md
[cni]: https://github.com/containernetworking/cni/
## CNM
![High-level CNM Diagram](documentation/network/CNM_overall_diagram.png)
__CNM lifecycle__
1. RequestPool
2. CreateNetwork
3. RequestAddress
4. CreateEndPoint
5. CreateContainer
6. Create config.json
7. Create PID and network namespace
8. ProcessExternalKey
9. JoinEndPoint
10. LaunchContainer
11. Launch
12. Run container
![Detailed CNM Diagram](documentation/network/CNM_detailed_diagram.png)
__Runtime network setup with CNM__
1. Read config.json
2. Create the network namespace ([code](https://github.com/containers/virtcontainers/blob/0.5.0/cnm.go#L108-L120))
3. Call the prestart hook (from inside the netns) ([code](https://github.com/containers/virtcontainers/blob/0.5.0/api.go#L46-L49))
4. Scan network interfaces inside netns and get the name of the interface created by prestart hook ([code](https://github.com/containers/virtcontainers/blob/0.5.0/cnm.go#L70-L106))
5. Create bridge, TAP, and link all together with network interface previously created ([code](https://github.com/containers/virtcontainers/blob/0.5.0/network.go#L123-L205))
6. Start VM inside the netns and start the container ([code](https://github.com/containers/virtcontainers/blob/0.5.0/api.go#L66-L70))
__Drawbacks of CNM__
There are three drawbacks about using CNM instead of CNI:
* The way we call into it is not very explicit: Have to re-exec dockerd binary so that it can accept parameters and execute the prestart hook related to network setup.
* Implicit way to designate the network namespace: Instead of explicitely giving the netns to dockerd, we give it the PID of our runtime so that it can find the netns from this PID. This means we have to make sure being in the right netns while calling the hook, otherwise the veth pair will be created with the wrong netns.
* No results are back from the hook: We have to scan the network interfaces to discover which one has been created inside the netns. This introduces more latency in the code because it forces us to scan the network in the CreatePod path, which is critical for starting the VM as quick as possible.
## CNI
![CNI Diagram](documentation/network/CNI_diagram.png)
__Runtime network setup with CNI__
1. Create the network namespace ([code](https://github.com/containers/virtcontainers/blob/0.5.0/cni.go#L64-L76))
2. Get CNI plugin information ([code](https://github.com/containers/virtcontainers/blob/0.5.0/cni.go#L29-L32))
3. Start the plugin (providing previously created netns) to add a network described into /etc/cni/net.d/ directory. At that time, the CNI plugin will create the cni0 network interface and a veth pair between the host and the created netns. It links cni0 to the veth pair before to exit. ([code](https://github.com/containers/virtcontainers/blob/0.5.0/cni.go#L34-L45))
4. Create bridge, TAP, and link all together with network interface previously created ([code](https://github.com/containers/virtcontainers/blob/0.5.0/network.go#L123-L205))
5. Start VM inside the netns and start the container ([code](https://github.com/containers/virtcontainers/blob/0.5.0/api.go#L66-L70))
# Storage
Container workloads are shared with the virtualized environment through 9pfs.
The devicemapper storage driver is a special case. The driver uses dedicated block devices rather than formatted filesystems, and operates at the block level rather than the file level. This knowledge has been used to directly use the underlying block device instead of the overlay file system for the container root file system. The block device maps to the top read-write layer for the overlay. This approach gives much better I/O performance compared to using 9pfs to share the container file system.
The approach above does introduce a limitation in terms of dynamic file copy in/out of the container via `docker cp` operations.
The copy operation from host to container accesses the mounted file system on the host side. This is not expected to work and may lead to inconsistencies as the block device will be simultaneously written to, from two different mounts.
The copy operation from container to host will work, provided the user calls `sync(1)` from within the container prior to the copy to make sure any outstanding cached data is written to the block device.
```
docker cp [OPTIONS] CONTAINER:SRC_PATH HOST:DEST_PATH
docker cp [OPTIONS] HOST:SRC_PATH CONTAINER:DEST_PATH
```
Ability to hotplug block devices has been added, which makes it possible to use block devices for containers started after the VM has been launched.
## How to check if container uses devicemapper block device as its rootfs
Start a container. Call mount(8) within the container. You should see '/' mounted on /dev/vda device.
# Devices
Support has been added to pass [VFIO](https://www.kernel.org/doc/Documentation/vfio.txt)
assigned devices on the docker command line with --device.
Support for passing other devices including block devices with --device has
not been added added yet.
## How to pass a device using VFIO-passthrough
1. Requirements
IOMMU group represents the smallest set of devices for which the IOMMU has
visibility and which is isolated from other groups. VFIO uses this information
to enforce safe ownership of devices for userspace.
You will need Intel VT-d capable hardware. Check if IOMMU is enabled in your host
kernel by verifying `CONFIG_VFIO_NOIOMMU` is not in the kernel config. If it is set,
you will need to rebuild your kernel.
The following kernel configs need to be enabled:
```
CONFIG_VFIO_IOMMU_TYPE1=m
CONFIG_VFIO=m
CONFIG_VFIO_PCI=m
```
In addition, you need to pass `intel_iommu=on` on the kernel command line.
2. Identify BDF(Bus-Device-Function) of the PCI device to be assigned.
```
$ lspci -D | grep -e Ethernet -e Network
0000:01:00.0 Ethernet controller: Intel Corporation Ethernet Controller 10-Gigabit X540-AT2 (rev 01)
$ BDF=0000:01:00.0
```
3. Find vendor and device id.
```
$ lspci -n -s $BDF
01:00.0 0200: 8086:1528 (rev 01)
```
4. Find IOMMU group.
```
$ readlink /sys/bus/pci/devices/$BDF/iommu_group
../../../../kernel/iommu_groups/16
```
5. Unbind the device from host driver.
```
$ echo $BDF | sudo tee /sys/bus/pci/devices/$BDF/driver/unbind
```
6. Bind the device to vfio-pci.
```
$ sudo modprobe vfio-pci
$ echo 8086 1528 | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id
$ echo $BDF | sudo tee --append /sys/bus/pci/drivers/vfio-pci/bind
```
7. Check /dev/vfio
```
$ ls /dev/vfio
16 vfio
```
8. Start a Clear Containers container passing the VFIO group on the docker command line.
```
docker run -it --device=/dev/vfio/16 centos/tools bash
```
9. Running `lspci` within the container should show the device among the
PCI devices. The driver for the device needs to be present within the
Clear Containers kernel. If the driver is missing, you can add it to your
custom container kernel using the [osbuilder](https://github.com/clearcontainers/osbuilder)
tooling.
# Developers
For information on how to build, develop and test `virtcontainers`, see the
[developer documentation](documentation/Developers.md).

175
virtcontainers/agent.go Normal file
View File

@@ -0,0 +1,175 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"syscall"
"github.com/mitchellh/mapstructure"
)
// AgentType describes the type of guest agent a Pod should run.
type AgentType string
// ProcessListOptions contains the options used to list running
// processes inside the container
type ProcessListOptions struct {
// Format describes the output format to list the running processes.
// Formats are unrelated to ps(1) formats, only two formats can be specified:
// "json" and "table"
Format string
// Args contains the list of arguments to run ps(1) command.
// If Args is empty the agent will use "-ef" as options to ps(1).
Args []string
}
// ProcessList represents the list of running processes inside the container
type ProcessList []byte
const (
// NoopAgentType is the No-Op agent.
NoopAgentType AgentType = "noop"
// HyperstartAgent is the Hyper hyperstart agent.
HyperstartAgent AgentType = "hyperstart"
// KataContainersAgent is the Kata Containers agent.
KataContainersAgent AgentType = "kata"
// SocketTypeVSOCK is a VSOCK socket type for talking to an agent.
SocketTypeVSOCK = "vsock"
// SocketTypeUNIX is a UNIX socket type for talking to an agent.
// It typically means the agent is living behind a host proxy.
SocketTypeUNIX = "unix"
)
// Set sets an agent type based on the input string.
func (agentType *AgentType) Set(value string) error {
switch value {
case "noop":
*agentType = NoopAgentType
return nil
case "hyperstart":
*agentType = HyperstartAgent
return nil
case "kata":
*agentType = KataContainersAgent
return nil
default:
return fmt.Errorf("Unknown agent type %s", value)
}
}
// String converts an agent type to a string.
func (agentType *AgentType) String() string {
switch *agentType {
case NoopAgentType:
return string(NoopAgentType)
case HyperstartAgent:
return string(HyperstartAgent)
case KataContainersAgent:
return string(KataContainersAgent)
default:
return ""
}
}
// newAgent returns an agent from an agent type.
func newAgent(agentType AgentType) agent {
switch agentType {
case NoopAgentType:
return &noopAgent{}
case HyperstartAgent:
return &hyper{}
case KataContainersAgent:
return &kataAgent{}
default:
return &noopAgent{}
}
}
// newAgentConfig returns an agent config from a generic PodConfig interface.
func newAgentConfig(config PodConfig) interface{} {
switch config.AgentType {
case NoopAgentType:
return nil
case HyperstartAgent:
var hyperConfig HyperConfig
err := mapstructure.Decode(config.AgentConfig, &hyperConfig)
if err != nil {
return err
}
return hyperConfig
case KataContainersAgent:
var kataAgentConfig KataAgentConfig
err := mapstructure.Decode(config.AgentConfig, &kataAgentConfig)
if err != nil {
return err
}
return kataAgentConfig
default:
return nil
}
}
// agent is the virtcontainers agent interface.
// Agents are running in the guest VM and handling
// communications between the host and guest.
type agent interface {
// init is used to pass agent specific configuration to the agent implementation.
// agent implementations also will typically start listening for agent events from
// init().
// After init() is called, agent implementations should be initialized and ready
// to handle all other Agent interface methods.
init(pod *Pod, config interface{}) error
// capabilities should return a structure that specifies the capabilities
// supported by the agent.
capabilities() capabilities
// createPod will tell the agent to perform necessary setup for a Pod.
createPod(pod *Pod) error
// exec will tell the agent to run a command in an already running container.
exec(pod *Pod, c Container, cmd Cmd) (*Process, error)
// startPod will tell the agent to start all containers related to the Pod.
startPod(pod Pod) error
// stopPod will tell the agent to stop all containers related to the Pod.
stopPod(pod Pod) error
// createContainer will tell the agent to create a container related to a Pod.
createContainer(pod *Pod, c *Container) (*Process, error)
// startContainer will tell the agent to start a container related to a Pod.
startContainer(pod Pod, c *Container) error
// stopContainer will tell the agent to stop a container related to a Pod.
stopContainer(pod Pod, c Container) error
// killContainer will tell the agent to send a signal to a
// container related to a Pod. If all is true, all processes in
// the container will be sent the signal.
killContainer(pod Pod, c Container, signal syscall.Signal, all bool) error
// processListContainer will list the processes running inside the container
processListContainer(pod Pod, c Container, options ProcessListOptions) (ProcessList, error)
}

156
virtcontainers/agent_test.go Normal file
View File

@@ -0,0 +1,156 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"reflect"
"testing"
)
func testSetAgentType(t *testing.T, value string, expected AgentType) {
var agentType AgentType
err := (&agentType).Set(value)
if err != nil {
t.Fatal(err)
}
if agentType != expected {
t.Fatal(err)
}
}
func TestSetNoopAgentType(t *testing.T) {
testSetAgentType(t, "noop", NoopAgentType)
}
func TestSetHyperstartAgentType(t *testing.T) {
testSetAgentType(t, "hyperstart", HyperstartAgent)
}
func TestSetKataAgentType(t *testing.T) {
testSetAgentType(t, "kata", KataContainersAgent)
}
func TestSetUnknownAgentType(t *testing.T) {
var agentType AgentType
err := (&agentType).Set("unknown")
if err == nil {
t.Fatal()
}
if agentType == NoopAgentType ||
agentType == HyperstartAgent {
t.Fatal()
}
}
func testStringFromAgentType(t *testing.T, agentType AgentType, expected string) {
agentTypeStr := (&agentType).String()
if agentTypeStr != expected {
t.Fatal()
}
}
func TestStringFromNoopAgentType(t *testing.T) {
testStringFromAgentType(t, NoopAgentType, "noop")
}
func TestStringFromHyperstartAgentType(t *testing.T) {
testStringFromAgentType(t, HyperstartAgent, "hyperstart")
}
func TestStringFromKataAgentType(t *testing.T) {
testStringFromAgentType(t, KataContainersAgent, "kata")
}
func TestStringFromUnknownAgentType(t *testing.T) {
var agentType AgentType
testStringFromAgentType(t, agentType, "")
}
func testNewAgentFromAgentType(t *testing.T, agentType AgentType, expected agent) {
ag := newAgent(agentType)
if reflect.DeepEqual(ag, expected) == false {
t.Fatal()
}
}
func TestNewAgentFromNoopAgentType(t *testing.T) {
testNewAgentFromAgentType(t, NoopAgentType, &noopAgent{})
}
func TestNewAgentFromHyperstartAgentType(t *testing.T) {
testNewAgentFromAgentType(t, HyperstartAgent, &hyper{})
}
func TestNewAgentFromKataAgentType(t *testing.T) {
testNewAgentFromAgentType(t, KataContainersAgent, &kataAgent{})
}
func TestNewAgentFromUnknownAgentType(t *testing.T) {
var agentType AgentType
testNewAgentFromAgentType(t, agentType, &noopAgent{})
}
func testNewAgentConfig(t *testing.T, config PodConfig, expected interface{}) {
agentConfig := newAgentConfig(config)
if reflect.DeepEqual(agentConfig, expected) == false {
t.Fatal()
}
}
func TestNewAgentConfigFromNoopAgentType(t *testing.T) {
var agentConfig interface{}
podConfig := PodConfig{
AgentType: NoopAgentType,
AgentConfig: agentConfig,
}
testNewAgentConfig(t, podConfig, agentConfig)
}
func TestNewAgentConfigFromHyperstartAgentType(t *testing.T) {
agentConfig := HyperConfig{}
podConfig := PodConfig{
AgentType: HyperstartAgent,
AgentConfig: agentConfig,
}
testNewAgentConfig(t, podConfig, agentConfig)
}
func TestNewAgentConfigFromKataAgentType(t *testing.T) {
agentConfig := KataAgentConfig{}
podConfig := PodConfig{
AgentType: KataContainersAgent,
AgentConfig: agentConfig,
}
testNewAgentConfig(t, podConfig, agentConfig)
}
func TestNewAgentConfigFromUnknownAgentType(t *testing.T) {
var agentConfig interface{}
testNewAgentConfig(t, PodConfig{}, agentConfig)
}

655
virtcontainers/api.go Normal file
View File

@@ -0,0 +1,655 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"os"
"runtime"
"syscall"
"github.com/sirupsen/logrus"
)
func init() {
runtime.LockOSThread()
}
var virtLog = logrus.FieldLogger(logrus.New())
// SetLogger sets the logger for virtcontainers package.
func SetLogger(logger logrus.FieldLogger) {
fields := logrus.Fields{
"source": "virtcontainers",
"arch": runtime.GOARCH,
}
virtLog = logger.WithFields(fields)
}
// CreatePod is the virtcontainers pod creation entry point.
// CreatePod creates a pod and its containers. It does not start them.
func CreatePod(podConfig PodConfig) (VCPod, error) {
return createPodFromConfig(podConfig)
}
func createPodFromConfig(podConfig PodConfig) (*Pod, error) {
// Create the pod.
p, err := createPod(podConfig)
if err != nil {
return nil, err
}
// Create the pod network
if err := p.createNetwork(); err != nil {
return nil, err
}
// Start the VM
if err := p.startVM(); err != nil {
return nil, err
}
// Create Containers
if err := p.createContainers(); err != nil {
return nil, err
}
// The pod is completely created now, we can store it.
if err := p.storePod(); err != nil {
return nil, err
}
return p, nil
}
// DeletePod is the virtcontainers pod deletion entry point.
// DeletePod will stop an already running container and then delete it.
func DeletePod(podID string) (VCPod, error) {
if podID == "" {
return nil, errNeedPodID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
// Fetch the pod from storage and create it.
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
// Delete it.
if err := p.delete(); err != nil {
return nil, err
}
return p, nil
}
// StartPod is the virtcontainers pod starting entry point.
// StartPod will talk to the given hypervisor to start an existing
// pod and all its containers.
// It returns the pod ID.
func StartPod(podID string) (VCPod, error) {
if podID == "" {
return nil, errNeedPodID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
// Fetch the pod from storage and create it.
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
return startPod(p)
}
func startPod(p *Pod) (*Pod, error) {
// Start it
err := p.start()
if err != nil {
return nil, err
}
// Execute poststart hooks.
if err := p.config.Hooks.postStartHooks(); err != nil {
return nil, err
}
return p, nil
}
// StopPod is the virtcontainers pod stopping entry point.
// StopPod will talk to the given agent to stop an existing pod and destroy all containers within that pod.
func StopPod(podID string) (VCPod, error) {
if podID == "" {
return nil, errNeedPod
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
// Fetch the pod from storage and create it.
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
// Stop it.
err = p.stop()
if err != nil {
p.delete()
return nil, err
}
// Remove the network.
if err := p.removeNetwork(); err != nil {
return nil, err
}
// Execute poststop hooks.
if err := p.config.Hooks.postStopHooks(); err != nil {
return nil, err
}
return p, nil
}
// RunPod is the virtcontainers pod running entry point.
// RunPod creates a pod and its containers and then it starts them.
func RunPod(podConfig PodConfig) (VCPod, error) {
p, err := createPodFromConfig(podConfig)
if err != nil {
return nil, err
}
lockFile, err := rwLockPod(p.id)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
return startPod(p)
}
// ListPod is the virtcontainers pod listing entry point.
func ListPod() ([]PodStatus, error) {
dir, err := os.Open(configStoragePath)
if err != nil {
if os.IsNotExist(err) {
// No pod directory is not an error
return []PodStatus{}, nil
}
return []PodStatus{}, err
}
defer dir.Close()
podsID, err := dir.Readdirnames(0)
if err != nil {
return []PodStatus{}, err
}
var podStatusList []PodStatus
for _, podID := range podsID {
podStatus, err := StatusPod(podID)
if err != nil {
continue
}
podStatusList = append(podStatusList, podStatus)
}
return podStatusList, nil
}
// StatusPod is the virtcontainers pod status entry point.
func StatusPod(podID string) (PodStatus, error) {
if podID == "" {
return PodStatus{}, errNeedPodID
}
lockFile, err := rLockPod(podID)
if err != nil {
return PodStatus{}, err
}
pod, err := fetchPod(podID)
if err != nil {
unlockPod(lockFile)
return PodStatus{}, err
}
// We need to potentially wait for a separate container.stop() routine
// that needs to be terminated before we return from this function.
// Deferring the synchronization here is very important since we want
// to avoid a deadlock. Indeed, the goroutine started by statusContainer
// will need to lock an exclusive lock, meaning that all other locks have
// to be released to let this happen. This call ensures this will be the
// last operation executed by this function.
defer pod.wg.Wait()
defer unlockPod(lockFile)
var contStatusList []ContainerStatus
for _, container := range pod.containers {
contStatus, err := statusContainer(pod, container.id)
if err != nil {
return PodStatus{}, err
}
contStatusList = append(contStatusList, contStatus)
}
podStatus := PodStatus{
ID: pod.id,
State: pod.state,
Hypervisor: pod.config.HypervisorType,
HypervisorConfig: pod.config.HypervisorConfig,
Agent: pod.config.AgentType,
ContainersStatus: contStatusList,
Annotations: pod.config.Annotations,
}
return podStatus, nil
}
// CreateContainer is the virtcontainers container creation entry point.
// CreateContainer creates a container on a given pod.
func CreateContainer(podID string, containerConfig ContainerConfig) (VCPod, VCContainer, error) {
if podID == "" {
return nil, nil, errNeedPodID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, nil, err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return nil, nil, err
}
// Create the container.
c, err := createContainer(p, containerConfig)
if err != nil {
return nil, nil, err
}
// Add the container to the containers list in the pod.
if err := p.addContainer(c); err != nil {
return nil, nil, err
}
// Store it.
err = c.storeContainer()
if err != nil {
return nil, nil, err
}
// Update pod config.
p.config.Containers = append(p.config.Containers, containerConfig)
err = p.storage.storePodResource(podID, configFileType, *(p.config))
if err != nil {
return nil, nil, err
}
return p, c, nil
}
// DeleteContainer is the virtcontainers container deletion entry point.
// DeleteContainer deletes a Container from a Pod. If the container is running,
// it needs to be stopped first.
func DeleteContainer(podID, containerID string) (VCContainer, error) {
if podID == "" {
return nil, errNeedPodID
}
if containerID == "" {
return nil, errNeedContainerID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
// Fetch the container.
c, err := p.findContainer(containerID)
if err != nil {
return nil, err
}
// Delete it.
err = c.delete()
if err != nil {
return nil, err
}
// Update pod config
for idx, contConfig := range p.config.Containers {
if contConfig.ID == containerID {
p.config.Containers = append(p.config.Containers[:idx], p.config.Containers[idx+1:]...)
break
}
}
err = p.storage.storePodResource(podID, configFileType, *(p.config))
if err != nil {
return nil, err
}
return c, nil
}
// StartContainer is the virtcontainers container starting entry point.
// StartContainer starts an already created container.
func StartContainer(podID, containerID string) (VCContainer, error) {
if podID == "" {
return nil, errNeedPodID
}
if containerID == "" {
return nil, errNeedContainerID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
// Fetch the container.
c, err := p.findContainer(containerID)
if err != nil {
return nil, err
}
// Start it.
err = c.start()
if err != nil {
c.delete()
return nil, err
}
return c, nil
}
// StopContainer is the virtcontainers container stopping entry point.
// StopContainer stops an already running container.
func StopContainer(podID, containerID string) (VCContainer, error) {
if podID == "" {
return nil, errNeedPodID
}
if containerID == "" {
return nil, errNeedContainerID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
// Fetch the container.
c, err := p.findContainer(containerID)
if err != nil {
return nil, err
}
// Stop it.
err = c.stop()
if err != nil {
c.delete()
return nil, err
}
return c, nil
}
// EnterContainer is the virtcontainers container command execution entry point.
// EnterContainer enters an already running container and runs a given command.
func EnterContainer(podID, containerID string, cmd Cmd) (VCPod, VCContainer, *Process, error) {
if podID == "" {
return nil, nil, nil, errNeedPodID
}
if containerID == "" {
return nil, nil, nil, errNeedContainerID
}
lockFile, err := rLockPod(podID)
if err != nil {
return nil, nil, nil, err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return nil, nil, nil, err
}
// Fetch the container.
c, err := p.findContainer(containerID)
if err != nil {
return nil, nil, nil, err
}
// Enter it.
process, err := c.enter(cmd)
if err != nil {
return nil, nil, nil, err
}
return p, c, process, nil
}
// StatusContainer is the virtcontainers container status entry point.
// StatusContainer returns a detailed container status.
func StatusContainer(podID, containerID string) (ContainerStatus, error) {
if podID == "" {
return ContainerStatus{}, errNeedPodID
}
if containerID == "" {
return ContainerStatus{}, errNeedContainerID
}
lockFile, err := rLockPod(podID)
if err != nil {
return ContainerStatus{}, err
}
pod, err := fetchPod(podID)
if err != nil {
unlockPod(lockFile)
return ContainerStatus{}, err
}
// We need to potentially wait for a separate container.stop() routine
// that needs to be terminated before we return from this function.
// Deferring the synchronization here is very important since we want
// to avoid a deadlock. Indeed, the goroutine started by statusContainer
// will need to lock an exclusive lock, meaning that all other locks have
// to be released to let this happen. This call ensures this will be the
// last operation executed by this function.
defer pod.wg.Wait()
defer unlockPod(lockFile)
return statusContainer(pod, containerID)
}
// This function is going to spawn a goroutine and it needs to be waited for
// by the caller.
func statusContainer(pod *Pod, containerID string) (ContainerStatus, error) {
for _, container := range pod.containers {
if container.id == containerID {
// We have to check for the process state to make sure
// we update the status in case the process is supposed
// to be running but has been killed or terminated.
if (container.state.State == StateReady ||
container.state.State == StateRunning ||
container.state.State == StatePaused) &&
container.process.Pid > 0 {
running, err := isShimRunning(container.process.Pid)
if err != nil {
return ContainerStatus{}, err
}
if !running {
pod.wg.Add(1)
go func() {
defer pod.wg.Done()
lockFile, err := rwLockPod(pod.id)
if err != nil {
return
}
defer unlockPod(lockFile)
if err := container.stop(); err != nil {
return
}
}()
}
}
return ContainerStatus{
ID: container.id,
State: container.state,
PID: container.process.Pid,
StartTime: container.process.StartTime,
RootFs: container.config.RootFs,
Annotations: container.config.Annotations,
}, nil
}
}
// No matching containers in the pod
return ContainerStatus{}, nil
}
// KillContainer is the virtcontainers entry point to send a signal
// to a container running inside a pod. If all is true, all processes in
// the container will be sent the signal.
func KillContainer(podID, containerID string, signal syscall.Signal, all bool) error {
if podID == "" {
return errNeedPodID
}
if containerID == "" {
return errNeedContainerID
}
lockFile, err := rwLockPod(podID)
if err != nil {
return err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return err
}
// Fetch the container.
c, err := p.findContainer(containerID)
if err != nil {
return err
}
// Send a signal to the process.
err = c.kill(signal, all)
if err != nil {
return err
}
return nil
}
// PausePod is the virtcontainers pausing entry point which pauses an
// already running pod.
func PausePod(podID string) (VCPod, error) {
return togglePausePod(podID, true)
}
// ResumePod is the virtcontainers resuming entry point which resumes
// (or unpauses) and already paused pod.
func ResumePod(podID string) (VCPod, error) {
return togglePausePod(podID, false)
}
// ProcessListContainer is the virtcontainers entry point to list
// processes running inside a container
func ProcessListContainer(podID, containerID string, options ProcessListOptions) (ProcessList, error) {
if podID == "" {
return nil, errNeedPodID
}
if containerID == "" {
return nil, errNeedContainerID
}
lockFile, err := rLockPod(podID)
if err != nil {
return nil, err
}
defer unlockPod(lockFile)
p, err := fetchPod(podID)
if err != nil {
return nil, err
}
// Fetch the container.
c, err := p.findContainer(containerID)
if err != nil {
return nil, err
}
return c.processList(options)
}

2231
virtcontainers/api_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

158
virtcontainers/asset.go Normal file
View File

@@ -0,0 +1,158 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"crypto/sha512"
"encoding/hex"
"fmt"
"io/ioutil"
"path/filepath"
"github.com/containers/virtcontainers/pkg/annotations"
)
type assetType string
func (t assetType) annotations() (string, string, error) {
switch t {
case kernelAsset:
return annotations.KernelPath, annotations.KernelHash, nil
case imageAsset:
return annotations.ImagePath, annotations.ImageHash, nil
case hypervisorAsset:
return annotations.HypervisorPath, annotations.HypervisorHash, nil
case firmwareAsset:
return annotations.FirmwarePath, annotations.FirmwareHash, nil
}
return "", "", fmt.Errorf("Wrong asset type %s", t)
}
const (
kernelAsset assetType = "kernel"
imageAsset assetType = "image"
hypervisorAsset assetType = "hypervisor"
firmwareAsset assetType = "firmware"
)
type asset struct {
path string
computedHash string
kind assetType
}
func (a *asset) valid() bool {
if !filepath.IsAbs(a.path) {
return false
}
switch a.kind {
case kernelAsset:
return true
case imageAsset:
return true
case hypervisorAsset:
return true
case firmwareAsset:
return true
}
return false
}
// hash returns the hex encoded string for the asset hash
func (a *asset) hash(hashType string) (string, error) {
var hashEncodedLen int
var hash string
// We read the actual asset content
bytes, err := ioutil.ReadFile(a.path)
if err != nil {
return "", err
}
if len(bytes) == 0 {
return "", fmt.Errorf("Empty asset file at %s", a.path)
}
// Build the asset hash and convert it to a string.
// We only support SHA512 for now.
switch hashType {
case annotations.SHA512:
virtLog.Debugf("Computing %v hash", a.path)
hashComputed := sha512.Sum512(bytes)
hashEncodedLen = hex.EncodedLen(len(hashComputed))
hashEncoded := make([]byte, hashEncodedLen)
hex.Encode(hashEncoded, hashComputed[:])
hash = string(hashEncoded[:])
virtLog.Debugf("%v hash: %s", a.path, hash)
default:
return "", fmt.Errorf("Invalid hash type %s", hashType)
}
a.computedHash = hash
return hash, nil
}
// newAsset returns a new asset from the pod annotations.
func newAsset(podConfig *PodConfig, t assetType) (*asset, error) {
pathAnnotation, hashAnnotation, err := t.annotations()
if err != nil {
return nil, err
}
if pathAnnotation == "" || hashAnnotation == "" {
return nil, fmt.Errorf("Missing annotation paths for %s", t)
}
path, ok := podConfig.Annotations[pathAnnotation]
if !ok || path == "" {
return nil, nil
}
if !filepath.IsAbs(path) {
return nil, fmt.Errorf("%s is not an absolute path", path)
}
a := &asset{path: path, kind: t}
hash, ok := podConfig.Annotations[hashAnnotation]
if !ok || hash == "" {
return a, nil
}
// We have a hash annotation, we need to verify the asset against it.
hashType, ok := podConfig.Annotations[annotations.AssetHashType]
if !ok {
virtLog.Warningf("Unrecognized hash type: %s, switching to %s", hashType, annotations.SHA512)
hashType = annotations.SHA512
}
hashComputed, err := a.hash(hashType)
if err != nil {
return a, err
}
// If our computed asset hash does not match the passed annotation, we must exit.
if hashComputed != hash {
return nil, fmt.Errorf("Invalid hash for %s: computed %s, expecting %s]", a.path, hashComputed, hash)
}
return a, nil
}

117
virtcontainers/asset_test.go Normal file
View File

@@ -0,0 +1,117 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"io/ioutil"
"os"
"testing"
"github.com/containers/virtcontainers/pkg/annotations"
"github.com/stretchr/testify/assert"
)
var assetContent = []byte("FakeAsset fake asset FAKE ASSET")
var assetContentHash = "92549f8d2018a95a294d28a65e795ed7d1a9d150009a28cea108ae10101178676f04ab82a6950d0099e4924f9c5e41dcba8ece56b75fc8b4e0a7492cb2a8c880"
var assetContentWrongHash = "92549f8d2018a95a294d28a65e795ed7d1a9d150009a28cea108ae10101178676f04ab82a6950d0099e4924f9c5e41dcba8ece56b75fc8b4e0a7492cb2a8c881"
func TestAssetWrongHashType(t *testing.T) {
assert := assert.New(t)
tmpfile, err := ioutil.TempFile("", "virtcontainers-test-")
assert.Nil(err)
defer func() {
tmpfile.Close()
os.Remove(tmpfile.Name()) // clean up
}()
_, err = tmpfile.Write(assetContent)
assert.Nil(err)
a := &asset{
path: tmpfile.Name(),
}
h, err := a.hash("shafoo")
assert.Equal(h, "")
assert.NotNil(err)
}
func TestAssetHash(t *testing.T) {
assert := assert.New(t)
tmpfile, err := ioutil.TempFile("", "virtcontainers-test-")
assert.Nil(err)
defer func() {
tmpfile.Close()
os.Remove(tmpfile.Name()) // clean up
}()
_, err = tmpfile.Write(assetContent)
assert.Nil(err)
a := &asset{
path: tmpfile.Name(),
}
hash, err := a.hash(annotations.SHA512)
assert.Nil(err)
assert.Equal(assetContentHash, hash)
assert.Equal(assetContentHash, a.computedHash)
}
func TestAssetNew(t *testing.T) {
assert := assert.New(t)
tmpfile, err := ioutil.TempFile("", "virtcontainers-test-")
assert.Nil(err)
defer func() {
tmpfile.Close()
os.Remove(tmpfile.Name()) // clean up
}()
_, err = tmpfile.Write(assetContent)
assert.Nil(err)
p := &PodConfig{
Annotations: map[string]string{
annotations.KernelPath: tmpfile.Name(),
annotations.KernelHash: assetContentHash,
},
}
a, err := newAsset(p, imageAsset)
assert.Nil(err)
assert.Nil(a)
a, err = newAsset(p, kernelAsset)
assert.Nil(err)
assert.Equal(assetContentHash, a.computedHash)
p = &PodConfig{
Annotations: map[string]string{
annotations.KernelPath: tmpfile.Name(),
annotations.KernelHash: assetContentWrongHash,
},
}
a, err = newAsset(p, kernelAsset)
assert.NotNil(err)
}

77
virtcontainers/bridge.go Normal file
View File

@@ -0,0 +1,77 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import "fmt"
type bridgeType string
const (
pciBridge bridgeType = "pci"
pcieBridge = "pcie"
)
const pciBridgeMaxCapacity = 30
// Bridge is a bridge where devices can be hot plugged
type Bridge struct {
// Address contains information about devices plugged and its address in the bridge
Address map[uint32]string
// Type is the type of the bridge (pci, pcie, etc)
Type bridgeType
//ID is used to identify the bridge in the hypervisor
ID string
}
// addDevice on success adds the device ID to the bridge and return the address
// where the device was added, otherwise an error is returned
func (b *Bridge) addDevice(ID string) (uint32, error) {
var addr uint32
// looking for the first available address
for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
if _, ok := b.Address[i]; !ok {
addr = i
break
}
}
if addr == 0 {
return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots")
}
// save address and device
b.Address[addr] = ID
return addr, nil
}
// removeDevice on success removes the device ID from the bridge and return nil,
// otherwise an error is returned
func (b *Bridge) removeDevice(ID string) error {
// check if the device was hot plugged in the bridge
for addr, devID := range b.Address {
if devID == ID {
// free address to re-use the same slot with other devices
delete(b.Address, addr)
return nil
}
}
return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID)
}

58
virtcontainers/bridge_test.go Normal file
View File

@@ -0,0 +1,58 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestAddRemoveDevice(t *testing.T) {
assert := assert.New(t)
// create a bridge
bridges := []*Bridge{{make(map[uint32]string), pciBridge, "rgb123"}}
// add device
devID := "abc123"
b := bridges[0]
addr, err := b.addDevice(devID)
assert.NoError(err)
if addr < 1 {
assert.Fail("address cannot be less than 1")
}
// remove device
err = b.removeDevice("")
assert.Error(err)
err = b.removeDevice(devID)
assert.NoError(err)
// add device when the bridge is full
bridges[0].Address = make(map[uint32]string)
for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
bridges[0].Address[i] = fmt.Sprintf("%d", i)
}
addr, err = b.addDevice(devID)
assert.Error(err)
if addr != 0 {
assert.Fail("address should be 0")
}
}

48
virtcontainers/capabilities.go Normal file
View File

@@ -0,0 +1,48 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
const (
blockDeviceSupport = 1 << iota
blockDeviceHotplugSupport
)
type capabilities struct {
flags uint
}
func (caps *capabilities) isBlockDeviceSupported() bool {
if caps.flags&blockDeviceSupport != 0 {
return true
}
return false
}
func (caps *capabilities) setBlockDeviceSupport() {
caps.flags = caps.flags | blockDeviceSupport
}
func (caps *capabilities) isBlockDeviceHotplugSupported() bool {
if caps.flags&blockDeviceHotplugSupport != 0 {
return true
}
return false
}
func (caps *capabilities) setBlockDeviceHotplugSupport() {
caps.flags |= blockDeviceHotplugSupport
}

47
virtcontainers/capabilities_test.go Normal file
View File

@@ -0,0 +1,47 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import "testing"
func TestBlockDeviceCapability(t *testing.T) {
var caps capabilities
if caps.isBlockDeviceSupported() {
t.Fatal()
}
caps.setBlockDeviceSupport()
if !caps.isBlockDeviceSupported() {
t.Fatal()
}
}
func TestBlockDeviceHotplugCapability(t *testing.T) {
var caps capabilities
if caps.isBlockDeviceHotplugSupported() {
t.Fatal()
}
caps.setBlockDeviceHotplugSupport()
if !caps.isBlockDeviceHotplugSupported() {
t.Fatal()
}
}

54
virtcontainers/cc_proxy.go Normal file
View File

@@ -0,0 +1,54 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"os/exec"
)
type ccProxy struct {
}
// start is the proxy start implementation for ccProxy.
func (p *ccProxy) start(pod Pod, params proxyParams) (int, string, error) {
config, err := newProxyConfig(pod.config)
if err != nil {
return -1, "", err
}
// construct the socket path the proxy instance will use
proxyURL, err := defaultProxyURL(pod, SocketTypeUNIX)
if err != nil {
return -1, "", err
}
args := []string{config.Path, "-uri", proxyURL}
if config.Debug {
args = append(args, "-log", "debug")
}
cmd := exec.Command(args[0], args[1:]...)
if err := cmd.Start(); err != nil {
return -1, "", err
}
return cmd.Process.Pid, proxyURL, nil
}
func (p *ccProxy) stop(pod Pod, pid int) error {
return nil
}

100
virtcontainers/cc_proxy_test.go Normal file
View File

@@ -0,0 +1,100 @@
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCCProxyStart(t *testing.T) {
assert := assert.New(t)
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
proxy := &ccProxy{}
type testData struct {
pod Pod
expectedURI string
expectError bool
}
invalidPath := filepath.Join(tmpdir, "enoent")
expectedSocketPath := filepath.Join(runStoragePath, testPodID, "proxy.sock")
expectedURI := fmt.Sprintf("unix://%s", expectedSocketPath)
data := []testData{
{Pod{}, "", true},
{
Pod{
config: &PodConfig{
ProxyType: "invalid",
},
}, "", true,
},
{
Pod{
config: &PodConfig{
ProxyType: CCProxyType,
ProxyConfig: ProxyConfig{
// invalid - no path
},
},
}, "", true,
},
{
Pod{
config: &PodConfig{
ProxyType: CCProxyType,
ProxyConfig: ProxyConfig{
Path: invalidPath,
},
},
}, "", true,
},
{
Pod{
id: testPodID,
config: &PodConfig{
ProxyType: CCProxyType,
ProxyConfig: ProxyConfig{
Path: "echo",
},
},
}, expectedURI, false,
},
}
for _, d := range data {
pid, uri, err := proxy.start(d.pod, proxyParams{})
if d.expectError {
assert.Error(err)
continue
}
assert.NoError(err)
assert.True(pid > 0)
assert.Equal(d.expectedURI, uri)
}
}

60
virtcontainers/cc_shim.go Normal file
View File

@@ -0,0 +1,60 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
)
type ccShim struct{}
// start is the ccShim start implementation.
// It starts the cc-shim binary with URL and token flags provided by
// the proxy.
func (s *ccShim) start(pod Pod, params ShimParams) (int, error) {
if pod.config == nil {
return -1, fmt.Errorf("Pod config cannot be nil")
}
config, ok := newShimConfig(*(pod.config)).(ShimConfig)
if !ok {
return -1, fmt.Errorf("Wrong shim config type, should be CCShimConfig type")
}
if config.Path == "" {
return -1, fmt.Errorf("Shim path cannot be empty")
}
if params.Token == "" {
return -1, fmt.Errorf("Token cannot be empty")
}
if params.URL == "" {
return -1, fmt.Errorf("URL cannot be empty")
}
if params.Container == "" {
return -1, fmt.Errorf("Container cannot be empty")
}
args := []string{config.Path, "-c", params.Container, "-t", params.Token, "-u", params.URL}
if config.Debug {
args = append(args, "-d")
}
return startShim(args, params)
}

368
virtcontainers/cc_shim_test.go Normal file
View File

@@ -0,0 +1,368 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"unsafe"
. "github.com/containers/virtcontainers/pkg/mock"
)
// These tests don't care about the format of the container ID
const testContainer = "testContainer"
var testCCShimPath = "/usr/bin/virtcontainers/bin/test/cc-shim"
var testProxyURL = "foo:///foo/clear-containers/proxy.sock"
var testWrongConsolePath = "/foo/wrong-console"
var testConsolePath = "tty-console"
func getMockCCShimBinPath() string {
if DefaultMockCCShimBinPath == "" {
return testCCShimPath
}
return DefaultMockCCShimBinPath
}
func testCCShimStart(t *testing.T, pod Pod, params ShimParams, expectFail bool) {
s := &ccShim{}
pid, err := s.start(pod, params)
if expectFail {
if err == nil || pid != -1 {
t.Fatalf("This test should fail (pod %+v, params %+v, expectFail %t)",
pod, params, expectFail)
}
} else {
if err != nil {
t.Fatalf("This test should pass (pod %+v, params %+v, expectFail %t): %s",
pod, params, expectFail, err)
}
if pid == -1 {
t.Fatalf("This test should pass (pod %+v, params %+v, expectFail %t)",
pod, params, expectFail)
}
}
}
func TestCCShimStartNilPodConfigFailure(t *testing.T) {
testCCShimStart(t, Pod{}, ShimParams{}, true)
}
func TestCCShimStartNilShimConfigFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{},
}
testCCShimStart(t, pod, ShimParams{}, true)
}
func TestCCShimStartShimPathEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{},
},
}
testCCShimStart(t, pod, ShimParams{}, true)
}
func TestCCShimStartShimTypeInvalid(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: "foo",
ShimConfig: ShimConfig{},
},
}
testCCShimStart(t, pod, ShimParams{}, true)
}
func TestCCShimStartParamsTokenEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: getMockCCShimBinPath(),
},
},
}
testCCShimStart(t, pod, ShimParams{}, true)
}
func TestCCShimStartParamsURLEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: getMockCCShimBinPath(),
},
},
}
params := ShimParams{
Token: "testToken",
}
testCCShimStart(t, pod, params, true)
}
func TestCCShimStartParamsContainerEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: getMockCCShimBinPath(),
},
},
}
params := ShimParams{
Token: "testToken",
URL: "unix://is/awesome",
}
testCCShimStart(t, pod, params, true)
}
func TestCCShimStartParamsInvalidCommand(t *testing.T) {
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
cmd := filepath.Join(dir, "does-not-exist")
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: cmd,
},
},
}
params := ShimParams{
Token: "testToken",
URL: "http://foo",
}
testCCShimStart(t, pod, params, true)
}
func startCCShimStartWithoutConsoleSuccessful(t *testing.T, detach bool) (*os.File, *os.File, *os.File, Pod, ShimParams, error) {
saveStdout := os.Stdout
rStdout, wStdout, err := os.Pipe()
if err != nil {
return nil, nil, nil, Pod{}, ShimParams{}, err
}
os.Stdout = wStdout
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: getMockCCShimBinPath(),
},
},
}
params := ShimParams{
Container: testContainer,
Token: "testToken",
URL: testProxyURL,
Detach: detach,
}
return rStdout, wStdout, saveStdout, pod, params, nil
}
func TestCCShimStartSuccessful(t *testing.T) {
rStdout, wStdout, saveStdout, pod, params, err := startCCShimStartWithoutConsoleSuccessful(t, false)
if err != nil {
t.Fatal(err)
}
defer func() {
os.Stdout = saveStdout
rStdout.Close()
wStdout.Close()
}()
testCCShimStart(t, pod, params, false)
bufStdout := make([]byte, 1024)
if _, err := rStdout.Read(bufStdout); err != nil {
t.Fatal(err)
}
if !strings.Contains(string(bufStdout), ShimStdoutOutput) {
t.Fatalf("Substring %q not found in %q", ShimStdoutOutput, string(bufStdout))
}
}
func TestCCShimStartDetachSuccessful(t *testing.T) {
rStdout, wStdout, saveStdout, pod, params, err := startCCShimStartWithoutConsoleSuccessful(t, true)
if err != nil {
t.Fatal(err)
}
defer func() {
os.Stdout = saveStdout
wStdout.Close()
rStdout.Close()
}()
testCCShimStart(t, pod, params, false)
readCh := make(chan error)
go func() {
bufStdout := make([]byte, 1024)
n, err := rStdout.Read(bufStdout)
if err != nil && err != io.EOF {
readCh <- err
return
}
if n > 0 {
readCh <- fmt.Errorf("Not expecting to read anything, Got %q", string(bufStdout))
return
}
readCh <- nil
}()
select {
case err := <-readCh:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Duration(20) * time.Millisecond):
return
}
}
func TestCCShimStartWithConsoleNonExistingFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: getMockCCShimBinPath(),
},
},
}
params := ShimParams{
Token: "testToken",
URL: testProxyURL,
Console: testWrongConsolePath,
}
testCCShimStart(t, pod, params, true)
}
func ioctl(fd uintptr, flag, data uintptr) error {
if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, flag, data); err != 0 {
return err
}
return nil
}
// unlockpt unlocks the slave pseudoterminal device corresponding to the master pseudoterminal referred to by f.
func unlockpt(f *os.File) error {
var u int32
return ioctl(f.Fd(), syscall.TIOCSPTLCK, uintptr(unsafe.Pointer(&u)))
}
// ptsname retrieves the name of the first available pts for the given master.
func ptsname(f *os.File) (string, error) {
var n int32
if err := ioctl(f.Fd(), syscall.TIOCGPTN, uintptr(unsafe.Pointer(&n))); err != nil {
return "", err
}
return fmt.Sprintf("/dev/pts/%d", n), nil
}
func newConsole() (*os.File, string, error) {
master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0)
if err != nil {
return nil, "", err
}
console, err := ptsname(master)
if err != nil {
return nil, "", err
}
if err := unlockpt(master); err != nil {
return nil, "", err
}
if err := os.Chmod(console, 0600); err != nil {
return nil, "", err
}
return master, console, nil
}
func TestCCShimStartWithConsoleSuccessful(t *testing.T) {
cleanUp()
master, console, err := newConsole()
t.Logf("Console created for tests:%s\n", console)
if err != nil {
t.Fatal(err)
}
pod := Pod{
config: &PodConfig{
ShimType: CCShimType,
ShimConfig: ShimConfig{
Path: getMockCCShimBinPath(),
},
},
}
params := ShimParams{
Container: testContainer,
Token: "testToken",
URL: testProxyURL,
Console: console,
}
testCCShimStart(t, pod, params, false)
master.Close()
}

179
virtcontainers/cni.go Normal file
View File

@@ -0,0 +1,179 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
cniTypes "github.com/containernetworking/cni/pkg/types"
cniV2Types "github.com/containernetworking/cni/pkg/types/020"
cniLatestTypes "github.com/containernetworking/cni/pkg/types/current"
cniPlugin "github.com/containers/virtcontainers/pkg/cni"
"github.com/sirupsen/logrus"
)
// CniPrimaryInterface Name chosen for the primary interface
// If CNI ever support multiple primary interfaces this should be revisited
const CniPrimaryInterface = "eth0"
// cni is a network implementation for the CNI plugin.
type cni struct{}
// Logger returns a logrus logger appropriate for logging cni messages
func (n *cni) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "cni")
}
func cniDNSToDNSInfo(cniDNS cniTypes.DNS) DNSInfo {
return DNSInfo{
Servers: cniDNS.Nameservers,
Domain: cniDNS.Domain,
Searches: cniDNS.Search,
Options: cniDNS.Options,
}
}
func convertLatestCNIResult(result *cniLatestTypes.Result) NetworkInfo {
return NetworkInfo{
DNS: cniDNSToDNSInfo(result.DNS),
}
}
func convertV2CNIResult(result *cniV2Types.Result) NetworkInfo {
return NetworkInfo{
DNS: cniDNSToDNSInfo(result.DNS),
}
}
func convertCNIResult(cniResult cniTypes.Result) (NetworkInfo, error) {
switch result := cniResult.(type) {
case *cniLatestTypes.Result:
return convertLatestCNIResult(result), nil
case *cniV2Types.Result:
return convertV2CNIResult(result), nil
default:
return NetworkInfo{}, fmt.Errorf("Unknown CNI result type %T", result)
}
}
func (n *cni) invokePluginsAdd(pod Pod, networkNS *NetworkNamespace) (*NetworkInfo, error) {
netPlugin, err := cniPlugin.NewNetworkPlugin()
if err != nil {
return nil, err
}
// Note: In the case of multus or cni-genie this will return only the results
// corresponding to the primary interface. The remaining results need to be
// derived
result, err := netPlugin.AddNetwork(pod.id, networkNS.NetNsPath, CniPrimaryInterface)
if err != nil {
return nil, err
}
netInfo, err := convertCNIResult(result)
if err != nil {
return nil, err
}
// We do not care about this for now but
// If present, the CNI DNS result has to be updated in resolv.conf
// if the kubelet has not supplied it already
n.Logger().Infof("AddNetwork results %s", result.String())
return &netInfo, nil
}
func (n *cni) invokePluginsDelete(pod Pod, networkNS NetworkNamespace) error {
netPlugin, err := cniPlugin.NewNetworkPlugin()
if err != nil {
return err
}
err = netPlugin.RemoveNetwork(pod.id, networkNS.NetNsPath, CniPrimaryInterface)
if err != nil {
return err
}
return nil
}
func (n *cni) updateEndpointsFromScan(networkNS *NetworkNamespace, netInfo *NetworkInfo, config NetworkConfig) error {
endpoints, err := createEndpointsFromScan(networkNS.NetNsPath, config)
if err != nil {
return err
}
for _, endpoint := range endpoints {
if CniPrimaryInterface == endpoint.Name() {
prop := endpoint.Properties()
prop.DNS = netInfo.DNS
endpoint.SetProperties(prop)
break
}
}
networkNS.Endpoints = endpoints
return nil
}
// init initializes the network, setting a new network namespace for the CNI network.
func (n *cni) init(config NetworkConfig) (string, bool, error) {
return initNetworkCommon(config)
}
// run runs a callback in the specified network namespace.
func (n *cni) run(networkNSPath string, cb func() error) error {
return runNetworkCommon(networkNSPath, cb)
}
// add adds all needed interfaces inside the network namespace for the CNI network.
func (n *cni) add(pod Pod, config NetworkConfig, netNsPath string, netNsCreated bool) (NetworkNamespace, error) {
networkNS := NetworkNamespace{
NetNsPath: netNsPath,
NetNsCreated: netNsCreated,
}
netInfo, err := n.invokePluginsAdd(pod, &networkNS)
if err != nil {
return NetworkNamespace{}, err
}
if err := n.updateEndpointsFromScan(&networkNS, netInfo, config); err != nil {
return NetworkNamespace{}, err
}
if err := addNetworkCommon(pod, &networkNS); err != nil {
return NetworkNamespace{}, err
}
return networkNS, nil
}
// remove unbridges and deletes TAP interfaces. It also removes virtual network
// interfaces and deletes the network namespace for the CNI network.
func (n *cni) remove(pod Pod, networkNS NetworkNamespace) error {
if err := removeNetworkCommon(networkNS); err != nil {
return err
}
if err := n.invokePluginsDelete(pod, networkNS); err != nil {
return err
}
return deleteNetNS(networkNS.NetNsPath, true)
}

70
virtcontainers/cnm.go Normal file
View File

@@ -0,0 +1,70 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"github.com/sirupsen/logrus"
)
// cnm is a network implementation for the CNM plugin.
type cnm struct {
config NetworkConfig
}
func cnmLogger() *logrus.Entry {
return virtLog.WithField("subsystem", "cnm")
}
// init initializes the network, setting a new network namespace for the CNM network.
func (n *cnm) init(config NetworkConfig) (string, bool, error) {
return initNetworkCommon(config)
}
// run runs a callback in the specified network namespace.
func (n *cnm) run(networkNSPath string, cb func() error) error {
return runNetworkCommon(networkNSPath, cb)
}
// add adds all needed interfaces inside the network namespace for the CNM network.
func (n *cnm) add(pod Pod, config NetworkConfig, netNsPath string, netNsCreated bool) (NetworkNamespace, error) {
endpoints, err := createEndpointsFromScan(netNsPath, config)
if err != nil {
return NetworkNamespace{}, err
}
networkNS := NetworkNamespace{
NetNsPath: netNsPath,
NetNsCreated: netNsCreated,
Endpoints: endpoints,
}
if err := addNetworkCommon(pod, &networkNS); err != nil {
return NetworkNamespace{}, err
}
return networkNS, nil
}
// remove unbridges and deletes TAP interfaces. It also removes virtual network
// interfaces and deletes the network namespace for the CNM network.
func (n *cnm) remove(pod Pod, networkNS NetworkNamespace) error {
if err := removeNetworkCommon(networkNS); err != nil {
return err
}
return deleteNetNS(networkNS.NetNsPath, true)
}

30
virtcontainers/cnm_test.go Normal file
View File

@@ -0,0 +1,30 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
type mockAddr struct {
network string
ipAddr string
}
func (m mockAddr) Network() string {
return m.network
}
func (m mockAddr) String() string {
return m.ipAddr
}

814
virtcontainers/container.go Normal file
View File

@@ -0,0 +1,814 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"encoding/hex"
"fmt"
"os"
"path/filepath"
"syscall"
"time"
"github.com/sirupsen/logrus"
)
// Process gathers data related to a container process.
type Process struct {
// Token is the process execution context ID. It must be
// unique per pod.
// Token is used to manipulate processes for containers
// that have not started yet, and later identify them
// uniquely within a pod.
Token string
// Pid is the process ID as seen by the host software
// stack, e.g. CRI-O, containerd. This is typically the
// shim PID.
Pid int
StartTime time.Time
}
// ContainerStatus describes a container status.
type ContainerStatus struct {
ID string
State State
PID int
StartTime time.Time
RootFs string
// Annotations allow clients to store arbitrary values,
// for example to add additional status values required
// to support particular specifications.
Annotations map[string]string
}
// ContainerResources describes container resources
type ContainerResources struct {
// CPUQuota specifies the total amount of time in microseconds
// The number of microseconds per CPUPeriod that the container is guaranteed CPU access
CPUQuota int64
// CPUPeriod specifies the CPU CFS scheduler period of time in microseconds
CPUPeriod uint64
// CPUShares specifies container's weight vs. other containers
CPUShares uint64
}
// ContainerConfig describes one container runtime configuration.
type ContainerConfig struct {
ID string
// RootFs is the container workload image on the host.
RootFs string
// ReadOnlyRootfs indicates if the rootfs should be mounted readonly
ReadonlyRootfs bool
// Cmd specifies the command to run on a container
Cmd Cmd
// Annotations allow clients to store arbitrary values,
// for example to add additional status values required
// to support particular specifications.
Annotations map[string]string
Mounts []Mount
// Device configuration for devices that must be available within the container.
DeviceInfos []DeviceInfo
// Resources container resources
Resources ContainerResources
}
// valid checks that the container configuration is valid.
func (c *ContainerConfig) valid() bool {
if c == nil {
return false
}
if c.ID == "" {
return false
}
return true
}
// SystemMountsInfo describes additional information for system mounts that the agent
// needs to handle
type SystemMountsInfo struct {
// Indicates if /dev has been passed as a bind mount for the host /dev
BindMountDev bool
// Size of /dev/shm assigned on the host.
DevShmSize uint
}
// Container is composed of a set of containers and a runtime environment.
// A Container can be created, deleted, started, stopped, listed, entered, paused and restored.
type Container struct {
id string
podID string
rootFs string
config *ContainerConfig
pod *Pod
runPath string
configPath string
containerPath string
state State
process Process
mounts []Mount
devices []Device
systemMountsInfo SystemMountsInfo
}
// ID returns the container identifier string.
func (c *Container) ID() string {
return c.id
}
// Logger returns a logrus logger appropriate for logging Container messages
func (c *Container) Logger() *logrus.Entry {
return virtLog.WithFields(logrus.Fields{
"subsystem": "container",
"container-id": c.id,
"pod-id": c.podID,
})
}
// Pod returns the pod handler related to this container.
func (c *Container) Pod() VCPod {
return c.pod
}
// Process returns the container process.
func (c *Container) Process() Process {
return c.process
}
// GetToken returns the token related to this container's process.
func (c *Container) GetToken() string {
return c.process.Token
}
// GetPid returns the pid related to this container's process.
func (c *Container) GetPid() int {
return c.process.Pid
}
// SetPid sets and stores the given pid as the pid of container's process.
func (c *Container) SetPid(pid int) error {
c.process.Pid = pid
return c.storeProcess()
}
func (c *Container) setStateBlockIndex(index int) error {
c.state.BlockIndex = index
err := c.pod.storage.storeContainerResource(c.pod.id, c.id, stateFileType, c.state)
if err != nil {
return err
}
return nil
}
func (c *Container) setStateFstype(fstype string) error {
c.state.Fstype = fstype
err := c.pod.storage.storeContainerResource(c.pod.id, c.id, stateFileType, c.state)
if err != nil {
return err
}
return nil
}
func (c *Container) setStateHotpluggedDrive(hotplugged bool) error {
c.state.HotpluggedDrive = hotplugged
err := c.pod.storage.storeContainerResource(c.pod.id, c.id, stateFileType, c.state)
if err != nil {
return err
}
return nil
}
// GetAnnotations returns container's annotations
func (c *Container) GetAnnotations() map[string]string {
return c.config.Annotations
}
func (c *Container) storeProcess() error {
return c.pod.storage.storeContainerProcess(c.podID, c.id, c.process)
}
func (c *Container) fetchProcess() (Process, error) {
return c.pod.storage.fetchContainerProcess(c.podID, c.id)
}
func (c *Container) storeMounts() error {
return c.pod.storage.storeContainerMounts(c.podID, c.id, c.mounts)
}
func (c *Container) fetchMounts() ([]Mount, error) {
return c.pod.storage.fetchContainerMounts(c.podID, c.id)
}
func (c *Container) storeDevices() error {
return c.pod.storage.storeContainerDevices(c.podID, c.id, c.devices)
}
func (c *Container) fetchDevices() ([]Device, error) {
return c.pod.storage.fetchContainerDevices(c.podID, c.id)
}
// storeContainer stores a container config.
func (c *Container) storeContainer() error {
fs := filesystem{}
err := fs.storeContainerResource(c.pod.id, c.id, configFileType, *(c.config))
if err != nil {
return err
}
return nil
}
// setContainerState sets both the in-memory and on-disk state of the
// container.
func (c *Container) setContainerState(state stateString) error {
if state == "" {
return errNeedState
}
// update in-memory state
c.state.State = state
// update on-disk state
err := c.pod.storage.storeContainerResource(c.pod.id, c.id, stateFileType, c.state)
if err != nil {
return err
}
return nil
}
func (c *Container) createContainersDirs() error {
err := os.MkdirAll(c.runPath, dirMode)
if err != nil {
return err
}
err = os.MkdirAll(c.configPath, dirMode)
if err != nil {
c.pod.storage.deleteContainerResources(c.podID, c.id, nil)
return err
}
return nil
}
// mountSharedDirMounts handles bind-mounts by bindmounting to the host shared
// directory which is mounted through 9pfs in the VM.
// It also updates the container mount list with the HostPath info, and store
// container mounts to the storage. This way, we will have the HostPath info
// available when we will need to unmount those mounts.
func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ([]Mount, error) {
var sharedDirMounts []Mount
for idx, m := range c.mounts {
if isSystemMount(m.Destination) || m.Type != "bind" {
continue
}
randBytes, err := generateRandomBytes(8)
if err != nil {
return nil, err
}
// These mounts are created in the shared dir
filename := fmt.Sprintf("%s-%s-%s", c.id, hex.EncodeToString(randBytes), filepath.Base(m.Destination))
mountDest := filepath.Join(hostSharedDir, c.pod.id, filename)
if err := bindMount(m.Source, mountDest, false); err != nil {
return nil, err
}
// Save HostPath mount value into the mount list of the container.
c.mounts[idx].HostPath = mountDest
// Check if mount is readonly, let the agent handle the readonly mount
// within the VM.
readonly := false
for _, flag := range m.Options {
if flag == "ro" {
readonly = true
}
}
sharedDirMount := Mount{
Source: filepath.Join(guestSharedDir, filename),
Destination: m.Destination,
Type: m.Type,
Options: m.Options,
ReadOnly: readonly,
}
sharedDirMounts = append(sharedDirMounts, sharedDirMount)
}
if err := c.storeMounts(); err != nil {
return nil, err
}
return sharedDirMounts, nil
}
func (c *Container) unmountHostMounts() error {
for _, m := range c.mounts {
if m.HostPath != "" {
if err := syscall.Unmount(m.HostPath, 0); err != nil {
c.Logger().WithFields(logrus.Fields{
"host-path": m.HostPath,
"error": err,
}).Warn("Could not umount")
return err
}
}
}
return nil
}
// newContainer creates a Container structure from a pod and a container configuration.
func newContainer(pod *Pod, contConfig ContainerConfig) (*Container, error) {
if contConfig.valid() == false {
return &Container{}, fmt.Errorf("Invalid container configuration")
}
c := &Container{
id: contConfig.ID,
podID: pod.id,
rootFs: contConfig.RootFs,
config: &contConfig,
pod: pod,
runPath: filepath.Join(runStoragePath, pod.id, contConfig.ID),
configPath: filepath.Join(configStoragePath, pod.id, contConfig.ID),
containerPath: filepath.Join(pod.id, contConfig.ID),
state: State{},
process: Process{},
mounts: contConfig.Mounts,
}
state, err := c.pod.storage.fetchContainerState(c.podID, c.id)
if err == nil {
c.state = state
}
process, err := c.pod.storage.fetchContainerProcess(c.podID, c.id)
if err == nil {
c.process = process
}
mounts, err := c.fetchMounts()
if err == nil {
c.mounts = mounts
}
// Devices will be found in storage after create stage has completed.
// We fetch devices from storage at all other stages.
storedDevices, err := c.fetchDevices()
if err == nil {
c.devices = storedDevices
} else {
// If devices were not found in storage, create Device implementations
// from the configuration. This should happen at create.
devices, err := newDevices(contConfig.DeviceInfos)
if err != nil {
return &Container{}, err
}
c.devices = devices
}
return c, nil
}
// createContainer creates and start a container inside a Pod. It has to be
// called only when a new container, not known by the pod, has to be created.
func createContainer(pod *Pod, contConfig ContainerConfig) (*Container, error) {
if pod == nil {
return nil, errNeedPod
}
c, err := newContainer(pod, contConfig)
if err != nil {
return nil, err
}
if err := c.createContainersDirs(); err != nil {
return nil, err
}
if !c.pod.config.HypervisorConfig.DisableBlockDeviceUse {
agentCaps := c.pod.agent.capabilities()
hypervisorCaps := c.pod.hypervisor.capabilities()
if agentCaps.isBlockDeviceSupported() && hypervisorCaps.isBlockDeviceHotplugSupported() {
if err := c.hotplugDrive(); err != nil {
return nil, err
}
}
}
// Attach devices
if err := c.attachDevices(); err != nil {
return nil, err
}
if err := c.addResources(); err != nil {
return nil, err
}
// Deduce additional system mount info that should be handled by the agent
// inside the VM
c.getSystemMountInfo()
if err := c.storeDevices(); err != nil {
return nil, err
}
process, err := pod.agent.createContainer(c.pod, c)
if err != nil {
return nil, err
}
c.process = *process
// Store the container process returned by the agent.
if err := c.storeProcess(); err != nil {
return nil, err
}
if err := c.setContainerState(StateReady); err != nil {
return nil, err
}
return c, nil
}
func (c *Container) delete() error {
if c.state.State != StateReady &&
c.state.State != StateStopped {
return fmt.Errorf("Container not ready or stopped, impossible to delete")
}
// Remove the container from pod structure
if err := c.pod.removeContainer(c.id); err != nil {
return err
}
return c.pod.storage.deleteContainerResources(c.podID, c.id, nil)
}
// checkPodRunning validates the container state.
//
// cmd specifies the operation (or verb) that the retrieval is destined
// for and is only used to make the returned error as descriptive as
// possible.
func (c *Container) checkPodRunning(cmd string) error {
if cmd == "" {
return fmt.Errorf("Cmd cannot be empty")
}
if c.pod.state.State != StateRunning {
return fmt.Errorf("Pod not running, impossible to %s the container", cmd)
}
return nil
}
func (c *Container) getSystemMountInfo() {
// check if /dev needs to be bind mounted from host /dev
c.systemMountsInfo.BindMountDev = false
for _, m := range c.mounts {
if m.Source == "/dev" && m.Destination == "/dev" && m.Type == "bind" {
c.systemMountsInfo.BindMountDev = true
}
}
// TODO Deduce /dev/shm size. See https://github.com/clearcontainers/runtime/issues/138
}
func (c *Container) start() error {
if err := c.checkPodRunning("start"); err != nil {
return err
}
if c.state.State != StateReady &&
c.state.State != StateStopped {
return fmt.Errorf("Container not ready or stopped, impossible to start")
}
if err := c.state.validTransition(c.state.State, StateRunning); err != nil {
return err
}
if err := c.pod.agent.startContainer(*(c.pod), c); err != nil {
c.Logger().WithError(err).Error("Failed to start container")
if err := c.stop(); err != nil {
c.Logger().WithError(err).Warn("Failed to stop container")
}
return err
}
return c.setContainerState(StateRunning)
}
func (c *Container) stop() error {
// In case the container status has been updated implicitly because
// the container process has terminated, it might be possible that
// someone try to stop the container, and we don't want to issue an
// error in that case. This should be a no-op.
//
// This has to be handled before the transition validation since this
// is an exception.
if c.state.State == StateStopped {
c.Logger().Info("Container already stopped")
return nil
}
if c.pod.state.State != StateReady && c.pod.state.State != StateRunning {
return fmt.Errorf("Pod not ready or running, impossible to stop the container")
}
if err := c.state.validTransition(c.state.State, StateStopped); err != nil {
return err
}
defer func() {
// If shim is still running something went wrong
// Make sure we stop the shim process
if running, _ := isShimRunning(c.process.Pid); running {
l := c.Logger()
l.Error("Failed to stop container so stopping dangling shim")
if err := stopShim(c.process.Pid); err != nil {
l.WithError(err).Warn("failed to stop shim")
}
}
}()
// Here we expect that stop() has been called because the container
// process returned or because it received a signal. In case of a
// signal, we want to give it some time to end the container process.
// However, if the signal didn't reach its goal, the caller still
// expects this container to be stopped, that's why we should not
// return an error, but instead try to kill it forcefully.
if err := waitForShim(c.process.Pid); err != nil {
// Force the container to be killed.
if err := c.pod.agent.killContainer(*(c.pod), *c, syscall.SIGKILL, true); err != nil {
return err
}
// Wait for the end of container process. We expect this call
// to succeed. Indeed, we have already given a second chance
// to the container by trying to kill it with SIGKILL, there
// is no reason to try to go further if we got an error.
if err := waitForShim(c.process.Pid); err != nil {
return err
}
}
if err := c.pod.agent.stopContainer(*(c.pod), *c); err != nil {
return err
}
if err := c.removeResources(); err != nil {
return err
}
if err := c.detachDevices(); err != nil {
return err
}
if err := c.removeDrive(); err != nil {
return err
}
return c.setContainerState(StateStopped)
}
func (c *Container) enter(cmd Cmd) (*Process, error) {
if err := c.checkPodRunning("enter"); err != nil {
return nil, err
}
if c.state.State != StateRunning {
return nil, fmt.Errorf("Container not running, impossible to enter")
}
process, err := c.pod.agent.exec(c.pod, *c, cmd)
if err != nil {
return nil, err
}
return process, nil
}
func (c *Container) kill(signal syscall.Signal, all bool) error {
if c.pod.state.State != StateReady && c.pod.state.State != StateRunning {
return fmt.Errorf("Pod not ready or running, impossible to signal the container")
}
if c.state.State != StateReady && c.state.State != StateRunning {
return fmt.Errorf("Container not ready or running, impossible to signal the container")
}
return c.pod.agent.killContainer(*(c.pod), *c, signal, all)
}
func (c *Container) processList(options ProcessListOptions) (ProcessList, error) {
if err := c.checkPodRunning("ps"); err != nil {
return nil, err
}
if c.state.State != StateRunning {
return nil, fmt.Errorf("Container not running, impossible to list processes")
}
return c.pod.agent.processListContainer(*(c.pod), *c, options)
}
func (c *Container) hotplugDrive() error {
dev, err := getDeviceForPath(c.rootFs)
if err == errMountPointNotFound {
return nil
}
if err != nil {
return err
}
c.Logger().WithFields(logrus.Fields{
"device-major": dev.major,
"device-minor": dev.minor,
"mount-point": dev.mountPoint,
}).Info("device details")
isDM, err := checkStorageDriver(dev.major, dev.minor)
if err != nil {
return err
}
if !isDM {
return nil
}
// If device mapper device, then fetch the full path of the device
devicePath, fsType, err := getDevicePathAndFsType(dev.mountPoint)
if err != nil {
return err
}
c.Logger().WithFields(logrus.Fields{
"device-path": devicePath,
"fs-type": fsType,
}).Info("Block device detected")
driveIndex, err := c.pod.getAndSetPodBlockIndex()
if err != nil {
return err
}
// Add drive with id as container id
devID := makeNameID("drive", c.id)
drive := Drive{
File: devicePath,
Format: "raw",
ID: devID,
Index: driveIndex,
}
if err := c.pod.hypervisor.hotplugAddDevice(drive, blockDev); err != nil {
return err
}
c.setStateHotpluggedDrive(true)
if err := c.setStateBlockIndex(driveIndex); err != nil {
return err
}
return c.setStateFstype(fsType)
}
// isDriveUsed checks if a drive has been used for container rootfs
func (c *Container) isDriveUsed() bool {
if c.state.Fstype == "" {
return false
}
return true
}
func (c *Container) removeDrive() (err error) {
if c.isDriveUsed() && c.state.HotpluggedDrive {
c.Logger().Info("unplugging block device")
devID := makeNameID("drive", c.id)
drive := Drive{
ID: devID,
}
l := c.Logger().WithField("device-id", devID)
l.Info("Unplugging block device")
if err := c.pod.hypervisor.hotplugRemoveDevice(drive, blockDev); err != nil {
l.WithError(err).Info("Failed to unplug block device")
return err
}
}
return nil
}
func (c *Container) attachDevices() error {
for _, device := range c.devices {
if err := device.attach(c.pod.hypervisor, c); err != nil {
return err
}
}
return nil
}
func (c *Container) detachDevices() error {
for _, device := range c.devices {
if err := device.detach(c.pod.hypervisor); err != nil {
return err
}
}
return nil
}
func (c *Container) addResources() error {
//TODO add support for memory, Issue: https://github.com/containers/virtcontainers/issues/578
if c.config == nil {
return nil
}
vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod)
if vCPUs != 0 {
virtLog.Debugf("hot adding %d vCPUs", vCPUs)
if err := c.pod.hypervisor.hotplugAddDevice(uint32(vCPUs), cpuDev); err != nil {
return err
}
}
return nil
}
func (c *Container) removeResources() error {
//TODO add support for memory, Issue: https://github.com/containers/virtcontainers/issues/578
if c.config == nil {
return nil
}
vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod)
if vCPUs != 0 {
virtLog.Debugf("hot removing %d vCPUs", vCPUs)
if err := c.pod.hypervisor.hotplugRemoveDevice(uint32(vCPUs), cpuDev); err != nil {
return err
}
}
return nil
}

334
virtcontainers/container_test.go Normal file
View File

@@ -0,0 +1,334 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"syscall"
"testing"
vcAnnotations "github.com/containers/virtcontainers/pkg/annotations"
"github.com/stretchr/testify/assert"
)
func TestGetAnnotations(t *testing.T) {
annotations := map[string]string{
"annotation1": "abc",
"annotation2": "xyz",
"annotation3": "123",
}
container := Container{
config: &ContainerConfig{
Annotations: annotations,
},
}
containerAnnotations := container.GetAnnotations()
for k, v := range containerAnnotations {
if annotations[k] != v {
t.Fatalf("Expecting ['%s']='%s', Got ['%s']='%s'\n", k, annotations[k], k, v)
}
}
}
func TestContainerSystemMountsInfo(t *testing.T) {
mounts := []Mount{
{
Source: "/dev",
Destination: "/dev",
Type: "bind",
},
{
Source: "procfs",
Destination: "/proc",
Type: "procfs",
},
}
c := Container{
mounts: mounts,
}
assert.False(t, c.systemMountsInfo.BindMountDev)
c.getSystemMountInfo()
assert.True(t, c.systemMountsInfo.BindMountDev)
c.mounts[0].Type = "tmpfs"
c.getSystemMountInfo()
assert.False(t, c.systemMountsInfo.BindMountDev)
}
func TestContainerPod(t *testing.T) {
expectedPod := &Pod{}
container := Container{
pod: expectedPod,
}
pod := container.Pod()
if !reflect.DeepEqual(pod, expectedPod) {
t.Fatalf("Expecting %+v\nGot %+v", expectedPod, pod)
}
}
func TestContainerRemoveDrive(t *testing.T) {
pod := &Pod{}
container := Container{
pod: pod,
id: "testContainer",
}
container.state.Fstype = ""
err := container.removeDrive()
// hotplugRemoveDevice for hypervisor should not be called.
// test should pass without a hypervisor created for the container's pod.
if err != nil {
t.Fatal("")
}
container.state.Fstype = "xfs"
container.state.HotpluggedDrive = false
err = container.removeDrive()
// hotplugRemoveDevice for hypervisor should not be called.
if err != nil {
t.Fatal("")
}
container.state.HotpluggedDrive = true
pod.hypervisor = &mockHypervisor{}
err = container.removeDrive()
if err != nil {
t.Fatal()
}
}
func testSetupFakeRootfs(t *testing.T) (testRawFile, loopDev, mntDir string, err error) {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
testRawFile = filepath.Join(tmpDir, "raw.img")
if _, err := os.Stat(testRawFile); !os.IsNotExist(err) {
os.Remove(testRawFile)
}
output, err := exec.Command("losetup", "-f").CombinedOutput()
if err != nil {
t.Fatalf("Skipping test since no loop device available for tests : %s, %s", output, err)
return
}
loopDev = strings.TrimSpace(string(output[:]))
output, err = exec.Command("fallocate", "-l", "256K", testRawFile).CombinedOutput()
if err != nil {
t.Fatalf("fallocate failed %s %s", output, err)
}
output, err = exec.Command("mkfs.ext4", "-F", testRawFile).CombinedOutput()
if err != nil {
t.Fatalf("mkfs.ext4 failed for %s: %s, %s", testRawFile, output, err)
}
output, err = exec.Command("losetup", loopDev, testRawFile).CombinedOutput()
if err != nil {
t.Fatalf("Losetup for %s at %s failed : %s, %s ", loopDev, testRawFile, output, err)
return
}
mntDir = filepath.Join(tmpDir, "rootfs")
err = os.Mkdir(mntDir, dirMode)
if err != nil {
t.Fatalf("Error creating dir %s: %s", mntDir, err)
}
err = syscall.Mount(loopDev, mntDir, "ext4", uintptr(0), "")
if err != nil {
t.Fatalf("Error while mounting loop device %s at %s: %s", loopDev, mntDir, err)
}
return
}
func cleanupFakeRootfsSetup(testRawFile, loopDev, mntDir string) {
// unmount loop device
if mntDir != "" {
syscall.Unmount(mntDir, 0)
}
// detach loop device
if loopDev != "" {
exec.Command("losetup", "-d", loopDev).CombinedOutput()
}
if _, err := os.Stat(testRawFile); err == nil {
tmpDir := filepath.Dir(testRawFile)
os.RemoveAll(tmpDir)
}
}
func TestContainerAddDriveDir(t *testing.T) {
testRawFile, loopDev, fakeRootfs, err := testSetupFakeRootfs(t)
defer cleanupFakeRootfsSetup(testRawFile, loopDev, fakeRootfs)
if err != nil {
t.Fatalf("Error while setting up fake rootfs: %v, Skipping test", err)
}
fs := &filesystem{}
pod := &Pod{
id: testPodID,
storage: fs,
hypervisor: &mockHypervisor{},
}
contID := "100"
container := Container{
pod: pod,
id: contID,
rootFs: fakeRootfs,
}
// create state file
path := filepath.Join(runStoragePath, testPodID, container.ID())
err = os.MkdirAll(path, dirMode)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(path)
stateFilePath := filepath.Join(path, stateFile)
os.Remove(stateFilePath)
_, err = os.Create(stateFilePath)
if err != nil {
t.Fatal(err)
}
defer os.Remove(stateFilePath)
// Make the checkStorageDriver func variable point to a fake check function
savedFunc := checkStorageDriver
checkStorageDriver = func(major, minor int) (bool, error) {
return true, nil
}
defer func() {
checkStorageDriver = savedFunc
}()
container.state.Fstype = ""
container.state.HotpluggedDrive = false
err = container.hotplugDrive()
if err != nil {
t.Fatalf("Error with hotplugDrive :%v", err)
}
if container.state.Fstype == "" || !container.state.HotpluggedDrive {
t.Fatal()
}
}
func TestCheckPodRunningEmptyCmdFailure(t *testing.T) {
c := &Container{}
err := c.checkPodRunning("")
assert.NotNil(t, err, "Should fail because provided command is empty")
}
func TestCheckPodRunningNotRunningFailure(t *testing.T) {
c := &Container{
pod: &Pod{},
}
err := c.checkPodRunning("test_cmd")
assert.NotNil(t, err, "Should fail because pod state is empty")
}
func TestCheckPodRunningSuccessful(t *testing.T) {
c := &Container{
pod: &Pod{
state: State{
State: StateRunning,
},
},
}
err := c.checkPodRunning("test_cmd")
assert.Nil(t, err, "%v", err)
}
func TestContainerAddResources(t *testing.T) {
assert := assert.New(t)
c := &Container{}
err := c.addResources()
assert.Nil(err)
c.config = &ContainerConfig{Annotations: make(map[string]string)}
c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodSandbox)
err = c.addResources()
assert.Nil(err)
c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodContainer)
err = c.addResources()
assert.Nil(err)
c.config.Resources = ContainerResources{
CPUQuota: 5000,
CPUPeriod: 1000,
}
c.pod = &Pod{hypervisor: &mockHypervisor{}}
err = c.addResources()
assert.Nil(err)
}
func TestContainerRemoveResources(t *testing.T) {
assert := assert.New(t)
c := &Container{}
err := c.addResources()
assert.Nil(err)
c.config = &ContainerConfig{Annotations: make(map[string]string)}
c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodSandbox)
err = c.removeResources()
assert.Nil(err)
c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodContainer)
err = c.removeResources()
assert.Nil(err)
c.config.Resources = ContainerResources{
CPUQuota: 5000,
CPUPeriod: 1000,
}
c.pod = &Pod{hypervisor: &mockHypervisor{}}
err = c.removeResources()
assert.Nil(err)
}

652
virtcontainers/device.go Normal file
View File

@@ -0,0 +1,652 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"encoding/hex"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/go-ini/ini"
"github.com/sirupsen/logrus"
)
const (
// DeviceVFIO is the VFIO device type
DeviceVFIO = "vfio"
// DeviceBlock is the block device type
DeviceBlock = "block"
// DeviceGeneric is a generic device type
DeviceGeneric = "generic"
)
// Defining this as a variable instead of a const, to allow
// overriding this in the tests.
var sysIOMMUPath = "/sys/kernel/iommu_groups"
var sysDevPrefix = "/sys/dev"
var blockPaths = []string{
"/dev/sd", //SCSI block device
"/dev/hd", //IDE block device
"/dev/vd", //Virtual Block device
"/dev/ida/", //Compaq Intelligent Drive Array devices
}
const (
vfioPath = "/dev/vfio/"
)
// Device is the virtcontainers device interface.
type Device interface {
attach(hypervisor, *Container) error
detach(hypervisor) error
deviceType() string
}
// DeviceInfo is an embedded type that contains device data common to all types of devices.
type DeviceInfo struct {
// Device path on host
HostPath string
// Device path inside the container
ContainerPath string
// Type of device: c, b, u or p
// c , u - character(unbuffered)
// p - FIFO
// b - block(buffered) special file
// More info in mknod(1).
DevType string
// Major, minor numbers for device.
Major int64
Minor int64
// FileMode permission bits for the device.
FileMode os.FileMode
// id of the device owner.
UID uint32
// id of the device group.
GID uint32
// Hotplugged is used to store device state indicating if the
// device was hotplugged.
Hotplugged bool
// ID for the device that is passed to the hypervisor.
ID string
}
func deviceLogger() *logrus.Entry {
return virtLog.WithField("subsystem", "device")
}
// VFIODevice is a vfio device meant to be passed to the hypervisor
// to be used by the Virtual Machine.
type VFIODevice struct {
DeviceType string
DeviceInfo DeviceInfo
BDF string
}
func newVFIODevice(devInfo DeviceInfo) *VFIODevice {
return &VFIODevice{
DeviceType: DeviceVFIO,
DeviceInfo: devInfo,
}
}
func (device *VFIODevice) attach(h hypervisor, c *Container) error {
vfioGroup := filepath.Base(device.DeviceInfo.HostPath)
iommuDevicesPath := filepath.Join(sysIOMMUPath, vfioGroup, "devices")
deviceFiles, err := ioutil.ReadDir(iommuDevicesPath)
if err != nil {
return err
}
// Pass all devices in iommu group
for _, deviceFile := range deviceFiles {
//Get bdf of device eg 0000:00:1c.0
deviceBDF, err := getBDF(deviceFile.Name())
if err != nil {
return err
}
device.BDF = deviceBDF
if err := h.addDevice(*device, vfioDev); err != nil {
deviceLogger().WithError(err).Error("Failed to add device")
return err
}
deviceLogger().WithFields(logrus.Fields{
"device-group": device.DeviceInfo.HostPath,
"device-type": "vfio-passthrough",
}).Info("Device group attached")
}
return nil
}
func (device *VFIODevice) detach(h hypervisor) error {
return nil
}
func (device *VFIODevice) deviceType() string {
return device.DeviceType
}
// VhostUserDeviceType - represents a vhost-user device type
// Currently support just VhostUserNet
type VhostUserDeviceType string
const (
//VhostUserSCSI - SCSI based vhost-user type
VhostUserSCSI = "vhost-user-scsi-pci"
//VhostUserNet - net based vhost-user type
VhostUserNet = "virtio-net-pci"
//VhostUserBlk represents a block vhostuser device type
VhostUserBlk = "vhost-user-blk-pci"
)
// VhostUserDevice represents a vhost-user device. Shared
// attributes of a vhost-user device can be retrieved using
// the Attrs() method. Unique data can be obtained by casting
// the object to the proper type.
type VhostUserDevice interface {
Attrs() *VhostUserDeviceAttrs
Type() string
}
// VhostUserDeviceAttrs represents data shared by most vhost-user devices
type VhostUserDeviceAttrs struct {
DeviceType string
DeviceInfo DeviceInfo
SocketPath string
ID string
}
// VhostUserNetDevice is a network vhost-user based device
type VhostUserNetDevice struct {
VhostUserDeviceAttrs
MacAddress string
}
// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device
func (vhostUserNetDevice *VhostUserNetDevice) Attrs() *VhostUserDeviceAttrs {
return &vhostUserNetDevice.VhostUserDeviceAttrs
}
// Type returns the type associated with the vhost-user device
func (vhostUserNetDevice *VhostUserNetDevice) Type() string {
return VhostUserNet
}
// VhostUserSCSIDevice is a SCSI vhost-user based device
type VhostUserSCSIDevice struct {
VhostUserDeviceAttrs
}
// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device
func (vhostUserSCSIDevice *VhostUserSCSIDevice) Attrs() *VhostUserDeviceAttrs {
return &vhostUserSCSIDevice.VhostUserDeviceAttrs
}
// Type returns the type associated with the vhost-user device
func (vhostUserSCSIDevice *VhostUserSCSIDevice) Type() string {
return VhostUserSCSI
}
// VhostUserBlkDevice is a block vhost-user based device
type VhostUserBlkDevice struct {
VhostUserDeviceAttrs
}
// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device
func (vhostUserBlkDevice *VhostUserBlkDevice) Attrs() *VhostUserDeviceAttrs {
return &vhostUserBlkDevice.VhostUserDeviceAttrs
}
// Type returns the type associated with the vhost-user device
func (vhostUserBlkDevice *VhostUserBlkDevice) Type() string {
return VhostUserBlk
}
// vhostUserAttach handles the common logic among all of the vhost-user device's
// attach functions
func vhostUserAttach(device VhostUserDevice, h hypervisor, c *Container) (err error) {
// generate a unique ID to be used for hypervisor commandline fields
randBytes, err := generateRandomBytes(8)
if err != nil {
return err
}
id := hex.EncodeToString(randBytes)
device.Attrs().ID = id
return h.addDevice(device, vhostuserDev)
}
//
// VhostUserNetDevice's implementation of the device interface:
//
func (vhostUserNetDevice *VhostUserNetDevice) attach(h hypervisor, c *Container) (err error) {
return vhostUserAttach(vhostUserNetDevice, h, c)
}
func (vhostUserNetDevice *VhostUserNetDevice) detach(h hypervisor) error {
return nil
}
func (vhostUserNetDevice *VhostUserNetDevice) deviceType() string {
return vhostUserNetDevice.DeviceType
}
//
// VhostUserBlkDevice's implementation of the device interface:
//
func (vhostUserBlkDevice *VhostUserBlkDevice) attach(h hypervisor, c *Container) (err error) {
return vhostUserAttach(vhostUserBlkDevice, h, c)
}
func (vhostUserBlkDevice *VhostUserBlkDevice) detach(h hypervisor) error {
return nil
}
func (vhostUserBlkDevice *VhostUserBlkDevice) deviceType() string {
return vhostUserBlkDevice.DeviceType
}
//
// VhostUserSCSIDevice's implementation of the device interface:
//
func (vhostUserSCSIDevice *VhostUserSCSIDevice) attach(h hypervisor, c *Container) (err error) {
return vhostUserAttach(vhostUserSCSIDevice, h, c)
}
func (vhostUserSCSIDevice *VhostUserSCSIDevice) detach(h hypervisor) error {
return nil
}
func (vhostUserSCSIDevice *VhostUserSCSIDevice) deviceType() string {
return vhostUserSCSIDevice.DeviceType
}
// Long term, this should be made more configurable. For now matching path
// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and
// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system
// using this path.
const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
// findVhostUserNetSocketPath checks if an interface is a dummy placeholder
// for a vhost-user socket, and if it is it returns the path to the socket
func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) {
if netInfo.Iface.Name == "lo" {
return "", nil
}
// check for socket file existence at known location.
for _, addr := range netInfo.Addrs {
socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP)
if _, err := os.Stat(socketPath); err == nil {
return socketPath, nil
}
}
return "", nil
}
// vhostUserSocketPath returns the path of the socket discovered. This discovery
// will vary depending on the type of vhost-user socket.
// Today only VhostUserNetDevice is supported.
func vhostUserSocketPath(info interface{}) (string, error) {
switch v := info.(type) {
case NetworkInfo:
return findVhostUserNetSocketPath(v)
default:
return "", nil
}
}
// BlockDevice refers to a block storage device implementation.
type BlockDevice struct {
DeviceType string
DeviceInfo DeviceInfo
// SCSI Address of the block device, in case the device is attached using SCSI driver
// SCSI address is in the format SCSI-Id:LUN
SCSIAddr string
// Path at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
}
func newBlockDevice(devInfo DeviceInfo) *BlockDevice {
return &BlockDevice{
DeviceType: DeviceBlock,
DeviceInfo: devInfo,
}
}
func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) {
randBytes, err := generateRandomBytes(8)
if err != nil {
return err
}
device.DeviceInfo.ID = hex.EncodeToString(randBytes)
// Increment the block index for the pod. This is used to determine the name
// for the block device in the case where the block device is used as container
// rootfs and the predicted block device name needs to be provided to the agent.
index, err := c.pod.getAndSetPodBlockIndex()
defer func() {
if err != nil {
c.pod.decrementPodBlockIndex()
}
}()
if err != nil {
return err
}
drive := Drive{
File: device.DeviceInfo.HostPath,
Format: "raw",
ID: makeNameID("drive", device.DeviceInfo.ID),
Index: index,
}
driveName, err := getVirtDriveName(index)
if err != nil {
return err
}
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
if err = h.hotplugAddDevice(drive, blockDev); err != nil {
return err
}
device.DeviceInfo.Hotplugged = true
if c.pod.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
device.VirtPath = filepath.Join("/dev", driveName)
} else {
scsiAddr, err := getSCSIAddress(index)
if err != nil {
return err
}
device.SCSIAddr = scsiAddr
}
return nil
}
func (device BlockDevice) detach(h hypervisor) error {
if device.DeviceInfo.Hotplugged {
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
drive := Drive{
ID: makeNameID("drive", device.DeviceInfo.ID),
}
if err := h.hotplugRemoveDevice(drive, blockDev); err != nil {
deviceLogger().WithError(err).Error("Failed to unplug block device")
return err
}
}
return nil
}
func (device *BlockDevice) deviceType() string {
return device.DeviceType
}
// GenericDevice refers to a device that is neither a VFIO device or block device.
type GenericDevice struct {
DeviceType string
DeviceInfo DeviceInfo
}
func newGenericDevice(devInfo DeviceInfo) *GenericDevice {
return &GenericDevice{
DeviceType: DeviceGeneric,
DeviceInfo: devInfo,
}
}
func (device *GenericDevice) attach(h hypervisor, c *Container) error {
return nil
}
func (device *GenericDevice) detach(h hypervisor) error {
return nil
}
func (device *GenericDevice) deviceType() string {
return device.DeviceType
}
// isVFIO checks if the device provided is a vfio group.
func isVFIO(hostPath string) bool {
// Ignore /dev/vfio/vfio character device
if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) {
return false
}
if strings.HasPrefix(hostPath, vfioPath) && len(hostPath) > len(vfioPath) {
return true
}
return false
}
// isBlock checks if the device is a block device.
func isBlock(devInfo DeviceInfo) bool {
if devInfo.DevType == "b" {
return true
}
return false
}
func createDevice(devInfo DeviceInfo) Device {
path := devInfo.HostPath
if isVFIO(path) {
return newVFIODevice(devInfo)
} else if isBlock(devInfo) {
return newBlockDevice(devInfo)
} else {
deviceLogger().WithField("device", path).Info("Device has not been passed to the container")
return newGenericDevice(devInfo)
}
}
// GetHostPath is used to fetcg the host path for the device.
// The path passed in the spec refers to the path that should appear inside the container.
// We need to find the actual device path on the host based on the major-minor numbers of the device.
func GetHostPath(devInfo DeviceInfo) (string, error) {
if devInfo.ContainerPath == "" {
return "", fmt.Errorf("Empty path provided for device")
}
var pathComp string
switch devInfo.DevType {
case "c", "u":
pathComp = "char"
case "b":
pathComp = "block"
default:
// Unsupported device types. Return nil error to ignore devices
// that cannot be handled currently.
return "", nil
}
format := strconv.FormatInt(devInfo.Major, 10) + ":" + strconv.FormatInt(devInfo.Minor, 10)
sysDevPath := filepath.Join(sysDevPrefix, pathComp, format, "uevent")
if _, err := os.Stat(sysDevPath); err != nil {
// Some devices(eg. /dev/fuse, /dev/cuse) do not always implement sysfs interface under /sys/dev
// These devices are passed by default by docker.
//
// Simply return the path passed in the device configuration, this does mean that no device renames are
// supported for these devices.
if os.IsNotExist(err) {
return devInfo.ContainerPath, nil
}
return "", err
}
content, err := ini.Load(sysDevPath)
if err != nil {
return "", err
}
devName, err := content.Section("").GetKey("DEVNAME")
if err != nil {
return "", err
}
return filepath.Join("/dev", devName.String()), nil
}
// GetHostPathFunc is function pointer used to mock GetHostPath in tests.
var GetHostPathFunc = GetHostPath
func newDevices(devInfos []DeviceInfo) ([]Device, error) {
var devices []Device
for _, devInfo := range devInfos {
hostPath, err := GetHostPathFunc(devInfo)
if err != nil {
return nil, err
}
devInfo.HostPath = hostPath
device := createDevice(devInfo)
devices = append(devices, device)
}
return devices, nil
}
// getBDF returns the BDF of pci device
// Expected input strng format is [<domain>]:[<bus>][<slot>].[<func>] eg. 0000:02:10.0
func getBDF(deviceSysStr string) (string, error) {
tokens := strings.Split(deviceSysStr, ":")
if len(tokens) != 3 {
return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr)
}
tokens = strings.SplitN(deviceSysStr, ":", 2)
return tokens[1], nil
}
// bind/unbind paths to aid in SRIOV VF bring-up/restore
var pciDriverUnbindPath = "/sys/bus/pci/devices/%s/driver/unbind"
var pciDriverBindPath = "/sys/bus/pci/drivers/%s/bind"
var vfioRemoveIDPath = "/sys/bus/pci/drivers/vfio-pci/remove_id"
var vfioNewIDPath = "/sys/bus/pci/drivers/vfio-pci/new_id"
// bindDevicetoVFIO binds the device to vfio driver after unbinding from host.
// Will be called by a network interface or a generic pcie device.
func bindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) error {
// Unbind from the host driver
unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf)
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": unbindDriverPath,
}).Info("Unbinding device from driver")
if err := writeToFile(unbindDriverPath, []byte(bdf)); err != nil {
return err
}
// Add device id to vfio driver.
deviceLogger().WithFields(logrus.Fields{
"vendor-device-id": vendorDeviceID,
"vfio-new-id-path": vfioNewIDPath,
}).Info("Writing vendor-device-id to vfio new-id path")
if err := writeToFile(vfioNewIDPath, []byte(vendorDeviceID)); err != nil {
return err
}
// Bind to vfio-pci driver.
bindDriverPath := fmt.Sprintf(pciDriverBindPath, "vfio-pci")
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": bindDriverPath,
}).Info("Binding device to vfio driver")
// Device may be already bound at this time because of earlier write to new_id, ignore error
writeToFile(bindDriverPath, []byte(bdf))
return nil
}
// bindDevicetoHost binds the device to the host driver driver after unbinding from vfio-pci.
func bindDevicetoHost(bdf, hostDriver, vendorDeviceID string) error {
// Unbind from vfio-pci driver
unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf)
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": unbindDriverPath,
}).Info("Unbinding device from driver")
if err := writeToFile(unbindDriverPath, []byte(bdf)); err != nil {
return err
}
// To prevent new VFs from binding to VFIO-PCI, remove_id
if err := writeToFile(vfioRemoveIDPath, []byte(vendorDeviceID)); err != nil {
return err
}
// Bind back to host driver
bindDriverPath := fmt.Sprintf(pciDriverBindPath, hostDriver)
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": bindDriverPath,
}).Info("Binding back device to host driver")
return writeToFile(bindDriverPath, []byte(bdf))
}

402
virtcontainers/device_test.go Normal file
View File

@@ -0,0 +1,402 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
"github.com/vishvananda/netlink"
)
const fileMode0640 = os.FileMode(0640)
func TestVhostUserSocketPath(t *testing.T) {
// First test case: search for existing:
addresses := []netlink.Addr{
{
IPNet: &net.IPNet{
IP: net.IPv4(192, 168, 0, 2),
Mask: net.IPv4Mask(192, 168, 0, 2),
},
},
{
IPNet: &net.IPNet{
IP: net.IPv4(192, 168, 0, 1),
Mask: net.IPv4Mask(192, 168, 0, 1),
},
},
}
expectedPath := "/tmp/vhostuser_192.168.0.1"
expectedFileName := "vhu.sock"
expectedResult := fmt.Sprintf("%s/%s", expectedPath, expectedFileName)
err := os.Mkdir(expectedPath, 0777)
if err != nil {
t.Fatal(err)
}
_, err = os.Create(expectedResult)
if err != nil {
t.Fatal(err)
}
netinfo := NetworkInfo{
Addrs: addresses,
}
path, _ := vhostUserSocketPath(netinfo)
if path != expectedResult {
t.Fatalf("Got %+v\nExpecting %+v", path, expectedResult)
}
// Second test case: search doesn't include matching vsock:
addressesFalse := []netlink.Addr{
{
IPNet: &net.IPNet{
IP: net.IPv4(192, 168, 0, 4),
Mask: net.IPv4Mask(192, 168, 0, 4),
},
},
}
netinfoFail := NetworkInfo{
Addrs: addressesFalse,
}
path, _ = vhostUserSocketPath(netinfoFail)
if path != "" {
t.Fatalf("Got %+v\nExpecting %+v", path, "")
}
err = os.Remove(expectedResult)
if err != nil {
t.Fatal(err)
}
err = os.Remove(expectedPath)
if err != nil {
t.Fatal(err)
}
}
func TestIsVFIO(t *testing.T) {
type testData struct {
path string
expected bool
}
data := []testData{
{"/dev/vfio/16", true},
{"/dev/vfio/1", true},
{"/dev/vfio/", false},
{"/dev/vfio", false},
{"/dev/vf", false},
{"/dev", false},
{"/dev/vfio/vfio", false},
{"/dev/vfio/vfio/12", false},
}
for _, d := range data {
isVFIO := isVFIO(d.path)
assert.Equal(t, d.expected, isVFIO)
}
}
func TestIsBlock(t *testing.T) {
type testData struct {
devType string
expected bool
}
data := []testData{
{"b", true},
{"c", false},
{"u", false},
}
for _, d := range data {
isBlock := isBlock(DeviceInfo{DevType: d.devType})
assert.Equal(t, d.expected, isBlock)
}
}
func TestCreateDevice(t *testing.T) {
devInfo := DeviceInfo{
HostPath: "/dev/vfio/8",
DevType: "b",
}
device := createDevice(devInfo)
_, ok := device.(*VFIODevice)
assert.True(t, ok)
devInfo.HostPath = "/dev/sda"
device = createDevice(devInfo)
_, ok = device.(*BlockDevice)
assert.True(t, ok)
devInfo.HostPath = "/dev/tty"
devInfo.DevType = "c"
device = createDevice(devInfo)
_, ok = device.(*GenericDevice)
assert.True(t, ok)
}
func TestNewDevices(t *testing.T) {
savedSysDevPrefix := sysDevPrefix
major := int64(252)
minor := int64(3)
tmpDir, err := ioutil.TempDir("", "")
assert.Nil(t, err)
sysDevPrefix = tmpDir
defer func() {
os.RemoveAll(tmpDir)
sysDevPrefix = savedSysDevPrefix
}()
path := "/dev/vfio/2"
deviceInfo := DeviceInfo{
ContainerPath: "",
Major: major,
Minor: minor,
UID: 2,
GID: 2,
DevType: "c",
}
_, err = newDevices([]DeviceInfo{deviceInfo})
assert.NotNil(t, err)
format := strconv.FormatInt(major, 10) + ":" + strconv.FormatInt(minor, 10)
ueventPathPrefix := filepath.Join(sysDevPrefix, "char", format)
ueventPath := filepath.Join(ueventPathPrefix, "uevent")
// Return true for non-existent /sys/dev path.
deviceInfo.ContainerPath = path
_, err = newDevices([]DeviceInfo{deviceInfo})
assert.Nil(t, err)
err = os.MkdirAll(ueventPathPrefix, dirMode)
assert.Nil(t, err)
// Should return error for bad data in uevent file
content := []byte("nonkeyvaluedata")
err = ioutil.WriteFile(ueventPath, content, fileMode0640)
assert.Nil(t, err)
_, err = newDevices([]DeviceInfo{deviceInfo})
assert.NotNil(t, err)
content = []byte("MAJOR=252\nMINOR=3\nDEVNAME=vfio/2")
err = ioutil.WriteFile(ueventPath, content, fileMode0640)
assert.Nil(t, err)
devices, err := newDevices([]DeviceInfo{deviceInfo})
assert.Nil(t, err)
assert.Equal(t, len(devices), 1)
vfioDev, ok := devices[0].(*VFIODevice)
assert.True(t, ok)
assert.Equal(t, vfioDev.DeviceInfo.HostPath, path)
assert.Equal(t, vfioDev.DeviceInfo.ContainerPath, path)
assert.Equal(t, vfioDev.DeviceInfo.DevType, "c")
assert.Equal(t, vfioDev.DeviceInfo.Major, major)
assert.Equal(t, vfioDev.DeviceInfo.Minor, minor)
assert.Equal(t, vfioDev.DeviceInfo.UID, uint32(2))
assert.Equal(t, vfioDev.DeviceInfo.GID, uint32(2))
}
func TestGetBDF(t *testing.T) {
type testData struct {
deviceStr string
expectedBDF string
}
data := []testData{
{"0000:02:10.0", "02:10.0"},
{"0000:0210.0", ""},
{"test", ""},
{"", ""},
}
for _, d := range data {
deviceBDF, err := getBDF(d.deviceStr)
assert.Equal(t, d.expectedBDF, deviceBDF)
if d.expectedBDF == "" {
assert.NotNil(t, err)
} else {
assert.Nil(t, err)
}
}
}
func TestAttachVFIODevice(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "")
assert.Nil(t, err)
os.RemoveAll(tmpDir)
testFDIOGroup := "2"
testDeviceBDFPath := "0000:00:1c.0"
devicesDir := filepath.Join(tmpDir, testFDIOGroup, "devices")
err = os.MkdirAll(devicesDir, dirMode)
assert.Nil(t, err)
deviceFile := filepath.Join(devicesDir, testDeviceBDFPath)
_, err = os.Create(deviceFile)
assert.Nil(t, err)
savedIOMMUPath := sysIOMMUPath
sysIOMMUPath = tmpDir
defer func() {
sysIOMMUPath = savedIOMMUPath
}()
path := filepath.Join(vfioPath, testFDIOGroup)
deviceInfo := DeviceInfo{
HostPath: path,
ContainerPath: path,
DevType: "c",
}
device := createDevice(deviceInfo)
_, ok := device.(*VFIODevice)
assert.True(t, ok)
hypervisor := &mockHypervisor{}
err = device.attach(hypervisor, &Container{})
assert.Nil(t, err)
err = device.detach(hypervisor)
assert.Nil(t, err)
}
func TestAttachGenericDevice(t *testing.T) {
path := "/dev/tty2"
deviceInfo := DeviceInfo{
HostPath: path,
ContainerPath: path,
DevType: "c",
}
device := createDevice(deviceInfo)
_, ok := device.(*GenericDevice)
assert.True(t, ok)
hypervisor := &mockHypervisor{}
err := device.attach(hypervisor, &Container{})
assert.Nil(t, err)
err = device.detach(hypervisor)
assert.Nil(t, err)
}
func TestAttachBlockDevice(t *testing.T) {
fs := &filesystem{}
hypervisor := &mockHypervisor{}
hConfig := HypervisorConfig{
BlockDeviceDriver: VirtioBlock,
}
config := &PodConfig{
HypervisorConfig: hConfig,
}
pod := &Pod{
id: testPodID,
storage: fs,
hypervisor: hypervisor,
config: config,
}
contID := "100"
container := Container{
pod: pod,
id: contID,
}
// create state file
path := filepath.Join(runStoragePath, testPodID, container.ID())
err := os.MkdirAll(path, dirMode)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(path)
stateFilePath := filepath.Join(path, stateFile)
os.Remove(stateFilePath)
_, err = os.Create(stateFilePath)
if err != nil {
t.Fatal(err)
}
defer os.Remove(stateFilePath)
path = "/dev/hda"
deviceInfo := DeviceInfo{
HostPath: path,
ContainerPath: path,
DevType: "b",
}
device := createDevice(deviceInfo)
_, ok := device.(*BlockDevice)
assert.True(t, ok)
container.state.State = ""
err = device.attach(hypervisor, &container)
assert.Nil(t, err)
err = device.detach(hypervisor)
assert.Nil(t, err)
container.state.State = StateReady
err = device.attach(hypervisor, &container)
assert.Nil(t, err)
err = device.detach(hypervisor)
assert.Nil(t, err)
container.pod.config.HypervisorConfig.BlockDeviceDriver = VirtioSCSI
err = device.attach(hypervisor, &container)
assert.Nil(t, err)
err = device.detach(hypervisor)
assert.Nil(t, err)
container.state.State = StateReady
err = device.attach(hypervisor, &container)
assert.Nil(t, err)
err = device.detach(hypervisor)
assert.Nil(t, err)
}

28
virtcontainers/doc.go Normal file
View File

@@ -0,0 +1,28 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
/*
Package virtcontainers manages hardware virtualized containers.
Each container belongs to a set of containers sharing the same networking
namespace and storage, also known as a pod.
Virtcontainers pods are hardware virtualized, i.e. they run on virtual machines.
Virtcontainers will create one VM per pod, and containers will be created as
processes within the pod VM.
The virtcontainers package manages both pods and containers lifecycles.
*/
package virtcontainers

56
virtcontainers/documentation/Developers.md Normal file
View File

@@ -0,0 +1,56 @@
Table of Contents
=================
* [Prerequisites](#prerequisites)
* [Building](#building)
* [Testing](#testing)
* [Submitting changes](#submitting-changes)
# Prerequisites
`virtcontainers` has a few prerequisites for development:
- docker
- CNI
- golang
- gometalinter
A number of these can be installed using the
[virtcontainers-setup.sh](../utils/virtcontainers-setup.sh) script.
# Building
To build `virtcontainers`, at the top level directory run:
```bash
# make
```
# Testing
Before testing `virtcontainers`, ensure you have met the [prerequisites](#prerequisites).
Before testing you need to install virtcontainers. The following command will install
`virtcontainers` into its own area (`/usr/bin/virtcontainers/bin/` by default).
```
# sudo -E PATH=$PATH make install
```
To test `virtcontainers`, at the top level run:
```
# make check
```
This will:
- run static code checks on the code base.
- run `go test` unit tests from the code base.
# Submitting changes
For details on the format and how to submit changes, refer to the
[Contributing](CONTRIBUTING.md) document.

932
virtcontainers/documentation/api/1.0/api.md Normal file
View File

@@ -0,0 +1,932 @@
# virtcontainers 1.0 API
The virtcontainers 1.0 API operates on two high level objects:
[Pods](#pod-api) and [containers](#container-api):
* [Pod API](#pod-api)
* [Container API](#container-api)
* [Examples](#examples)
## Pod API
The virtcontainers 1.0 pod API manages hardware virtualized
[pod lifecycles](#pod-functions). The virtcontainers pod
semantics strictly follow the
[Kubernetes](https://kubernetes.io/docs/concepts/workloads/pods/pod/) ones.
The pod API allows callers to [create](#createpod), [delete](#deletepod),
[start](#startpod), [stop](#stoppod), [run](#runpod), [pause](#pausepod),
[resume](resumepod) and [list](#listpod) VM (Virtual Machine) based pods.
To initially create a pod, the API caller must prepare a
[`PodConfig`](#podconfig) and pass it to either [`CreatePod`](#createpod)
or [`RunPod`](#runpod). Upon successful pod creation, the virtcontainers
API will return a [`VCPod`](#vcpod) interface back to the caller.
The `VCPod` interface is a pod abstraction hiding the internal and private
virtcontainers pod structure. It is a handle for API callers to manage the
pod lifecycle through the rest of the [pod API](#pod-functions).
* [Structures](#pod-structures)
* [Functions](#pod-functions)
### Pod Structures
* [PodConfig](#podconfig)
* [Resources](#resources)
* [HypervisorType](#hypervisortype)
* [HypervisorConfig](#hypervisorconfig)
* [AgentType](#agenttype)
* [ProxyType](#proxytype)
* [ProxyConfig](#proxyconfig)
* [ShimType](#shimtype)
* [NetworkModel](#networkmodel)
* [NetworkConfig](#networkconfig)
* [NetInterworkingModel](#netinterworkingmodel)
* [Volume](#volume)
* [ContainerConfig](#containerconfig)
* [Cmd](#cmd)
* [Mount](#mount)
* [DeviceInfo](#deviceinfo)
* [VCPod](#vcpod)
#### `PodConfig`
```Go
// PodConfig is a Pod configuration.
type PodConfig struct {
ID string
Hostname string
// Field specific to OCI specs, needed to setup all the hooks
Hooks Hooks
// VMConfig is the VM configuration to set for this pod.
VMConfig Resources
HypervisorType HypervisorType
HypervisorConfig HypervisorConfig
AgentType AgentType
AgentConfig interface{}
ProxyType ProxyType
ProxyConfig ProxyConfig
ShimType ShimType
ShimConfig interface{}
NetworkModel NetworkModel
NetworkConfig NetworkConfig
// Volumes is a list of shared volumes between the host and the Pod.
Volumes []Volume
// Containers describe the list of containers within a Pod.
// This list can be empty and populated by adding containers
// to the Pod a posteriori.
Containers []ContainerConfig
// Annotations keys must be unique strings and must be name-spaced
// with e.g. reverse domain notation (org.clearlinux.key).
Annotations map[string]string
}
```
##### `Resources`
```Go
// Resources describes VM resources configuration.
type Resources struct {
// VCPUs is the number of available virtual CPUs.
VCPUs uint
// Memory is the amount of available memory in MiB.
Memory uint
}
```
##### `HypervisorType`
```Go
// HypervisorType describes an hypervisor type.
type HypervisorType string
const (
// QemuHypervisor is the QEMU hypervisor.
QemuHypervisor HypervisorType = "qemu"
// MockHypervisor is a mock hypervisor for testing purposes
MockHypervisor HypervisorType = "mock"
)
```
##### `HypervisorConfig`
```Go
// HypervisorConfig is the hypervisor configuration.
type HypervisorConfig struct {
// KernelPath is the guest kernel host path.
KernelPath string
// ImagePath is the guest image host path.
ImagePath string
// FirmwarePath is the bios host path
FirmwarePath string
// MachineAccelerators are machine specific accelerators
MachineAccelerators string
// HypervisorPath is the hypervisor executable host path.
HypervisorPath string
// DisableBlockDeviceUse disallows a block device from being used.
DisableBlockDeviceUse bool
// KernelParams are additional guest kernel parameters.
KernelParams []Param
// HypervisorParams are additional hypervisor parameters.
HypervisorParams []Param
// HypervisorMachineType specifies the type of machine being
// emulated.
HypervisorMachineType string
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available.
Debug bool
// DefaultVCPUs specifies default number of vCPUs for the VM.
// Pod configuration VMConfig.VCPUs overwrites this.
DefaultVCPUs uint32
// DefaultMem specifies default memory size in MiB for the VM.
// Pod configuration VMConfig.Memory overwrites this.
DefaultMemSz uint32
// DefaultBridges specifies default number of bridges for the VM.
// Bridges can be used to hot plug devices
DefaultBridges uint32
// MemPrealloc specifies if the memory should be pre-allocated
MemPrealloc bool
// HugePages specifies if the memory should be pre-allocated from huge pages
HugePages bool
// Realtime Used to enable/disable realtime
Realtime bool
// Mlock is used to control memory locking when Realtime is enabled
// Realtime=true and Mlock=false, allows for swapping out of VM memory
// enabling higher density
Mlock bool
// DisableNestingChecks is used to override customizations performed
// when running on top of another VMM.
DisableNestingChecks bool
}
```
##### `AgentType`
```Go
// AgentType describes the type of guest agent a Pod should run.
type AgentType string
const (
// NoopAgentType is the No-Op agent.
NoopAgentType AgentType = "noop"
// HyperstartAgent is the Hyper hyperstart agent.
HyperstartAgent AgentType = "hyperstart"
// KataContainersAgent is the Kata Containers agent.
KataContainersAgent AgentType = "kata"
// SocketTypeVSOCK is a VSOCK socket type for talking to an agent.
SocketTypeVSOCK = "vsock"
// SocketTypeUNIX is a UNIX socket type for talking to an agent.
// It typically means the agent is living behind a host proxy.
SocketTypeUNIX = "unix"
)
```
##### `ProxyType`
```Go
// ProxyType describes a proxy type.
type ProxyType string
const (
// NoopProxyType is the noopProxy.
NoopProxyType ProxyType = "noopProxy"
// NoProxyType is the noProxy.
NoProxyType ProxyType = "noProxy"
// CCProxyType is the ccProxy.
CCProxyType ProxyType = "ccProxy"
// KataProxyType is the kataProxy.
KataProxyType ProxyType = "kataProxy"
)
```
##### `ProxyConfig`
```Go
// ProxyConfig is a structure storing information needed from any
// proxy in order to be properly initialized.
type ProxyConfig struct {
Path string
Debug bool
}
```
##### `ShimType`
```Go
// ShimType describes a shim type.
type ShimType string
const (
// CCShimType is the ccShim.
CCShimType ShimType = "ccShim"
// NoopShimType is the noopShim.
NoopShimType ShimType = "noopShim"
// KataShimType is the Kata Containers shim type.
KataShimType ShimType = "kataShim"
)
```
##### `NetworkModel`
```Go
// NetworkModel describes the type of network specification.
type NetworkModel string
const (
// NoopNetworkModel is the No-Op network.
NoopNetworkModel NetworkModel = "noop"
// CNINetworkModel is the CNI network.
CNINetworkModel NetworkModel = "CNI"
// CNMNetworkModel is the CNM network.
CNMNetworkModel NetworkModel = "CNM"
)
```
##### `NetworkConfig`
```Go
// NetworkConfig is the network configuration related to a network.
type NetworkConfig struct {
NetNSPath string
NumInterfaces int
InterworkingModel NetInterworkingModel
}
```
###### `NetInterworkingModel`
```Go
// NetInterworkingModel defines the network model connecting
// the network interface to the virtual machine.
type NetInterworkingModel int
const (
// NetXConnectDefaultModel Ask to use DefaultNetInterworkingModel
NetXConnectDefaultModel NetInterworkingModel = iota
// NetXConnectBridgedModel uses a linux bridge to interconnect
// the container interface to the VM. This is the
// safe default that works for most cases except
// macvlan and ipvlan
NetXConnectBridgedModel
// NetXConnectMacVtapModel can be used when the Container network
// interface can be bridged using macvtap
NetXConnectMacVtapModel
// NetXConnectEnlightenedModel can be used when the Network plugins
// are enlightened to create VM native interfaces
// when requested by the runtime
// This will be used for vethtap, macvtap, ipvtap
NetXConnectEnlightenedModel
// NetXConnectInvalidModel is the last item to check valid values by IsValid()
NetXConnectInvalidModel
)
```
##### `Volume`
```Go
// Volume is a shared volume between the host and the VM,
// defined by its mount tag and its host path.
type Volume struct {
// MountTag is a label used as a hint to the guest.
MountTag string
// HostPath is the host filesystem path for this volume.
HostPath string
}
```
##### `ContainerConfig`
```Go
// ContainerConfig describes one container runtime configuration.
type ContainerConfig struct {
ID string
// RootFs is the container workload image on the host.
RootFs string
// ReadOnlyRootfs indicates if the rootfs should be mounted readonly
ReadonlyRootfs bool
// Cmd specifies the command to run on a container
Cmd Cmd
// Annotations allow clients to store arbitrary values,
// for example to add additional status values required
// to support particular specifications.
Annotations map[string]string
Mounts []Mount
// Device configuration for devices that must be available within the container.
DeviceInfos []DeviceInfo
}
```
###### `Cmd`
```Go
// Cmd represents a command to execute in a running container.
type Cmd struct {
Args []string
Envs []EnvVar
WorkDir string
// Note that these fields *MUST* remain as strings.
//
// The reason being that we want runtimes to be able to support CLI
// operations like "exec --user=". That option allows the
// specification of a user (either as a string username or a numeric
// UID), and may optionally also include a group (groupame or GID).
//
// Since this type is the interface to allow the runtime to specify
// the user and group the workload can run as, these user and group
// fields cannot be encoded as integer values since that would imply
// the runtime itself would need to perform a UID/GID lookup on the
// user-specified username/groupname. But that isn't practically
// possible given that to do so would require the runtime to access
// the image to allow it to interrogate the appropriate databases to
// convert the username/groupnames to UID/GID values.
//
// Note that this argument applies solely to the _runtime_ supporting
// a "--user=" option when running in a "standalone mode" - there is
// no issue when the runtime is called by a container manager since
// all the user and group mapping is handled by the container manager
// and specified to the runtime in terms of UID/GID's in the
// configuration file generated by the container manager.
User string
PrimaryGroup string
SupplementaryGroups []string
Interactive bool
Console string
Detach bool
NoNewPrivileges bool
Capabilities LinuxCapabilities
}
```
###### `Mount`
```Go
// Mount describes a container mount.
type Mount struct {
Source string
Destination string
// Type specifies the type of filesystem to mount.
Type string
// Options list all the mount options of the filesystem.
Options []string
// HostPath used to store host side bind mount path
HostPath string
// ReadOnly specifies if the mount should be read only or not
ReadOnly bool
}
```
###### `DeviceInfo`
```Go
// DeviceInfo is an embedded type that contains device data common to all types of devices.
type DeviceInfo struct {
// Device path on host
HostPath string
// Device path inside the container
ContainerPath string
// Type of device: c, b, u or p
// c , u - character(unbuffered)
// p - FIFO
// b - block(buffered) special file
// More info in mknod(1).
DevType string
// Major, minor numbers for device.
Major int64
Minor int64
// FileMode permission bits for the device.
FileMode os.FileMode
// id of the device owner.
UID uint32
// id of the device group.
GID uint32
// Hotplugged is used to store device state indicating if the
// device was hotplugged.
Hotplugged bool
// ID for the device that is passed to the hypervisor.
ID string
}
```
#### `VCPod`
```Go
// VCPod is the Pod interface
// (required since virtcontainers.Pod only contains private fields)
type VCPod interface {
Annotations(key string) (string, error)
GetAllContainers() []VCContainer
GetAnnotations() map[string]string
GetContainer(containerID string) VCContainer
ID() string
SetAnnotations(annotations map[string]string) error
}
```
### Pod Functions
* [CreatePod](#createpod)
* [DeletePod](#deletepod)
* [StartPod](#startpod)
* [StopPod](#stoppod)
* [RunPod](#runpod)
* [ListPod](#listpod)
* [StatusPod](#statuspod)
* [PausePod](#pausepod)
* [ResumePod](#resumepod)
#### `CreatePod`
```Go
// CreatePod is the virtcontainers pod creation entry point.
// CreatePod creates a pod and its containers. It does not start them.
func CreatePod(podConfig PodConfig) (VCPod, error)
```
#### `DeletePod`
```Go
// DeletePod is the virtcontainers pod deletion entry point.
// DeletePod will stop an already running container and then delete it.
func DeletePod(podID string) (VCPod, error)
```
#### `StartPod`
```Go
// StartPod is the virtcontainers pod starting entry point.
// StartPod will talk to the given hypervisor to start an existing
// pod and all its containers.
func StartPod(podID string) (VCPod, error)
```
#### `StopPod`
```Go
// StopPod is the virtcontainers pod stopping entry point.
// StopPod will talk to the given agent to stop an existing pod
// and destroy all containers within that pod.
func StopPod(podID string) (VCPod, error)
```
#### `RunPod`
```Go
// RunPod is the virtcontainers pod running entry point.
// RunPod creates a pod and its containers and then it starts them.
func RunPod(podConfig PodConfig) (VCPod, error)
```
#### `ListPod`
```Go
// ListPod is the virtcontainers pod listing entry point.
func ListPod() ([]PodStatus, error)
```
#### `StatusPod`
```Go
// StatusPod is the virtcontainers pod status entry point.
func StatusPod(podID string) (PodStatus, error)
```
#### `PausePod`
```Go
// PausePod is the virtcontainers pausing entry point which pauses an
// already running pod.
func PausePod(podID string) (VCPod, error)
```
#### `ResumePod`
```Go
// ResumePod is the virtcontainers resuming entry point which resumes
// (or unpauses) and already paused pod.
func ResumePod(podID string) (VCPod, error)
```
## Container API
The virtcontainers 1.0 container API manages pod
[container lifecycles](#container-functions).
A virtcontainers container is process running inside a containerized
environment, as part of a hardware virtualized context. In other words,
a virtcontainers container is just a regular container running inside a
virtual machine's guest OS.
A virtcontainers container always belong to one and only one
virtcontainers pod, again following the
[Kubernetes](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/)
logic and semantics.
The container API allows callers to [create](#createcontainer),
[delete](#deletecontainer), [start](#startcontainer), [stop](#stopcontainer),
[kill](#killcontainer) and [observe](#statuscontainer) containers. It also
allows for running [additional processes](#entercontainer) inside a
specific container.
As a virtcontainers container is always linked to a pod, the entire container
API always takes a pod ID as its first argument.
To create a container, the API caller must prepare a
[`ContainerConfig`](#containerconfig) and pass it to
[`CreateContainer`](#createcontainer) together with a pod ID. Upon successful
container creation, the virtcontainers API will return a
[`VCContainer`](#vccontainer) interface back to the caller.
The `VCContainer` interface is a container abstraction hiding the internal
and private virtcontainers container structure. It is a handle for API callers
to manage the container lifecycle through the rest of the
[container API](#container-functions).
* [Structures](#container-structures)
* [Functions](#container-functions)
### Container Structures
* [ContainerConfig](#containerconfig-1)
* [Cmd](#cmd-1)
* [Mount](#mount-1)
* [DeviceInfo](#deviceinfo-1)
* [Process](#process)
* [ContainerStatus](#containerstatus)
* [ProcessListOptions](#processlistoptions)
* [VCContainer](#vccontainer)
#### `ContainerConfig`
```Go
// ContainerConfig describes one container runtime configuration.
type ContainerConfig struct {
ID string
// RootFs is the container workload image on the host.
RootFs string
// ReadOnlyRootfs indicates if the rootfs should be mounted readonly
ReadonlyRootfs bool
// Cmd specifies the command to run on a container
Cmd Cmd
// Annotations allow clients to store arbitrary values,
// for example to add additional status values required
// to support particular specifications.
Annotations map[string]string
Mounts []Mount
// Device configuration for devices that must be available within the container.
DeviceInfos []DeviceInfo
}
```
##### `Cmd`
```Go
// Cmd represents a command to execute in a running container.
type Cmd struct {
Args []string
Envs []EnvVar
WorkDir string
// Note that these fields *MUST* remain as strings.
//
// The reason being that we want runtimes to be able to support CLI
// operations like "exec --user=". That option allows the
// specification of a user (either as a string username or a numeric
// UID), and may optionally also include a group (groupame or GID).
//
// Since this type is the interface to allow the runtime to specify
// the user and group the workload can run as, these user and group
// fields cannot be encoded as integer values since that would imply
// the runtime itself would need to perform a UID/GID lookup on the
// user-specified username/groupname. But that isn't practically
// possible given that to do so would require the runtime to access
// the image to allow it to interrogate the appropriate databases to
// convert the username/groupnames to UID/GID values.
//
// Note that this argument applies solely to the _runtime_ supporting
// a "--user=" option when running in a "standalone mode" - there is
// no issue when the runtime is called by a container manager since
// all the user and group mapping is handled by the container manager
// and specified to the runtime in terms of UID/GID's in the
// configuration file generated by the container manager.
User string
PrimaryGroup string
SupplementaryGroups []string
Interactive bool
Console string
Detach bool
NoNewPrivileges bool
Capabilities LinuxCapabilities
}
```
##### `Mount`
```Go
// Mount describes a container mount.
type Mount struct {
Source string
Destination string
// Type specifies the type of filesystem to mount.
Type string
// Options list all the mount options of the filesystem.
Options []string
// HostPath used to store host side bind mount path
HostPath string
// ReadOnly specifies if the mount should be read only or not
ReadOnly bool
}
```
##### `DeviceInfo`
```Go
// DeviceInfo is an embedded type that contains device data common to all types of devices.
type DeviceInfo struct {
// Device path on host
HostPath string
// Device path inside the container
ContainerPath string
// Type of device: c, b, u or p
// c , u - character(unbuffered)
// p - FIFO
// b - block(buffered) special file
// More info in mknod(1).
DevType string
// Major, minor numbers for device.
Major int64
Minor int64
// FileMode permission bits for the device.
FileMode os.FileMode
// id of the device owner.
UID uint32
// id of the device group.
GID uint32
// Hotplugged is used to store device state indicating if the
// device was hotplugged.
Hotplugged bool
// ID for the device that is passed to the hypervisor.
ID string
}
```
#### `Process`
```Go
// Process gathers data related to a container process.
type Process struct {
// Token is the process execution context ID. It must be
// unique per pod.
// Token is used to manipulate processes for containers
// that have not started yet, and later identify them
// uniquely within a pod.
Token string
// Pid is the process ID as seen by the host software
// stack, e.g. CRI-O, containerd. This is typically the
// shim PID.
Pid int
StartTime time.Time
}
```
#### `ContainerStatus`
```Go
// ContainerStatus describes a container status.
type ContainerStatus struct {
ID string
State State
PID int
StartTime time.Time
RootFs string
// Annotations allow clients to store arbitrary values,
// for example to add additional status values required
// to support particular specifications.
Annotations map[string]string
}
```
#### `ProcessListOptions`
```Go
// ProcessListOptions contains the options used to list running
// processes inside the container
type ProcessListOptions struct {
// Format describes the output format to list the running processes.
// Formats are unrelated to ps(1) formats, only two formats can be specified:
// "json" and "table"
Format string
// Args contains the list of arguments to run ps(1) command.
// If Args is empty the agent will use "-ef" as options to ps(1).
Args []string
}
```
#### `VCContainer`
```Go
// VCContainer is the Container interface
// (required since virtcontainers.Container only contains private fields)
type VCContainer interface {
GetAnnotations() map[string]string
GetPid() int
GetToken() string
ID() string
Pod() VCPod
Process() Process
SetPid(pid int) error
}
```
### Container Functions
* [CreateContainer](#createcontainer)
* [DeleteContainer](#deletecontainer)
* [StartContainer](#startcontainer)
* [StopContainer](#stopcontainer)
* [EnterContainer](#entercontainer)
* [StatusContainer](#statuscontainer)
* [KillContainer](#killcontainer)
* [ProcessListContainer](#processlistcontainer)
#### `CreateContainer`
```Go
// CreateContainer is the virtcontainers container creation entry point.
// CreateContainer creates a container on a given pod.
func CreateContainer(podID string, containerConfig ContainerConfig) (VCPod, VCContainer, error)
```
#### `DeleteContainer`
```Go
// DeleteContainer is the virtcontainers container deletion entry point.
// DeleteContainer deletes a Container from a Pod. If the container is running,
// it needs to be stopped first.
func DeleteContainer(podID, containerID string) (VCContainer, error)
```
#### `StartContainer`
```Go
// StartContainer is the virtcontainers container starting entry point.
// StartContainer starts an already created container.
func StartContainer(podID, containerID string) (VCContainer, error)
```
#### `StopContainer`
```Go
// StopContainer is the virtcontainers container stopping entry point.
// StopContainer stops an already running container.
func StopContainer(podID, containerID string) (VCContainer, error)
```
#### `EnterContainer`
```Go
// EnterContainer is the virtcontainers container command execution entry point.
// EnterContainer enters an already running container and runs a given command.
func EnterContainer(podID, containerID string, cmd Cmd) (VCPod, VCContainer, *Process, error)
```
#### `StatusContainer`
```Go
// StatusContainer is the virtcontainers container status entry point.
// StatusContainer returns a detailed container status.
func StatusContainer(podID, containerID string) (ContainerStatus, error)
```
#### `KillContainer`
```Go
// KillContainer is the virtcontainers entry point to send a signal
// to a container running inside a pod. If all is true, all processes in
// the container will be sent the signal.
func KillContainer(podID, containerID string, signal syscall.Signal, all bool) error
```
#### `ProcessListContainer`
```Go
// ProcessListContainer is the virtcontainers entry point to list
// processes running inside a container
func ProcessListContainer(podID, containerID string, options ProcessListOptions) (ProcessList, error)
```
## Examples
### Preparing and running a pod
```Go
// This example creates and starts a single container pod,
// using qemu as the hypervisor and hyperstart as the VM agent.
func Example_createAndStartPod() {
envs := []vc.EnvVar{
{
Var: "PATH",
Value: "/bin:/usr/bin:/sbin:/usr/sbin",
},
}
cmd := vc.Cmd{
Args: strings.Split("/bin/sh", " "),
Envs: envs,
WorkDir: "/",
}
// Define the container command and bundle.
container := vc.ContainerConfig{
ID: "1",
RootFs: containerRootfs,
Cmd: cmd,
}
// Sets the hypervisor configuration.
hypervisorConfig := vc.HypervisorConfig{
KernelPath: "/usr/share/clear-containers/vmlinux.container",
ImagePath: "/usr/share/clear-containers/clear-containers.img",
HypervisorPath: "/usr/bin/qemu-lite-system-x86_64",
}
// Use hyperstart default values for the agent.
agConfig := vc.HyperConfig{}
// VM resources
vmConfig := vc.Resources{
VCPUs: 4,
Memory: 1024,
}
// The pod configuration:
// - One container
// - Hypervisor is QEMU
// - Agent is hyperstart
podConfig := vc.PodConfig{
VMConfig: vmConfig,
HypervisorType: vc.QemuHypervisor,
HypervisorConfig: hypervisorConfig,
AgentType: vc.HyperstartAgent,
AgentConfig: agConfig,
Containers: []vc.ContainerConfig{container},
}
_, err := vc.RunPod(podConfig)
if err != nil {
fmt.Printf("Could not run pod: %s", err)
}
return
}
```

BIN
virtcontainers/documentation/network/CNI_diagram.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 23 KiB

BIN
virtcontainers/documentation/network/CNM_detailed_diagram.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 23 KiB

BIN
virtcontainers/documentation/network/CNM_overall_diagram.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 25 KiB

31
virtcontainers/errors.go Normal file
View File

@@ -0,0 +1,31 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"errors"
)
// common error objects used for argument checking
var (
errNeedPod = errors.New("Pod must be specified")
errNeedPodID = errors.New("Pod ID cannot be empty")
errNeedContainerID = errors.New("Container ID cannot be empty")
errNeedFile = errors.New("File cannot be empty")
errNeedState = errors.New("State cannot be empty")
errInvalidResource = errors.New("Invalid pod resource")
)

88
virtcontainers/example_pod_run_test.go Normal file
View File

@@ -0,0 +1,88 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers_test
import (
"fmt"
"strings"
vc "github.com/containers/virtcontainers"
)
const containerRootfs = "/var/lib/container/bundle/"
// This example creates and starts a single container pod,
// using qemu as the hypervisor and hyperstart as the VM agent.
func Example_createAndStartPod() {
envs := []vc.EnvVar{
{
Var: "PATH",
Value: "/bin:/usr/bin:/sbin:/usr/sbin",
},
}
cmd := vc.Cmd{
Args: strings.Split("/bin/sh", " "),
Envs: envs,
WorkDir: "/",
}
// Define the container command and bundle.
container := vc.ContainerConfig{
ID: "1",
RootFs: containerRootfs,
Cmd: cmd,
}
// Sets the hypervisor configuration.
hypervisorConfig := vc.HypervisorConfig{
KernelPath: "/usr/share/clear-containers/vmlinux.container",
ImagePath: "/usr/share/clear-containers/clear-containers.img",
HypervisorPath: "/usr/bin/qemu-lite-system-x86_64",
}
// Use hyperstart default values for the agent.
agConfig := vc.HyperConfig{}
// VM resources
vmConfig := vc.Resources{
Memory: 1024,
}
// The pod configuration:
// - One container
// - Hypervisor is QEMU
// - Agent is hyperstart
podConfig := vc.PodConfig{
VMConfig: vmConfig,
HypervisorType: vc.QemuHypervisor,
HypervisorConfig: hypervisorConfig,
AgentType: vc.HyperstartAgent,
AgentConfig: agConfig,
Containers: []vc.ContainerConfig{container},
}
_, err := vc.RunPod(podConfig)
if err != nil {
fmt.Printf("Could not run pod: %s", err)
}
return
}

767
virtcontainers/filesystem.go Normal file
View File

@@ -0,0 +1,767 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
)
// podResource is an int representing a pod resource type.
//
// Note that some are specific to the pod itself and others can apply to
// pods and containers.
type podResource int
const (
// configFileType represents a configuration file type
configFileType podResource = iota
// stateFileType represents a state file type
stateFileType
// networkFileType represents a network file type (pod only)
networkFileType
// hypervisorFileType represents a hypervisor file type (pod only)
hypervisorFileType
// agentFileType represents an agent file type (pod only)
agentFileType
// processFileType represents a process file type
processFileType
// lockFileType represents a lock file type (pod only)
lockFileType
// mountsFileType represents a mount file type
mountsFileType
// devicesFileType represents a device file type
devicesFileType
)
// configFile is the file name used for every JSON pod configuration.
const configFile = "config.json"
// stateFile is the file name storing a pod state.
const stateFile = "state.json"
// networkFile is the file name storing a pod network.
const networkFile = "network.json"
// hypervisorFile is the file name storing a hypervisor's state.
const hypervisorFile = "hypervisor.json"
// agentFile is the file name storing an agent's state.
const agentFile = "agent.json"
// processFile is the file name storing a container process.
const processFile = "process.json"
// lockFile is the file name locking the usage of a pod.
const lockFileName = "lock"
const mountsFile = "mounts.json"
// devicesFile is the file name storing a container's devices.
const devicesFile = "devices.json"
// dirMode is the permission bits used for creating a directory
const dirMode = os.FileMode(0750) | os.ModeDir
// storagePathSuffix is the suffix used for all storage paths
const storagePathSuffix = "/virtcontainers/pods"
// configStoragePath is the pod configuration directory.
// It will contain one config.json file for each created pod.
var configStoragePath = filepath.Join("/var/lib", storagePathSuffix)
// runStoragePath is the pod runtime directory.
// It will contain one state.json and one lock file for each created pod.
var runStoragePath = filepath.Join("/run", storagePathSuffix)
// resourceStorage is the virtcontainers resources (configuration, state, etc...)
// storage interface.
// The default resource storage implementation is filesystem.
type resourceStorage interface {
// Create all resources for a pod
createAllResources(pod Pod) error
// Resources URIs functions return both the URI
// for the actual resource and the URI base.
containerURI(podID, containerID string, resource podResource) (string, string, error)
podURI(podID string, resource podResource) (string, string, error)
// Pod resources
storePodResource(podID string, resource podResource, data interface{}) error
deletePodResources(podID string, resources []podResource) error
fetchPodConfig(podID string) (PodConfig, error)
fetchPodState(podID string) (State, error)
fetchPodNetwork(podID string) (NetworkNamespace, error)
storePodNetwork(podID string, networkNS NetworkNamespace) error
// Hypervisor resources
fetchHypervisorState(podID string, state interface{}) error
storeHypervisorState(podID string, state interface{}) error
// Agent resources
fetchAgentState(podID string, state interface{}) error
storeAgentState(podID string, state interface{}) error
// Container resources
storeContainerResource(podID, containerID string, resource podResource, data interface{}) error
deleteContainerResources(podID, containerID string, resources []podResource) error
fetchContainerConfig(podID, containerID string) (ContainerConfig, error)
fetchContainerState(podID, containerID string) (State, error)
fetchContainerProcess(podID, containerID string) (Process, error)
storeContainerProcess(podID, containerID string, process Process) error
fetchContainerMounts(podID, containerID string) ([]Mount, error)
storeContainerMounts(podID, containerID string, mounts []Mount) error
fetchContainerDevices(podID, containerID string) ([]Device, error)
storeContainerDevices(podID, containerID string, devices []Device) error
}
// filesystem is a resourceStorage interface implementation for a local filesystem.
type filesystem struct {
}
// Logger returns a logrus logger appropriate for logging filesystem messages
func (fs *filesystem) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "filesystem")
}
func (fs *filesystem) createAllResources(pod Pod) (err error) {
for _, resource := range []podResource{stateFileType, configFileType} {
_, path, _ := fs.podURI(pod.id, resource)
err = os.MkdirAll(path, dirMode)
if err != nil {
return err
}
}
for _, container := range pod.containers {
for _, resource := range []podResource{stateFileType, configFileType} {
_, path, _ := fs.containerURI(pod.id, container.id, resource)
err = os.MkdirAll(path, dirMode)
if err != nil {
fs.deletePodResources(pod.id, nil)
return err
}
}
}
podlockFile, _, err := fs.podURI(pod.id, lockFileType)
if err != nil {
fs.deletePodResources(pod.id, nil)
return err
}
_, err = os.Stat(podlockFile)
if err != nil {
lockFile, err := os.Create(podlockFile)
if err != nil {
fs.deletePodResources(pod.id, nil)
return err
}
lockFile.Close()
}
return nil
}
func (fs *filesystem) storeFile(file string, data interface{}) error {
if file == "" {
return errNeedFile
}
f, err := os.Create(file)
if err != nil {
return err
}
defer f.Close()
jsonOut, err := json.Marshal(data)
if err != nil {
return fmt.Errorf("Could not marshall data: %s", err)
}
f.Write(jsonOut)
return nil
}
// TypedDevice is used as an intermediate representation for marshalling
// and unmarshalling Device implementations.
type TypedDevice struct {
Type string
// Data is assigned the Device object.
// This being declared as RawMessage prevents it from being marshalled/unmarshalled.
// We do that explicitly depending on Type.
Data json.RawMessage
}
// storeDeviceFile is used to provide custom marshalling for Device objects.
// Device is first marshalled into TypedDevice to include the type
// of the Device object.
func (fs *filesystem) storeDeviceFile(file string, data interface{}) error {
if file == "" {
return errNeedFile
}
f, err := os.Create(file)
if err != nil {
return err
}
defer f.Close()
devices, ok := data.([]Device)
if !ok {
return fmt.Errorf("Incorrect data type received, Expected []Device")
}
var typedDevices []TypedDevice
for _, d := range devices {
tempJSON, _ := json.Marshal(d)
typedDevice := TypedDevice{
Type: d.deviceType(),
Data: tempJSON,
}
typedDevices = append(typedDevices, typedDevice)
}
jsonOut, err := json.Marshal(typedDevices)
if err != nil {
return fmt.Errorf("Could not marshal devices: %s", err)
}
if _, err := f.Write(jsonOut); err != nil {
return err
}
return nil
}
func (fs *filesystem) fetchFile(file string, resource podResource, data interface{}) error {
if file == "" {
return errNeedFile
}
fileData, err := ioutil.ReadFile(file)
if err != nil {
return err
}
switch resource {
case devicesFileType:
devices, ok := data.(*[]Device)
if !ok {
return fmt.Errorf("Could not cast %v into *[]Device type", data)
}
return fs.fetchDeviceFile(fileData, devices)
}
return json.Unmarshal(fileData, data)
}
// fetchDeviceFile is used for custom unmarshalling of device interface objects.
func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]Device) error {
var typedDevices []TypedDevice
if err := json.Unmarshal(fileData, &typedDevices); err != nil {
return err
}
var tempDevices []Device
for _, d := range typedDevices {
l := fs.Logger().WithField("device-type", d.Type)
l.Info("Device type found")
switch d.Type {
case DeviceVFIO:
var device VFIODevice
if err := json.Unmarshal(d.Data, &device); err != nil {
return err
}
tempDevices = append(tempDevices, &device)
l.Infof("VFIO device unmarshalled [%v]", device)
case DeviceBlock:
var device BlockDevice
if err := json.Unmarshal(d.Data, &device); err != nil {
return err
}
tempDevices = append(tempDevices, &device)
l.Infof("Block Device unmarshalled [%v]", device)
case DeviceGeneric:
var device GenericDevice
if err := json.Unmarshal(d.Data, &device); err != nil {
return err
}
tempDevices = append(tempDevices, &device)
l.Infof("Generic device unmarshalled [%v]", device)
default:
return fmt.Errorf("Unknown device type, could not unmarshal")
}
}
*devices = tempDevices
return nil
}
// resourceNeedsContainerID determines if the specified
// podResource needs a containerID. Since some podResources can
// be used for both pods and containers, it is necessary to specify
// whether the resource is being used in a pod-specific context using
// the podSpecific parameter.
func resourceNeedsContainerID(podSpecific bool, resource podResource) bool {
switch resource {
case lockFileType, networkFileType, hypervisorFileType, agentFileType:
// pod-specific resources
return false
default:
return !podSpecific
}
}
func resourceDir(podSpecific bool, podID, containerID string, resource podResource) (string, error) {
if podID == "" {
return "", errNeedPodID
}
if resourceNeedsContainerID(podSpecific, resource) == true && containerID == "" {
return "", errNeedContainerID
}
var path string
switch resource {
case configFileType:
path = configStoragePath
break
case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType, hypervisorFileType, agentFileType:
path = runStoragePath
break
default:
return "", errInvalidResource
}
dirPath := filepath.Join(path, podID, containerID)
return dirPath, nil
}
// If podSpecific is true, the resource is being applied for an empty
// pod (meaning containerID may be blank).
// Note that this function defers determining if containerID can be
// blank to resourceDIR()
func (fs *filesystem) resourceURI(podSpecific bool, podID, containerID string, resource podResource) (string, string, error) {
if podID == "" {
return "", "", errNeedPodID
}
var filename string
dirPath, err := resourceDir(podSpecific, podID, containerID, resource)
if err != nil {
return "", "", err
}
switch resource {
case configFileType:
filename = configFile
break
case stateFileType:
filename = stateFile
case networkFileType:
filename = networkFile
case hypervisorFileType:
filename = hypervisorFile
case agentFileType:
filename = agentFile
case processFileType:
filename = processFile
case lockFileType:
filename = lockFileName
break
case mountsFileType:
filename = mountsFile
break
case devicesFileType:
filename = devicesFile
break
default:
return "", "", errInvalidResource
}
filePath := filepath.Join(dirPath, filename)
return filePath, dirPath, nil
}
func (fs *filesystem) containerURI(podID, containerID string, resource podResource) (string, string, error) {
if podID == "" {
return "", "", errNeedPodID
}
if containerID == "" {
return "", "", errNeedContainerID
}
return fs.resourceURI(false, podID, containerID, resource)
}
func (fs *filesystem) podURI(podID string, resource podResource) (string, string, error) {
return fs.resourceURI(true, podID, "", resource)
}
// commonResourceChecks performs basic checks common to both setting and
// getting a podResource.
func (fs *filesystem) commonResourceChecks(podSpecific bool, podID, containerID string, resource podResource) error {
if podID == "" {
return errNeedPodID
}
if resourceNeedsContainerID(podSpecific, resource) == true && containerID == "" {
return errNeedContainerID
}
switch resource {
case configFileType:
case stateFileType:
case networkFileType:
case hypervisorFileType:
case agentFileType:
case processFileType:
case mountsFileType:
case devicesFileType:
default:
return errInvalidResource
}
return nil
}
func (fs *filesystem) storePodAndContainerConfigResource(podSpecific bool, podID, containerID string, resource podResource, file interface{}) error {
if resource != configFileType {
return errInvalidResource
}
configFile, _, err := fs.resourceURI(podSpecific, podID, containerID, configFileType)
if err != nil {
return err
}
return fs.storeFile(configFile, file)
}
func (fs *filesystem) storeStateResource(podSpecific bool, podID, containerID string, resource podResource, file interface{}) error {
if resource != stateFileType {
return errInvalidResource
}
stateFile, _, err := fs.resourceURI(podSpecific, podID, containerID, stateFileType)
if err != nil {
return err
}
return fs.storeFile(stateFile, file)
}
func (fs *filesystem) storeNetworkResource(podSpecific bool, podID, containerID string, resource podResource, file interface{}) error {
if resource != networkFileType {
return errInvalidResource
}
// pod only resource
networkFile, _, err := fs.resourceURI(true, podID, containerID, networkFileType)
if err != nil {
return err
}
return fs.storeFile(networkFile, file)
}
func (fs *filesystem) storeProcessResource(podSpecific bool, podID, containerID string, resource podResource, file interface{}) error {
if resource != processFileType {
return errInvalidResource
}
processFile, _, err := fs.resourceURI(podSpecific, podID, containerID, processFileType)
if err != nil {
return err
}
return fs.storeFile(processFile, file)
}
func (fs *filesystem) storeMountResource(podSpecific bool, podID, containerID string, resource podResource, file interface{}) error {
if resource != mountsFileType {
return errInvalidResource
}
mountsFile, _, err := fs.resourceURI(podSpecific, podID, containerID, mountsFileType)
if err != nil {
return err
}
return fs.storeFile(mountsFile, file)
}
func (fs *filesystem) storeDeviceResource(podSpecific bool, podID, containerID string, resource podResource, file interface{}) error {
if resource != devicesFileType {
return errInvalidResource
}
devicesFile, _, err := fs.resourceURI(podSpecific, podID, containerID, devicesFileType)
if err != nil {
return err
}
return fs.storeDeviceFile(devicesFile, file)
}
func (fs *filesystem) storeResource(podSpecific bool, podID, containerID string, resource podResource, data interface{}) error {
if err := fs.commonResourceChecks(podSpecific, podID, containerID, resource); err != nil {
return err
}
switch file := data.(type) {
case PodConfig, ContainerConfig:
return fs.storePodAndContainerConfigResource(podSpecific, podID, containerID, resource, file)
case State:
return fs.storeStateResource(podSpecific, podID, containerID, resource, file)
case NetworkNamespace:
return fs.storeNetworkResource(podSpecific, podID, containerID, resource, file)
case Process:
return fs.storeProcessResource(podSpecific, podID, containerID, resource, file)
case []Mount:
return fs.storeMountResource(podSpecific, podID, containerID, resource, file)
case []Device:
return fs.storeDeviceResource(podSpecific, podID, containerID, resource, file)
default:
return fmt.Errorf("Invalid resource data type")
}
}
func (fs *filesystem) fetchResource(podSpecific bool, podID, containerID string, resource podResource, data interface{}) error {
if err := fs.commonResourceChecks(podSpecific, podID, containerID, resource); err != nil {
return err
}
path, _, err := fs.resourceURI(podSpecific, podID, containerID, resource)
if err != nil {
return err
}
return fs.fetchFile(path, resource, data)
}
func (fs *filesystem) storePodResource(podID string, resource podResource, data interface{}) error {
return fs.storeResource(true, podID, "", resource, data)
}
func (fs *filesystem) fetchPodConfig(podID string) (PodConfig, error) {
var podConfig PodConfig
if err := fs.fetchResource(true, podID, "", configFileType, &podConfig); err != nil {
return PodConfig{}, err
}
return podConfig, nil
}
func (fs *filesystem) fetchPodState(podID string) (State, error) {
var state State
if err := fs.fetchResource(true, podID, "", stateFileType, &state); err != nil {
return State{}, err
}
return state, nil
}
func (fs *filesystem) fetchPodNetwork(podID string) (NetworkNamespace, error) {
var networkNS NetworkNamespace
if err := fs.fetchResource(true, podID, "", networkFileType, &networkNS); err != nil {
return NetworkNamespace{}, err
}
return networkNS, nil
}
func (fs *filesystem) fetchHypervisorState(podID string, state interface{}) error {
return fs.fetchResource(true, podID, "", hypervisorFileType, state)
}
func (fs *filesystem) fetchAgentState(podID string, state interface{}) error {
return fs.fetchResource(true, podID, "", agentFileType, state)
}
func (fs *filesystem) storePodNetwork(podID string, networkNS NetworkNamespace) error {
return fs.storePodResource(podID, networkFileType, networkNS)
}
func (fs *filesystem) storeHypervisorState(podID string, state interface{}) error {
hypervisorFile, _, err := fs.resourceURI(true, podID, "", hypervisorFileType)
if err != nil {
return err
}
return fs.storeFile(hypervisorFile, state)
}
func (fs *filesystem) storeAgentState(podID string, state interface{}) error {
agentFile, _, err := fs.resourceURI(true, podID, "", agentFileType)
if err != nil {
return err
}
return fs.storeFile(agentFile, state)
}
func (fs *filesystem) deletePodResources(podID string, resources []podResource) error {
if resources == nil {
resources = []podResource{configFileType, stateFileType}
}
for _, resource := range resources {
_, dir, err := fs.podURI(podID, resource)
if err != nil {
return err
}
err = os.RemoveAll(dir)
if err != nil {
return err
}
}
return nil
}
func (fs *filesystem) storeContainerResource(podID, containerID string, resource podResource, data interface{}) error {
if podID == "" {
return errNeedPodID
}
if containerID == "" {
return errNeedContainerID
}
return fs.storeResource(false, podID, containerID, resource, data)
}
func (fs *filesystem) fetchContainerConfig(podID, containerID string) (ContainerConfig, error) {
var config ContainerConfig
if err := fs.fetchResource(false, podID, containerID, configFileType, &config); err != nil {
return ContainerConfig{}, err
}
return config, nil
}
func (fs *filesystem) fetchContainerState(podID, containerID string) (State, error) {
var state State
if err := fs.fetchResource(false, podID, containerID, stateFileType, &state); err != nil {
return State{}, err
}
return state, nil
}
func (fs *filesystem) fetchContainerProcess(podID, containerID string) (Process, error) {
var process Process
if err := fs.fetchResource(false, podID, containerID, processFileType, &process); err != nil {
return Process{}, err
}
return process, nil
}
func (fs *filesystem) storeContainerProcess(podID, containerID string, process Process) error {
return fs.storeContainerResource(podID, containerID, processFileType, process)
}
func (fs *filesystem) fetchContainerMounts(podID, containerID string) ([]Mount, error) {
var mounts []Mount
if err := fs.fetchResource(false, podID, containerID, mountsFileType, &mounts); err != nil {
return []Mount{}, err
}
return mounts, nil
}
func (fs *filesystem) fetchContainerDevices(podID, containerID string) ([]Device, error) {
var devices []Device
if err := fs.fetchResource(false, podID, containerID, devicesFileType, &devices); err != nil {
return []Device{}, err
}
return devices, nil
}
func (fs *filesystem) storeContainerMounts(podID, containerID string, mounts []Mount) error {
return fs.storeContainerResource(podID, containerID, mountsFileType, mounts)
}
func (fs *filesystem) storeContainerDevices(podID, containerID string, devices []Device) error {
return fs.storeContainerResource(podID, containerID, devicesFileType, devices)
}
func (fs *filesystem) deleteContainerResources(podID, containerID string, resources []podResource) error {
if resources == nil {
resources = []podResource{configFileType, stateFileType}
}
for _, resource := range resources {
_, dir, err := fs.podURI(podID, resource)
if err != nil {
return err
}
containerDir := filepath.Join(dir, containerID, "/")
err = os.RemoveAll(containerDir)
if err != nil {
return err
}
}
return nil
}

571
virtcontainers/filesystem_test.go Normal file
View File

@@ -0,0 +1,571 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
)
func TestFilesystemCreateAllResourcesSuccessful(t *testing.T) {
fs := &filesystem{}
contConfigs := []ContainerConfig{
{ID: "1"},
{ID: "10"},
{ID: "100"},
}
podConfig := &PodConfig{
Containers: contConfigs,
}
pod := Pod{
id: testPodID,
storage: fs,
config: podConfig,
}
if err := pod.newContainers(); err != nil {
t.Fatal(err)
}
podConfigPath := filepath.Join(configStoragePath, testPodID)
podRunPath := filepath.Join(runStoragePath, testPodID)
os.RemoveAll(podConfigPath)
os.RemoveAll(podRunPath)
for _, container := range contConfigs {
configPath := filepath.Join(configStoragePath, testPodID, container.ID)
os.RemoveAll(configPath)
runPath := filepath.Join(runStoragePath, testPodID, container.ID)
os.RemoveAll(runPath)
}
err := fs.createAllResources(pod)
if err != nil {
t.Fatal(err)
}
// Check resources
_, err = os.Stat(podConfigPath)
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(podRunPath)
if err != nil {
t.Fatal(err)
}
for _, container := range contConfigs {
configPath := filepath.Join(configStoragePath, testPodID, container.ID)
s, err := os.Stat(configPath)
if err != nil {
t.Fatal(err)
}
// Check we created the dirs with the correct mode
if s.Mode() != dirMode {
t.Fatal(fmt.Errorf("dirmode [%v] != expected [%v]", s.Mode(), dirMode))
}
runPath := filepath.Join(runStoragePath, testPodID, container.ID)
s, err = os.Stat(runPath)
if err != nil {
t.Fatal(err)
}
// Check we created the dirs with the correct mode
if s.Mode() != dirMode {
t.Fatal(fmt.Errorf("dirmode [%v] != expected [%v]", s.Mode(), dirMode))
}
}
}
func TestFilesystemCreateAllResourcesFailingPodIDEmpty(t *testing.T) {
fs := &filesystem{}
pod := Pod{}
err := fs.createAllResources(pod)
if err == nil {
t.Fatal()
}
}
func TestFilesystemCreateAllResourcesFailingContainerIDEmpty(t *testing.T) {
fs := &filesystem{}
containers := []*Container{
{id: ""},
}
pod := Pod{
id: testPodID,
containers: containers,
}
err := fs.createAllResources(pod)
if err == nil {
t.Fatal()
}
}
type TestNoopStructure struct {
Field1 string
Field2 string
}
func TestFilesystemStoreFileSuccessfulNotExisting(t *testing.T) {
fs := &filesystem{}
path := filepath.Join(testDir, "testFilesystem")
os.Remove(path)
data := TestNoopStructure{
Field1: "value1",
Field2: "value2",
}
expected := "{\"Field1\":\"value1\",\"Field2\":\"value2\"}"
err := fs.storeFile(path, data)
if err != nil {
t.Fatal(err)
}
fileData, err := ioutil.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(fileData) != expected {
t.Fatal()
}
}
func TestFilesystemStoreFileSuccessfulExisting(t *testing.T) {
fs := &filesystem{}
path := filepath.Join(testDir, "testFilesystem")
os.Remove(path)
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
f.Close()
data := TestNoopStructure{
Field1: "value1",
Field2: "value2",
}
expected := "{\"Field1\":\"value1\",\"Field2\":\"value2\"}"
err = fs.storeFile(path, data)
if err != nil {
t.Fatal(err)
}
fileData, err := ioutil.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(fileData) != expected {
t.Fatal()
}
}
func TestFilesystemStoreFileFailingMarshalling(t *testing.T) {
fs := &filesystem{}
path := filepath.Join(testDir, "testFilesystem")
os.Remove(path)
data := make(chan bool)
err := fs.storeFile(path, data)
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchFileSuccessful(t *testing.T) {
fs := &filesystem{}
data := TestNoopStructure{}
path := filepath.Join(testDir, "testFilesystem")
os.Remove(path)
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
dataToWrite := "{\"Field1\":\"value1\",\"Field2\":\"value2\"}"
n, err := f.WriteString(dataToWrite)
if err != nil || n != len(dataToWrite) {
f.Close()
t.Fatal(err)
}
f.Close()
err = fs.fetchFile(path, podResource(-1), &data)
if err != nil {
t.Fatal(err)
}
expected := TestNoopStructure{
Field1: "value1",
Field2: "value2",
}
if reflect.DeepEqual(data, expected) == false {
t.Fatal()
}
}
func TestFilesystemFetchFileFailingNoFile(t *testing.T) {
fs := &filesystem{}
data := TestNoopStructure{}
path := filepath.Join(testDir, "testFilesystem")
os.Remove(path)
err := fs.fetchFile(path, podResource(-1), &data)
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchFileFailingUnMarshalling(t *testing.T) {
fs := &filesystem{}
data := TestNoopStructure{}
path := filepath.Join(testDir, "testFilesystem")
os.Remove(path)
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
f.Close()
err = fs.fetchFile(path, podResource(-1), data)
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchContainerConfigSuccessful(t *testing.T) {
fs := &filesystem{}
contID := "100"
rootFs := "rootfs"
contConfigDir := filepath.Join(configStoragePath, testPodID, contID)
os.MkdirAll(contConfigDir, dirMode)
path := filepath.Join(contConfigDir, configFile)
os.Remove(path)
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
configData := fmt.Sprintf("{\"ID\":\"%s\",\"RootFs\":\"%s\"}", contID, rootFs)
n, err := f.WriteString(configData)
if err != nil || n != len(configData) {
f.Close()
t.Fatal(err)
}
f.Close()
data, err := fs.fetchContainerConfig(testPodID, contID)
if err != nil {
t.Fatal(err)
}
expected := ContainerConfig{
ID: contID,
RootFs: rootFs,
}
if reflect.DeepEqual(data, expected) == false {
t.Fatal()
}
}
func TestFilesystemFetchContainerConfigFailingContIDEmpty(t *testing.T) {
fs := &filesystem{}
_, err := fs.fetchContainerConfig(testPodID, "")
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchContainerConfigFailingPodIDEmpty(t *testing.T) {
fs := &filesystem{}
_, err := fs.fetchContainerConfig("", "100")
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchContainerMountsSuccessful(t *testing.T) {
fs := &filesystem{}
contID := "100"
contMountsDir := filepath.Join(runStoragePath, testPodID, contID)
os.MkdirAll(contMountsDir, dirMode)
path := filepath.Join(contMountsDir, mountsFile)
os.Remove(path)
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
source := "/dev/sda1"
dest := "/root"
mntType := "ext4"
options := "rw"
hostPath := "/tmp/root"
mountData := fmt.Sprintf(`
[
{
"Source":"%s",
"Destination":"%s",
"Type":"%s",
"Options": ["%s"],
"HostPath":"%s"
}
]
`, source, dest, mntType, options, hostPath)
n, err := f.WriteString(mountData)
if err != nil || n != len(mountData) {
f.Close()
t.Fatal(err)
}
f.Close()
data, err := fs.fetchContainerMounts(testPodID, contID)
if err != nil {
data, _ := ioutil.ReadFile(path)
t.Logf("Data from file : %s", string(data[:]))
t.Fatal(err)
}
expected := []Mount{
{
Source: source,
Destination: dest,
Type: mntType,
Options: []string{"rw"},
HostPath: hostPath,
},
}
if reflect.DeepEqual(data, expected) == false {
t.Fatalf("Expected : [%v]\n, Got : [%v]\n", expected, data)
}
}
func TestFilesystemFetchContainerMountsInvalidType(t *testing.T) {
fs := &filesystem{}
contID := "100"
contMountsDir := filepath.Join(runStoragePath, testPodID, contID)
os.MkdirAll(contMountsDir, dirMode)
path := filepath.Join(contMountsDir, mountsFile)
os.Remove(path)
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
configData := fmt.Sprintf("{\"ID\":\"%s\",\"RootFs\":\"rootfs\"}", contID)
n, err := f.WriteString(configData)
if err != nil || n != len(configData) {
f.Close()
t.Fatal(err)
}
f.Close()
_, err = fs.fetchContainerMounts(testPodID, contID)
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchContainerMountsFailingContIDEmpty(t *testing.T) {
fs := &filesystem{}
_, err := fs.fetchContainerMounts(testPodID, "")
if err == nil {
t.Fatal()
}
}
func TestFilesystemFetchContainerMountsFailingPodIDEmpty(t *testing.T) {
fs := &filesystem{}
_, err := fs.fetchContainerMounts("", "100")
if err == nil {
t.Fatal()
}
}
func TestFilesystemResourceDirFailingPodIDEmpty(t *testing.T) {
for _, b := range []bool{true, false} {
_, err := resourceDir(b, "", "", configFileType)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemResourceDirFailingInvalidResource(t *testing.T) {
for _, b := range []bool{true, false} {
_, err := resourceDir(b, testPodID, "100", podResource(-1))
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemResourceURIFailingResourceDir(t *testing.T) {
fs := &filesystem{}
for _, b := range []bool{true, false} {
_, _, err := fs.resourceURI(b, testPodID, "100", podResource(-1))
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingPodConfigStateFileType(t *testing.T) {
fs := &filesystem{}
data := PodConfig{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, testPodID, "100", stateFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingContainerConfigStateFileType(t *testing.T) {
fs := &filesystem{}
data := ContainerConfig{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, testPodID, "100", stateFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingPodConfigResourceURI(t *testing.T) {
fs := &filesystem{}
data := PodConfig{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, "", "100", configFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingContainerConfigResourceURI(t *testing.T) {
fs := &filesystem{}
data := ContainerConfig{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, "", "100", configFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingStateConfigFileType(t *testing.T) {
fs := &filesystem{}
data := State{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, testPodID, "100", configFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingStateResourceURI(t *testing.T) {
fs := &filesystem{}
data := State{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, "", "100", stateFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemStoreResourceFailingWrongDataType(t *testing.T) {
fs := &filesystem{}
data := TestNoopStructure{}
for _, b := range []bool{true, false} {
err := fs.storeResource(b, testPodID, "100", configFileType, data)
if err == nil {
t.Fatal()
}
}
}
func TestFilesystemFetchResourceFailingWrongResourceType(t *testing.T) {
fs := &filesystem{}
for _, b := range []bool{true, false} {
if err := fs.fetchResource(b, testPodID, "100", lockFileType, nil); err == nil {
t.Fatal()
}
}
}

240
virtcontainers/hack/virtc/README.md Normal file
View File

@@ -0,0 +1,240 @@
# virtc
`virtc` is a simple command-line tool that serves to demonstrate typical usage of the virtcontainers API.
This is example software; unlike other projects like runc, runv, or rkt, virtcontainers is not a full container runtime.
## Virtc example
Here we explain how to use the pod and container API from `virtc` command line.
### Prepare your environment
#### Get your kernel
_Fedora_
```
$ sudo -E dnf config-manager --add-repo http://download.opensuse.org/repositories/home:clearlinux:preview:clear-containers-2.1/Fedora_25/home:clearlinux:preview:clear-containers-2.1.repo
$ sudo dnf install linux-container
```
_Ubuntu_
```
$ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/clearlinux:/preview:/clear-containers-2.1/xUbuntu_16.10/ /' >> /etc/apt/sources.list.d/cc-oci-runtime.list"
$ sudo apt install linux-container
```
#### Get your image
Retrieve a recent Clear Containers image to make sure it contains a recent version of hyperstart agent.
To download and install the latest image:
```
$ latest_version=$(curl -sL https://download.clearlinux.org/latest)
$ curl -LO "https://download.clearlinux.org/current/clear-${latest_version}-containers.img.xz"
$ unxz clear-${latest_version}-containers.img.xz
$ sudo mkdir -p /usr/share/clear-containers/
$ sudo install --owner root --group root --mode 0644 clear-${latest_version}-containers.img /usr/share/clear-containers/
$ sudo ln -fs /usr/share/clear-containers/clear-${latest_version}-containers.img /usr/share/clear-containers/clear-containers.img
```
#### Get virtc
_Download virtcontainers project_
```
$ go get github.com/containers/virtcontainers
```
_Build and setup your environment_
```
$ cd $GOPATH/src/github.com/containers/virtcontainers
$ go build -o virtc hack/virtc/main.go
$ sudo -E bash ./utils/virtcontainers-setup.sh
```
`virtcontainers-setup.sh` setup your environment performing different tasks. Particularly, it creates a __busybox__ bundle, and it creates CNI configuration files needed to run `virtc` with CNI plugins.
### Get cc-proxy (optional)
If you plan to start `virtc` with the hyperstart agent, you will have to use [cc-proxy](https://github.com/clearcontainers/proxy) as a proxy, meaning you have to perform extra steps to setup your environment.
```
$ go get github.com/clearcontainers/proxy
$ cd $GOPATH/src/github.com/clearcontainers/proxy
$ make
$ sudo make install
```
If you want to see the traces from the proxy when `virtc` will run, you can manually start it with appropriate debug level:
```
$ sudo /usr/libexec/clearcontainers/cc-proxy -v 3
```
This will generate output similar to the following:
```
I0410 08:58:49.058881 5384 proxy.go:521] listening on /var/run/clearcontainers/proxy.sock
I0410 08:58:49.059044 5384 proxy.go:566] proxy started
```
The proxy socket specified in the example log output has to be used as `virtc`'s `--proxy-url` option.
### Get cc-shim (optional)
If you plan to start `virtc` with the hyperstart agent (implying the use of `cc-proxy` as a proxy), you will have to rely on [cc-shim](https://github.com/clearcontainers/shim) in order to interact with the process running inside your container.
First, you will have to perform extra steps to setup your environment.
```
$ go get github.com/clearcontainers/shim
$ cd $GOPATH/src/github.com/clearcontainers/shim && ./autogen.sh
$ make
$ sudo make install
```
The shim will be installed at the following location: `/usr/libexec/clear-containers/cc-shim`. There will be three cases where you will be able to interact with your container's process through `cc-shim`:
_Start a new container_
```
# ./virtc container start --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
_Execute a new process on a running container_
```
# ./virtc container enter --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
_Start a pod with container(s) previously created_
```
# ./virtc pod start --id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
Notice that in both cases, the `--pod-id` and `--id` options have been defined when previously creating a pod and a container.
### Run virtc
All following commands __MUST__ be run as root. By default, and unless you decide to modify it and rebuild it, `virtc` starts empty pods (no container started).
#### Run a new pod (Create + Start)
```
# ./virtc pod run --agent="hyperstart" --network="CNI" --proxy="ccProxy" --proxy-url="unix:///var/run/clearcontainers/proxy.sock" --shim="ccShim" --shim-path="/usr/libexec/cc-shim"
```
#### Create a new pod
```
# ./virtc pod run --agent="hyperstart" --network="CNI" --proxy="ccProxy" --proxy-url="unix:///var/run/clearcontainers/proxy.sock" --shim="ccShim" --shim-path="/usr/libexec/cc-shim"
```
This will generate output similar to the following:
```
Pod 306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 created
```
#### Start an existing pod
```
# ./virtc pod start --id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following:
```
Pod 306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 started
```
#### Stop an existing pod
```
# ./virtc pod stop --id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following:
```
Pod 306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 stopped
```
#### Get the status of an existing pod and its containers
```
# ./virtc pod status --id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following (assuming the pod has been started):
```
POD ID STATE HYPERVISOR AGENT
306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 running qemu hyperstart
CONTAINER ID STATE
```
#### Delete an existing pod
```
# ./virtc pod delete --id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following:
```
Pod 306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 deleted
```
#### List all existing pods
```
# ./virtc pod list
```
This should generate that kind of output
```
POD ID STATE HYPERVISOR AGENT
306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 running qemu hyperstart
92d73f74-4514-4a0d-81df-db1cc4c59100 running qemu hyperstart
7088148c-049b-4be7-b1be-89b3ae3c551c ready qemu hyperstart
6d57654e-4804-4a91-b72d-b5fe375ed3e1 ready qemu hyperstart
```
#### Create a new container
```
# ./virtc container create --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 --rootfs="/tmp/bundles/busybox/rootfs" --cmd="/bin/ifconfig" --console="/dev/pts/30"
```
This will generate output similar to the following:
```
Container 1 created
```
__Note:__ The option `--console` can be any existing console.
Don't try to provide `$(tty)` as it is your current console, and you would not be
able to get your console back as the shim would be listening to this indefinitely.
Instead, you would prefer to open a new shell and get the `$(tty)` from this shell.
That way, you make sure you have a dedicated input/output terminal.
#### Start an existing container
```
# ./virtc container start --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following:
```
Container 1 started
```
#### Run a new process on an existing container
```
# ./virtc container enter --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8 --cmd="/bin/ps" --console="/dev/pts/30"
```
This will generate output similar to the following:
```
Container 1 entered
```
__Note:__ The option `--console` can be any existing console.
Don't try to provide `$(tty)` as it is your current console, and you would not be
able to get your console back as the shim would be listening to this indefinitely.
Instead, you would prefer to open a new shell and get the `$(tty)` from this shell.
That way, you make sure you have a dedicated input/output terminal.
#### Stop an existing container
```
# ./virtc container stop --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following:
```
Container 1 stopped
```
#### Delete an existing container
```
# ./virtc container delete --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following:
```
Container 1 deleted
```
#### Get the status of an existing container
```
# ./virtc container status --id=1 --pod-id=306ecdcf-0a6f-4a06-a03e-86a7b868ffc8
```
This will generate output similar to the following (assuming the container has been started):
```
CONTAINER ID STATE
1 running
```

976
virtcontainers/hack/virtc/main.go Normal file
View File

@@ -0,0 +1,976 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"errors"
"fmt"
"os"
"os/exec"
"strings"
"text/tabwriter"
"github.com/containers/virtcontainers/pkg/uuid"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
vc "github.com/containers/virtcontainers"
)
var virtcLog = logrus.New()
var listFormat = "%s\t%s\t%s\t%s\n"
var statusFormat = "%s\t%s\n"
var (
errNeedContainerID = errors.New("Container ID cannot be empty")
errNeedPodID = errors.New("Pod ID cannot be empty")
)
var podConfigFlags = []cli.Flag{
cli.GenericFlag{
Name: "agent",
Value: new(vc.AgentType),
Usage: "the guest agent",
},
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier (default: auto-generated)",
},
cli.StringFlag{
Name: "machine-type",
Value: vc.QemuPC,
Usage: "hypervisor machine type",
},
cli.GenericFlag{
Name: "network",
Value: new(vc.NetworkModel),
Usage: "the network model",
},
cli.GenericFlag{
Name: "proxy",
Value: new(vc.ProxyType),
Usage: "the agent's proxy",
},
cli.StringFlag{
Name: "proxy-path",
Value: "",
Usage: "path to proxy binary",
},
cli.GenericFlag{
Name: "shim",
Value: new(vc.ShimType),
Usage: "the shim type",
},
cli.StringFlag{
Name: "shim-path",
Value: "",
Usage: "the shim binary path",
},
cli.StringFlag{
Name: "hyper-ctl-sock-name",
Value: "",
Usage: "the hyperstart control socket name",
},
cli.StringFlag{
Name: "hyper-tty-sock-name",
Value: "",
Usage: "the hyperstart tty socket name",
},
cli.UintFlag{
Name: "cpus",
Value: 0,
Usage: "the number of virtual cpus available for this pod",
},
cli.UintFlag{
Name: "memory",
Value: 0,
Usage: "the amount of memory available for this pod in MiB",
},
}
var ccKernelParams = []vc.Param{
{
Key: "init",
Value: "/usr/lib/systemd/systemd",
},
{
Key: "systemd.unit",
Value: "clear-containers.target",
},
{
Key: "systemd.mask",
Value: "systemd-networkd.service",
},
{
Key: "systemd.mask",
Value: "systemd-networkd.socket",
},
}
func buildKernelParams(config *vc.HypervisorConfig) error {
for _, p := range ccKernelParams {
if err := config.AddKernelParam(p); err != nil {
return err
}
}
return nil
}
func buildPodConfig(context *cli.Context) (vc.PodConfig, error) {
var agConfig interface{}
hyperCtlSockName := context.String("hyper-ctl-sock-name")
hyperTtySockName := context.String("hyper-tty-sock-name")
proxyPath := context.String("proxy-path")
shimPath := context.String("shim-path")
machineType := context.String("machine-type")
vmMemory := context.Uint("vm-memory")
agentType, ok := context.Generic("agent").(*vc.AgentType)
if ok != true {
return vc.PodConfig{}, fmt.Errorf("Could not convert agent type")
}
networkModel, ok := context.Generic("network").(*vc.NetworkModel)
if ok != true {
return vc.PodConfig{}, fmt.Errorf("Could not convert network model")
}
proxyType, ok := context.Generic("proxy").(*vc.ProxyType)
if ok != true {
return vc.PodConfig{}, fmt.Errorf("Could not convert proxy type")
}
shimType, ok := context.Generic("shim").(*vc.ShimType)
if ok != true {
return vc.PodConfig{}, fmt.Errorf("Could not convert shim type")
}
kernelPath := "/usr/share/clear-containers/vmlinuz.container"
if machineType == vc.QemuPCLite {
kernelPath = "/usr/share/clear-containers/vmlinux.container"
}
hypervisorConfig := vc.HypervisorConfig{
KernelPath: kernelPath,
ImagePath: "/usr/share/clear-containers/clear-containers.img",
HypervisorMachineType: machineType,
}
if err := buildKernelParams(&hypervisorConfig); err != nil {
return vc.PodConfig{}, err
}
netConfig := vc.NetworkConfig{
NumInterfaces: 1,
}
switch *agentType {
case vc.HyperstartAgent:
agConfig = vc.HyperConfig{
SockCtlName: hyperCtlSockName,
SockTtyName: hyperTtySockName,
}
default:
agConfig = nil
}
proxyConfig := getProxyConfig(*proxyType, proxyPath)
shimConfig := getShimConfig(*shimType, shimPath)
vmConfig := vc.Resources{
Memory: vmMemory,
}
id := context.String("id")
if id == "" {
// auto-generate pod name
id = uuid.Generate().String()
}
podConfig := vc.PodConfig{
ID: id,
VMConfig: vmConfig,
HypervisorType: vc.QemuHypervisor,
HypervisorConfig: hypervisorConfig,
AgentType: *agentType,
AgentConfig: agConfig,
NetworkModel: *networkModel,
NetworkConfig: netConfig,
ProxyType: *proxyType,
ProxyConfig: proxyConfig,
ShimType: *shimType,
ShimConfig: shimConfig,
Containers: []vc.ContainerConfig{},
}
return podConfig, nil
}
func getProxyConfig(proxyType vc.ProxyType, path string) vc.ProxyConfig {
var proxyConfig vc.ProxyConfig
switch proxyType {
case vc.KataProxyType:
fallthrough
case vc.CCProxyType:
proxyConfig = vc.ProxyConfig{
Path: path,
}
}
return proxyConfig
}
func getShimConfig(shimType vc.ShimType, path string) interface{} {
var shimConfig interface{}
switch shimType {
case vc.CCShimType, vc.KataShimType:
shimConfig = vc.ShimConfig{
Path: path,
}
default:
shimConfig = nil
}
return shimConfig
}
// checkRequiredPodArgs checks to ensure the required command-line
// arguments have been specified for the pod sub-command specified by
// the context argument.
func checkRequiredPodArgs(context *cli.Context) error {
if context == nil {
return fmt.Errorf("BUG: need Context")
}
// sub-sub-command name
name := context.Command.Name
switch name {
case "create":
fallthrough
case "list":
fallthrough
case "run":
// these commands don't require any arguments
return nil
}
id := context.String("id")
if id == "" {
return errNeedPodID
}
return nil
}
// checkRequiredContainerArgs checks to ensure the required command-line
// arguments have been specified for the container sub-command specified
// by the context argument.
func checkRequiredContainerArgs(context *cli.Context) error {
if context == nil {
return fmt.Errorf("BUG: need Context")
}
// sub-sub-command name
name := context.Command.Name
podID := context.String("pod-id")
if podID == "" {
return errNeedPodID
}
rootfs := context.String("rootfs")
if name == "create" && rootfs == "" {
return fmt.Errorf("%s: need rootfs", name)
}
id := context.String("id")
if id == "" {
return errNeedContainerID
}
return nil
}
func runPod(context *cli.Context) error {
podConfig, err := buildPodConfig(context)
if err != nil {
return fmt.Errorf("Could not build pod config: %s", err)
}
_, err = vc.RunPod(podConfig)
if err != nil {
return fmt.Errorf("Could not run pod: %s", err)
}
return nil
}
func createPod(context *cli.Context) error {
podConfig, err := buildPodConfig(context)
if err != nil {
return fmt.Errorf("Could not build pod config: %s", err)
}
p, err := vc.CreatePod(podConfig)
if err != nil {
return fmt.Errorf("Could not create pod: %s", err)
}
fmt.Printf("Pod %s created\n", p.ID())
return nil
}
func checkPodArgs(context *cli.Context, f func(context *cli.Context) error) error {
if err := checkRequiredPodArgs(context); err != nil {
return err
}
return f(context)
}
func checkContainerArgs(context *cli.Context, f func(context *cli.Context) error) error {
if err := checkRequiredContainerArgs(context); err != nil {
return err
}
return f(context)
}
func deletePod(context *cli.Context) error {
p, err := vc.DeletePod(context.String("id"))
if err != nil {
return fmt.Errorf("Could not delete pod: %s", err)
}
fmt.Printf("Pod %s deleted\n", p.ID())
return nil
}
func startPod(context *cli.Context) error {
p, err := vc.StartPod(context.String("id"))
if err != nil {
return fmt.Errorf("Could not start pod: %s", err)
}
fmt.Printf("Pod %s started\n", p.ID())
return nil
}
func stopPod(context *cli.Context) error {
p, err := vc.StopPod(context.String("id"))
if err != nil {
return fmt.Errorf("Could not stop pod: %s", err)
}
fmt.Printf("Pod %s stopped\n", p.ID())
return nil
}
func pausePod(context *cli.Context) error {
p, err := vc.PausePod(context.String("id"))
if err != nil {
return fmt.Errorf("Could not pause pod: %s", err)
}
fmt.Printf("Pod %s paused\n", p.ID())
return nil
}
func resumePod(context *cli.Context) error {
p, err := vc.ResumePod(context.String("id"))
if err != nil {
return fmt.Errorf("Could not resume pod: %s", err)
}
fmt.Printf("Pod %s resumed\n", p.ID())
return nil
}
func listPods(context *cli.Context) error {
podStatusList, err := vc.ListPod()
if err != nil {
return fmt.Errorf("Could not list pod: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 2, 8, 1, '\t', 0)
fmt.Fprintf(w, listFormat, "POD ID", "STATE", "HYPERVISOR", "AGENT")
for _, podStatus := range podStatusList {
fmt.Fprintf(w, listFormat,
podStatus.ID, podStatus.State.State, podStatus.Hypervisor, podStatus.Agent)
}
w.Flush()
return nil
}
func statusPod(context *cli.Context) error {
podStatus, err := vc.StatusPod(context.String("id"))
if err != nil {
return fmt.Errorf("Could not get pod status: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 2, 8, 1, '\t', 0)
fmt.Fprintf(w, listFormat, "POD ID", "STATE", "HYPERVISOR", "AGENT")
fmt.Fprintf(w, listFormat+"\n",
podStatus.ID, podStatus.State.State, podStatus.Hypervisor, podStatus.Agent)
fmt.Fprintf(w, statusFormat, "CONTAINER ID", "STATE")
for _, contStatus := range podStatus.ContainersStatus {
fmt.Fprintf(w, statusFormat, contStatus.ID, contStatus.State.State)
}
w.Flush()
return nil
}
var runPodCommand = cli.Command{
Name: "run",
Usage: "run a pod",
Flags: podConfigFlags,
Action: func(context *cli.Context) error {
return checkPodArgs(context, runPod)
},
}
var createPodCommand = cli.Command{
Name: "create",
Usage: "create a pod",
Flags: podConfigFlags,
Action: func(context *cli.Context) error {
return checkPodArgs(context, createPod)
},
}
var deletePodCommand = cli.Command{
Name: "delete",
Usage: "delete an existing pod",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkPodArgs(context, deletePod)
},
}
var startPodCommand = cli.Command{
Name: "start",
Usage: "start an existing pod",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkPodArgs(context, startPod)
},
}
var stopPodCommand = cli.Command{
Name: "stop",
Usage: "stop an existing pod",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkPodArgs(context, stopPod)
},
}
var listPodsCommand = cli.Command{
Name: "list",
Usage: "list all existing pods",
Action: func(context *cli.Context) error {
return checkPodArgs(context, listPods)
},
}
var statusPodCommand = cli.Command{
Name: "status",
Usage: "returns a detailed pod status",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkPodArgs(context, statusPod)
},
}
var pausePodCommand = cli.Command{
Name: "pause",
Usage: "pause an existing pod",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkPodArgs(context, pausePod)
},
}
var resumePodCommand = cli.Command{
Name: "resume",
Usage: "unpause a paused pod",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkPodArgs(context, resumePod)
},
}
func createContainer(context *cli.Context) error {
console := context.String("console")
interactive := false
if console != "" {
interactive = true
}
envs := []vc.EnvVar{
{
Var: "PATH",
Value: "/bin:/usr/bin:/sbin:/usr/sbin",
},
}
cmd := vc.Cmd{
Args: strings.Split(context.String("cmd"), " "),
Envs: envs,
WorkDir: "/",
Interactive: interactive,
Console: console,
}
id := context.String("id")
if id == "" {
// auto-generate container name
id = uuid.Generate().String()
}
containerConfig := vc.ContainerConfig{
ID: id,
RootFs: context.String("rootfs"),
Cmd: cmd,
}
_, c, err := vc.CreateContainer(context.String("pod-id"), containerConfig)
if err != nil {
return fmt.Errorf("Could not create container: %s", err)
}
fmt.Printf("Container %s created\n", c.ID())
return nil
}
func deleteContainer(context *cli.Context) error {
c, err := vc.DeleteContainer(context.String("pod-id"), context.String("id"))
if err != nil {
return fmt.Errorf("Could not delete container: %s", err)
}
fmt.Printf("Container %s deleted\n", c.ID())
return nil
}
func startContainer(context *cli.Context) error {
c, err := vc.StartContainer(context.String("pod-id"), context.String("id"))
if err != nil {
return fmt.Errorf("Could not start container: %s", err)
}
fmt.Printf("Container %s started\n", c.ID())
return nil
}
func stopContainer(context *cli.Context) error {
c, err := vc.StopContainer(context.String("pod-id"), context.String("id"))
if err != nil {
return fmt.Errorf("Could not stop container: %s", err)
}
fmt.Printf("Container %s stopped\n", c.ID())
return nil
}
func enterContainer(context *cli.Context) error {
console := context.String("console")
interactive := false
if console != "" {
interactive = true
}
envs := []vc.EnvVar{
{
Var: "PATH",
Value: "/bin:/usr/bin:/sbin:/usr/sbin",
},
}
cmd := vc.Cmd{
Args: strings.Split(context.String("cmd"), " "),
Envs: envs,
WorkDir: "/",
Interactive: interactive,
Console: console,
}
_, c, _, err := vc.EnterContainer(context.String("pod-id"), context.String("id"), cmd)
if err != nil {
return fmt.Errorf("Could not enter container: %s", err)
}
fmt.Printf("Container %s entered\n", c.ID())
return nil
}
func statusContainer(context *cli.Context) error {
contStatus, err := vc.StatusContainer(context.String("pod-id"), context.String("id"))
if err != nil {
return fmt.Errorf("Could not get container status: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 2, 8, 1, '\t', 0)
fmt.Fprintf(w, statusFormat, "CONTAINER ID", "STATE")
fmt.Fprintf(w, statusFormat, contStatus.ID, contStatus.State.State)
w.Flush()
return nil
}
var createContainerCommand = cli.Command{
Name: "create",
Usage: "create a container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the container identifier (default: auto-generated)",
},
cli.StringFlag{
Name: "pod-id",
Value: "",
Usage: "the pod identifier",
},
cli.StringFlag{
Name: "rootfs",
Value: "",
Usage: "the container rootfs directory",
},
cli.StringFlag{
Name: "cmd",
Value: "",
Usage: "the command executed inside the container",
},
cli.StringFlag{
Name: "console",
Value: "",
Usage: "the container console",
},
},
Action: func(context *cli.Context) error {
return checkContainerArgs(context, createContainer)
},
}
var deleteContainerCommand = cli.Command{
Name: "delete",
Usage: "delete an existing container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the container identifier",
},
cli.StringFlag{
Name: "pod-id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkContainerArgs(context, deleteContainer)
},
}
var startContainerCommand = cli.Command{
Name: "start",
Usage: "start an existing container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the container identifier",
},
cli.StringFlag{
Name: "pod-id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkContainerArgs(context, startContainer)
},
}
var stopContainerCommand = cli.Command{
Name: "stop",
Usage: "stop an existing container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the container identifier",
},
cli.StringFlag{
Name: "pod-id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkContainerArgs(context, stopContainer)
},
}
var enterContainerCommand = cli.Command{
Name: "enter",
Usage: "enter an existing container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the container identifier",
},
cli.StringFlag{
Name: "pod-id",
Value: "",
Usage: "the pod identifier",
},
cli.StringFlag{
Name: "cmd",
Value: "echo",
Usage: "the command executed inside the container",
},
cli.StringFlag{
Name: "console",
Value: "",
Usage: "the process console",
},
},
Action: func(context *cli.Context) error {
return checkContainerArgs(context, enterContainer)
},
}
var statusContainerCommand = cli.Command{
Name: "status",
Usage: "returns detailed container status",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "the container identifier",
},
cli.StringFlag{
Name: "pod-id",
Value: "",
Usage: "the pod identifier",
},
},
Action: func(context *cli.Context) error {
return checkContainerArgs(context, statusContainer)
},
}
func startCCShim(process *vc.Process, shimPath, url string) error {
if process.Token == "" {
return fmt.Errorf("Token cannot be empty")
}
if url == "" {
return fmt.Errorf("URL cannot be empty")
}
if shimPath == "" {
return fmt.Errorf("Shim path cannot be empty")
}
cmd := exec.Command(shimPath, "-t", process.Token, "-u", url)
cmd.Env = os.Environ()
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
func main() {
cli.VersionFlag = cli.BoolFlag{
Name: "version",
Usage: "print the version",
}
virtc := cli.NewApp()
virtc.Name = "VirtContainers CLI"
virtc.Version = "0.0.1"
virtc.Flags = []cli.Flag{
cli.BoolFlag{
Name: "debug",
Usage: "enable debug output for logging",
},
cli.StringFlag{
Name: "log",
Value: "",
Usage: "set the log file path where internal debug information is written",
},
cli.StringFlag{
Name: "log-format",
Value: "text",
Usage: "set the format used by logs ('text' (default), or 'json')",
},
}
virtc.Commands = []cli.Command{
{
Name: "pod",
Usage: "pod commands",
Subcommands: []cli.Command{
createPodCommand,
deletePodCommand,
listPodsCommand,
pausePodCommand,
resumePodCommand,
runPodCommand,
startPodCommand,
stopPodCommand,
statusPodCommand,
},
},
{
Name: "container",
Usage: "container commands",
Subcommands: []cli.Command{
createContainerCommand,
deleteContainerCommand,
startContainerCommand,
stopContainerCommand,
enterContainerCommand,
statusContainerCommand,
},
},
}
virtc.Before = func(context *cli.Context) error {
if context.GlobalBool("debug") {
virtcLog.Level = logrus.DebugLevel
}
if path := context.GlobalString("log"); path != "" {
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
if err != nil {
return err
}
virtcLog.Out = f
}
switch context.GlobalString("log-format") {
case "text":
// retain logrus's default.
case "json":
virtcLog.Formatter = new(logrus.JSONFormatter)
default:
return fmt.Errorf("unknown log-format %q", context.GlobalString("log-format"))
}
// Set virtcontainers logger.
vc.SetLogger(virtcLog)
return nil
}
err := virtc.Run(os.Args)
if err != nil {
virtcLog.Fatal(err)
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

160
virtcontainers/hook.go Normal file
View File

@@ -0,0 +1,160 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"time"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
// Hook represents an OCI hook, including its required parameters.
type Hook struct {
Path string
Args []string
Env []string
Timeout int
}
// Hooks gathers all existing OCI hooks list.
type Hooks struct {
PreStartHooks []Hook
PostStartHooks []Hook
PostStopHooks []Hook
}
// Logger returns a logrus logger appropriate for logging Hooks messages
func (h *Hooks) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "hooks")
}
func buildHookState(processID int) specs.State {
return specs.State{
Pid: processID,
}
}
func (h *Hook) runHook() error {
state := buildHookState(os.Getpid())
stateJSON, err := json.Marshal(state)
if err != nil {
return err
}
var stdout, stderr bytes.Buffer
cmd := &exec.Cmd{
Path: h.Path,
Args: h.Args,
Env: h.Env,
Stdin: bytes.NewReader(stateJSON),
Stdout: &stdout,
Stderr: &stderr,
}
err = cmd.Start()
if err != nil {
return err
}
if h.Timeout == 0 {
err = cmd.Wait()
if err != nil {
return fmt.Errorf("%s: stdout: %s, stderr: %s", err, stdout.String(), stderr.String())
}
} else {
done := make(chan error)
go func() { done <- cmd.Wait() }()
select {
case err := <-done:
if err != nil {
return fmt.Errorf("%s: stdout: %s, stderr: %s", err, stdout.String(), stderr.String())
}
case <-time.After(time.Duration(h.Timeout) * time.Second):
return fmt.Errorf("Hook timeout")
}
}
return nil
}
func (h *Hooks) preStartHooks() error {
if len(h.PreStartHooks) == 0 {
return nil
}
for _, hook := range h.PreStartHooks {
err := hook.runHook()
if err != nil {
h.Logger().WithFields(logrus.Fields{
"hook-type": "pre-start",
"error": err,
}).Error("hook error")
return err
}
}
return nil
}
func (h *Hooks) postStartHooks() error {
if len(h.PostStartHooks) == 0 {
return nil
}
for _, hook := range h.PostStartHooks {
err := hook.runHook()
if err != nil {
// In case of post start hook, the error is not fatal,
// just need to be logged.
h.Logger().WithFields(logrus.Fields{
"hook-type": "post-start",
"error": err,
}).Info("hook error")
}
}
return nil
}
func (h *Hooks) postStopHooks() error {
if len(h.PostStopHooks) == 0 {
return nil
}
for _, hook := range h.PostStopHooks {
err := hook.runHook()
if err != nil {
// In case of post stop hook, the error is not fatal,
// just need to be logged.
h.Logger().WithFields(logrus.Fields{
"hook-type": "post-stop",
"error": err,
}).Info("hook error")
}
}
return nil
}

101
virtcontainers/hook/mock/hook.go Normal file
View File

@@ -0,0 +1,101 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strconv"
"time"
"github.com/opencontainers/runc/libcontainer/configs"
)
var logFile = "/tmp/mock_hook.log"
var testKey = "test-key"
var testContainerID = "test-container-id"
var testControllerID = "test-controller-id"
func main() {
err := os.RemoveAll(logFile)
if err != nil {
os.Exit(1)
}
f, err := os.Create(logFile)
if err != nil {
os.Exit(1)
}
defer f.Close()
fmt.Fprintf(f, "args = %s\n", os.Args)
if len(os.Args) < 3 {
fmt.Fprintf(f, "At least 3 args expected, only %d received\n", len(os.Args))
os.Exit(1)
}
if os.Args[0] != testKey {
fmt.Fprintf(f, "args[0] should be \"%s\", received \"%s\" instead\n", testKey, os.Args[0])
os.Exit(1)
}
if os.Args[1] != testContainerID {
fmt.Fprintf(f, "argv[1] should be \"%s\", received \"%s\" instead\n", testContainerID, os.Args[1])
os.Exit(1)
}
if os.Args[2] != testControllerID {
fmt.Fprintf(f, "argv[2] should be \"%s\", received \"%s\" instead\n", testControllerID, os.Args[2])
os.Exit(1)
}
stateBuf, err := ioutil.ReadAll(os.Stdin)
if err != nil {
fmt.Fprintf(f, "Could not read on stdin: %s\n", err)
os.Exit(1)
}
var state configs.HookState
err = json.Unmarshal(stateBuf, &state)
if err != nil {
fmt.Fprintf(f, "Could not unmarshal HookState json: %s\n", err)
os.Exit(1)
}
if state.Pid < 1 {
fmt.Fprintf(f, "Invalid PID: %d\n", state.Pid)
os.Exit(1)
}
// Intended to sleep, so as to make the test passing/failing.
if len(os.Args) >= 4 {
timeout, err := strconv.Atoi(os.Args[3])
if err != nil {
fmt.Fprintf(f, "Could not retrieve timeout %s from args[3]\n", os.Args[3])
os.Exit(1)
}
time.Sleep(time.Duration(timeout) * time.Second)
if len(os.Args) == 5 {
msg := "panicking at user request"
fmt.Fprintln(f, msg)
panic(msg)
}
}
}

239
virtcontainers/hook_test.go Normal file
View File

@@ -0,0 +1,239 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
. "github.com/containers/virtcontainers/pkg/mock"
specs "github.com/opencontainers/runtime-spec/specs-go"
)
// Important to keep these values in sync with hook test binary
var testKeyHook = "test-key"
var testContainerIDHook = "test-container-id"
var testControllerIDHook = "test-controller-id"
var testProcessIDHook = 12345
var testBinHookPath = "/usr/bin/virtcontainers/bin/test/hook"
func getMockHookBinPath() string {
if DefaultMockHookBinPath == "" {
return testBinHookPath
}
return DefaultMockHookBinPath
}
func TestBuildHookState(t *testing.T) {
expected := specs.State{
Pid: testProcessIDHook,
}
hookState := buildHookState(testProcessIDHook)
if reflect.DeepEqual(hookState, expected) == false {
t.Fatal()
}
}
func createHook(timeout int) *Hook {
return &Hook{
Path: getMockHookBinPath(),
Args: []string{testKeyHook, testContainerIDHook, testControllerIDHook},
Env: os.Environ(),
Timeout: timeout,
}
}
func createWrongHook() *Hook {
return &Hook{
Path: getMockHookBinPath(),
Args: []string{"wrong-args"},
Env: os.Environ(),
}
}
func testRunHookFull(t *testing.T, timeout int, expectFail bool) {
hook := createHook(timeout)
err := hook.runHook()
if expectFail {
if err == nil {
t.Fatal("unexpected success")
}
} else {
if err != nil {
t.Fatalf("unexpected failure: %v", err)
}
}
}
func testRunHook(t *testing.T, timeout int) {
testRunHookFull(t, timeout, false)
}
func TestRunHook(t *testing.T) {
cleanUp()
testRunHook(t, 0)
}
func TestRunHookTimeout(t *testing.T) {
testRunHook(t, 1)
}
func TestRunHookExitFailure(t *testing.T) {
hook := createWrongHook()
err := hook.runHook()
if err == nil {
t.Fatal()
}
}
func TestRunHookTimeoutFailure(t *testing.T) {
hook := createHook(1)
hook.Args = append(hook.Args, "2")
err := hook.runHook()
if err == nil {
t.Fatal()
}
}
func TestRunHookWaitFailure(t *testing.T) {
hook := createHook(60)
hook.Args = append(hook.Args, "1", "panic")
err := hook.runHook()
if err == nil {
t.Fatal()
}
}
func testRunHookInvalidCommand(t *testing.T, timeout int) {
cleanUp()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
cmd := filepath.Join(dir, "does-not-exist")
savedDefaultMockHookBinPath := DefaultMockHookBinPath
DefaultMockHookBinPath = cmd
defer func() {
DefaultMockHookBinPath = savedDefaultMockHookBinPath
}()
testRunHookFull(t, timeout, true)
}
func TestRunHookInvalidCommand(t *testing.T) {
testRunHookInvalidCommand(t, 0)
}
func TestRunHookTimeoutInvalidCommand(t *testing.T) {
testRunHookInvalidCommand(t, 1)
}
func testHooks(t *testing.T, hook *Hook) {
hooks := &Hooks{
PreStartHooks: []Hook{*hook},
PostStartHooks: []Hook{*hook},
PostStopHooks: []Hook{*hook},
}
err := hooks.preStartHooks()
if err != nil {
t.Fatal(err)
}
err = hooks.postStartHooks()
if err != nil {
t.Fatal(err)
}
err = hooks.postStopHooks()
if err != nil {
t.Fatal(err)
}
}
func testFailingHooks(t *testing.T, hook *Hook) {
hooks := &Hooks{
PreStartHooks: []Hook{*hook},
PostStartHooks: []Hook{*hook},
PostStopHooks: []Hook{*hook},
}
err := hooks.preStartHooks()
if err == nil {
t.Fatal(err)
}
err = hooks.postStartHooks()
if err != nil {
t.Fatal(err)
}
err = hooks.postStopHooks()
if err != nil {
t.Fatal(err)
}
}
func TestHooks(t *testing.T) {
testHooks(t, createHook(0))
}
func TestHooksTimeout(t *testing.T) {
testHooks(t, createHook(1))
}
func TestFailingHooks(t *testing.T) {
testFailingHooks(t, createWrongHook())
}
func TestEmptyHooks(t *testing.T) {
hooks := &Hooks{}
err := hooks.preStartHooks()
if err != nil {
t.Fatal(err)
}
err = hooks.postStartHooks()
if err != nil {
t.Fatal(err)
}
err = hooks.postStopHooks()
if err != nil {
t.Fatal(err)
}
}

794
virtcontainers/hyperstart_agent.go Normal file
View File

@@ -0,0 +1,794 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"net"
"net/url"
"os"
"path/filepath"
"syscall"
"time"
proxyClient "github.com/clearcontainers/proxy/client"
"github.com/containers/virtcontainers/pkg/hyperstart"
ns "github.com/containers/virtcontainers/pkg/nsenter"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
)
var defaultSockPathTemplates = []string{"%s/%s/hyper.sock", "%s/%s/tty.sock"}
var defaultChannelTemplate = "sh.hyper.channel.%d"
var defaultDeviceIDTemplate = "channel%d"
var defaultIDTemplate = "charch%d"
var defaultSharedDir = "/run/hyper/shared/pods/"
var mountTag = "hyperShared"
var maxHostnameLen = 64
const (
unixSocket = "unix"
)
// HyperConfig is a structure storing information needed for
// hyperstart agent initialization.
type HyperConfig struct {
SockCtlName string
SockTtyName string
}
func (h *hyper) generateSockets(pod Pod, c HyperConfig) {
podSocketPaths := []string{
fmt.Sprintf(defaultSockPathTemplates[0], runStoragePath, pod.id),
fmt.Sprintf(defaultSockPathTemplates[1], runStoragePath, pod.id),
}
if c.SockCtlName != "" {
podSocketPaths[0] = c.SockCtlName
}
if c.SockTtyName != "" {
podSocketPaths[1] = c.SockTtyName
}
for i := 0; i < len(podSocketPaths); i++ {
s := Socket{
DeviceID: fmt.Sprintf(defaultDeviceIDTemplate, i),
ID: fmt.Sprintf(defaultIDTemplate, i),
HostPath: podSocketPaths[i],
Name: fmt.Sprintf(defaultChannelTemplate, i),
}
h.sockets = append(h.sockets, s)
}
}
// HyperAgentState is the structure describing the data stored from this
// agent implementation.
type HyperAgentState struct {
ProxyPid int
URL string
}
// hyper is the Agent interface implementation for hyperstart.
type hyper struct {
pod Pod
shim shim
proxy proxy
client *proxyClient.Client
state HyperAgentState
sockets []Socket
}
type hyperstartProxyCmd struct {
cmd string
message interface{}
token string
}
// Logger returns a logrus logger appropriate for logging hyper messages
func (h *hyper) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "hyper")
}
func (h *hyper) buildHyperContainerProcess(cmd Cmd) (*hyperstart.Process, error) {
var envVars []hyperstart.EnvironmentVar
for _, e := range cmd.Envs {
envVar := hyperstart.EnvironmentVar{
Env: e.Var,
Value: e.Value,
}
envVars = append(envVars, envVar)
}
process := &hyperstart.Process{
Terminal: cmd.Interactive,
Args: cmd.Args,
Envs: envVars,
Workdir: cmd.WorkDir,
User: cmd.User,
Group: cmd.PrimaryGroup,
AdditionalGroups: cmd.SupplementaryGroups,
NoNewPrivileges: cmd.NoNewPrivileges,
}
process.Capabilities = hyperstart.Capabilities{
Bounding: cmd.Capabilities.Bounding,
Effective: cmd.Capabilities.Effective,
Inheritable: cmd.Capabilities.Inheritable,
Permitted: cmd.Capabilities.Permitted,
Ambient: cmd.Capabilities.Ambient,
}
return process, nil
}
func (h *hyper) processHyperRoute(route netlink.Route, deviceName string) *hyperstart.Route {
gateway := route.Gw.String()
if gateway == "<nil>" {
gateway = ""
}
var destination string
if route.Dst == nil {
destination = ""
} else {
destination = route.Dst.String()
if destination == defaultRouteDest {
destination = defaultRouteLabel
}
// Skip IPv6 because not supported by hyperstart
if route.Dst.IP.To4() == nil {
h.Logger().WithFields(logrus.Fields{
"unsupported-route-type": "ipv6",
"destination": destination,
}).Warn("unsupported route")
return nil
}
}
return &hyperstart.Route{
Dest: destination,
Gateway: gateway,
Device: deviceName,
}
}
func (h *hyper) buildNetworkInterfacesAndRoutes(pod Pod) ([]hyperstart.NetworkIface, []hyperstart.Route, error) {
if pod.networkNS.NetNsPath == "" {
return []hyperstart.NetworkIface{}, []hyperstart.Route{}, nil
}
var ifaces []hyperstart.NetworkIface
var routes []hyperstart.Route
for _, endpoint := range pod.networkNS.Endpoints {
var ipAddresses []hyperstart.IPAddress
for _, addr := range endpoint.Properties().Addrs {
// Skip IPv6 because not supported by hyperstart.
// Skip localhost interface.
if addr.IP.To4() == nil || addr.IP.IsLoopback() {
continue
}
netMask, _ := addr.Mask.Size()
ipAddress := hyperstart.IPAddress{
IPAddress: addr.IP.String(),
NetMask: fmt.Sprintf("%d", netMask),
}
ipAddresses = append(ipAddresses, ipAddress)
}
iface := hyperstart.NetworkIface{
NewDevice: endpoint.Name(),
IPAddresses: ipAddresses,
MTU: endpoint.Properties().Iface.MTU,
MACAddr: endpoint.HardwareAddr(),
}
ifaces = append(ifaces, iface)
for _, r := range endpoint.Properties().Routes {
route := h.processHyperRoute(r, endpoint.Name())
if route == nil {
continue
}
routes = append(routes, *route)
}
}
return ifaces, routes, nil
}
func fsMapFromMounts(mounts []Mount) []*hyperstart.FsmapDescriptor {
var fsmap []*hyperstart.FsmapDescriptor
for _, m := range mounts {
fsmapDesc := &hyperstart.FsmapDescriptor{
Source: m.Source,
Path: m.Destination,
ReadOnly: m.ReadOnly,
DockerVolume: false,
}
fsmap = append(fsmap, fsmapDesc)
}
return fsmap
}
// init is the agent initialization implementation for hyperstart.
func (h *hyper) init(pod *Pod, config interface{}) (err error) {
switch c := config.(type) {
case HyperConfig:
// Create agent sockets from paths provided through
// configuration, or generate them from scratch.
h.generateSockets(*pod, c)
h.pod = *pod
default:
return fmt.Errorf("Invalid config type")
}
h.proxy, err = newProxy(pod.config.ProxyType)
if err != nil {
return err
}
h.shim, err = newShim(pod.config.ShimType)
if err != nil {
return err
}
// Fetch agent runtime info.
if err := pod.storage.fetchAgentState(pod.id, &h.state); err != nil {
h.Logger().Debug("Could not retrieve anything from storage")
}
return nil
}
func (h *hyper) createPod(pod *Pod) (err error) {
for _, socket := range h.sockets {
err := pod.hypervisor.addDevice(socket, serialPortDev)
if err != nil {
return err
}
}
// Adding the hyper shared volume.
// This volume contains all bind mounted container bundles.
sharedVolume := Volume{
MountTag: mountTag,
HostPath: filepath.Join(defaultSharedDir, pod.id),
}
if err := os.MkdirAll(sharedVolume.HostPath, dirMode); err != nil {
return err
}
return pod.hypervisor.addDevice(sharedVolume, fsDev)
}
func (h *hyper) capabilities() capabilities {
var caps capabilities
// add all capabilities supported by agent
caps.setBlockDeviceSupport()
return caps
}
// exec is the agent command execution implementation for hyperstart.
func (h *hyper) exec(pod *Pod, c Container, cmd Cmd) (*Process, error) {
token, err := h.attach()
if err != nil {
return nil, err
}
hyperProcess, err := h.buildHyperContainerProcess(cmd)
if err != nil {
return nil, err
}
execCommand := hyperstart.ExecCommand{
Container: c.id,
Process: *hyperProcess,
}
enterNSList := []ns.Namespace{
{
PID: c.process.Pid,
Type: ns.NSTypeNet,
},
{
PID: c.process.Pid,
Type: ns.NSTypePID,
},
}
process, err := prepareAndStartShim(pod, h.shim, c.id,
token, h.state.URL, cmd, []ns.NSType{}, enterNSList)
if err != nil {
return nil, err
}
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.ExecCmd,
message: execCommand,
token: process.Token,
}
if _, err := h.sendCmd(proxyCmd); err != nil {
return nil, err
}
return process, nil
}
// startPod is the agent Pod starting implementation for hyperstart.
func (h *hyper) startPod(pod Pod) error {
// Start the proxy here
pid, uri, err := h.proxy.start(pod, proxyParams{})
if err != nil {
return err
}
// Fill agent state with proxy information, and store them.
h.state.ProxyPid = pid
h.state.URL = uri
if err := pod.storage.storeAgentState(pod.id, h.state); err != nil {
return err
}
h.Logger().WithField("proxy-pid", pid).Info("proxy started")
if err := h.register(); err != nil {
return err
}
ifaces, routes, err := h.buildNetworkInterfacesAndRoutes(pod)
if err != nil {
return err
}
hostname := pod.config.Hostname
if len(hostname) > maxHostnameLen {
hostname = hostname[:maxHostnameLen]
}
hyperPod := hyperstart.Pod{
Hostname: hostname,
Containers: []hyperstart.Container{},
Interfaces: ifaces,
Routes: routes,
ShareDir: mountTag,
}
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.StartPod,
message: hyperPod,
}
_, err = h.sendCmd(proxyCmd)
return err
}
// stopPod is the agent Pod stopping implementation for hyperstart.
func (h *hyper) stopPod(pod Pod) error {
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.DestroyPod,
message: nil,
}
if _, err := h.sendCmd(proxyCmd); err != nil {
return err
}
if err := h.unregister(); err != nil {
return err
}
return h.proxy.stop(pod, h.state.ProxyPid)
}
func (h *hyper) startOneContainer(pod Pod, c *Container) error {
process, err := h.buildHyperContainerProcess(c.config.Cmd)
if err != nil {
return err
}
container := hyperstart.Container{
ID: c.id,
Image: c.id,
Rootfs: rootfsDir,
Process: process,
}
if c.config.Resources.CPUQuota != 0 && c.config.Resources.CPUPeriod != 0 {
container.Constraints = hyperstart.Constraints{
CPUQuota: c.config.Resources.CPUQuota,
CPUPeriod: c.config.Resources.CPUPeriod,
}
}
if c.config.Resources.CPUShares != 0 {
container.Constraints.CPUShares = c.config.Resources.CPUShares
}
container.SystemMountsInfo.BindMountDev = c.systemMountsInfo.BindMountDev
if c.state.Fstype != "" {
// Pass a drive name only in case of block driver
if pod.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
driveName, err := getVirtDriveName(c.state.BlockIndex)
if err != nil {
return err
}
container.Image = driveName
} else {
scsiAddr, err := getSCSIAddress(c.state.BlockIndex)
if err != nil {
return err
}
container.SCSIAddr = scsiAddr
}
container.Fstype = c.state.Fstype
} else {
if err := bindMountContainerRootfs(defaultSharedDir, pod.id, c.id, c.rootFs, false); err != nil {
bindUnmountAllRootfs(defaultSharedDir, pod)
return err
}
}
//TODO : Enter mount namespace
// Handle container mounts
newMounts, err := c.mountSharedDirMounts(defaultSharedDir, "")
if err != nil {
bindUnmountAllRootfs(defaultSharedDir, pod)
return err
}
fsmap := fsMapFromMounts(newMounts)
// Append container mounts for block devices passed with --device.
for _, device := range c.devices {
d, ok := device.(*BlockDevice)
if ok {
fsmapDesc := &hyperstart.FsmapDescriptor{
Source: d.VirtPath,
Path: d.DeviceInfo.ContainerPath,
AbsolutePath: true,
DockerVolume: false,
SCSIAddr: d.SCSIAddr,
}
fsmap = append(fsmap, fsmapDesc)
}
}
// Assign fsmap for hyperstart to mount these at the correct location within the container
container.Fsmap = fsmap
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.NewContainer,
message: container,
token: c.process.Token,
}
if _, err := h.sendCmd(proxyCmd); err != nil {
return err
}
return nil
}
// createContainer is the agent Container creation implementation for hyperstart.
func (h *hyper) createContainer(pod *Pod, c *Container) (*Process, error) {
token, err := h.attach()
if err != nil {
return nil, err
}
createNSList := []ns.NSType{ns.NSTypePID}
enterNSList := []ns.Namespace{
{
Path: pod.networkNS.NetNsPath,
Type: ns.NSTypeNet,
},
}
return prepareAndStartShim(pod, h.shim, c.id, token,
h.state.URL, c.config.Cmd, createNSList, enterNSList)
}
// startContainer is the agent Container starting implementation for hyperstart.
func (h *hyper) startContainer(pod Pod, c *Container) error {
return h.startOneContainer(pod, c)
}
// stopContainer is the agent Container stopping implementation for hyperstart.
func (h *hyper) stopContainer(pod Pod, c Container) error {
// Nothing to be done in case the container has not been started.
if c.state.State == StateReady {
return nil
}
return h.stopOneContainer(pod.id, c)
}
func (h *hyper) stopOneContainer(podID string, c Container) error {
removeCommand := hyperstart.RemoveCommand{
Container: c.id,
}
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.RemoveContainer,
message: removeCommand,
}
if _, err := h.sendCmd(proxyCmd); err != nil {
return err
}
if err := c.unmountHostMounts(); err != nil {
return err
}
if c.state.Fstype == "" {
if err := bindUnmountContainerRootfs(defaultSharedDir, podID, c.id); err != nil {
return err
}
}
return nil
}
// killContainer is the agent process signal implementation for hyperstart.
func (h *hyper) killContainer(pod Pod, c Container, signal syscall.Signal, all bool) error {
// Send the signal to the shim directly in case the container has not
// been started yet.
if c.state.State == StateReady {
return signalShim(c.process.Pid, signal)
}
return h.killOneContainer(c.id, signal, all)
}
func (h *hyper) killOneContainer(cID string, signal syscall.Signal, all bool) error {
killCmd := hyperstart.KillCommand{
Container: cID,
Signal: signal,
AllProcesses: all,
}
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.KillContainer,
message: killCmd,
}
if _, err := h.sendCmd(proxyCmd); err != nil {
return err
}
return nil
}
func (h *hyper) processListContainer(pod Pod, c Container, options ProcessListOptions) (ProcessList, error) {
return h.processListOneContainer(pod.id, c.id, options)
}
func (h *hyper) processListOneContainer(podID, cID string, options ProcessListOptions) (ProcessList, error) {
psCmd := hyperstart.PsCommand{
Container: cID,
Format: options.Format,
PsArgs: options.Args,
}
proxyCmd := hyperstartProxyCmd{
cmd: hyperstart.PsContainer,
message: psCmd,
}
response, err := h.sendCmd(proxyCmd)
if err != nil {
return nil, err
}
msg, ok := response.([]byte)
if !ok {
return nil, fmt.Errorf("failed to get response message from container %s pod %s", cID, podID)
}
return msg, nil
}
// connectProxyRetry repeatedly tries to connect to the proxy on the specified
// address until a timeout state is reached, when it will fail.
func (h *hyper) connectProxyRetry(scheme, address string) (conn net.Conn, err error) {
attempt := 1
timeoutSecs := time.Duration(waitForProxyTimeoutSecs * time.Second)
startTime := time.Now()
lastLogTime := startTime
for {
conn, err = net.Dial(scheme, address)
if err == nil {
// If the initial connection was unsuccessful,
// ensure a log message is generated when successfully
// connected.
if attempt > 1 {
h.Logger().WithField("attempt", fmt.Sprintf("%d", attempt)).Info("Connected to proxy")
}
return conn, nil
}
attempt++
now := time.Now()
delta := now.Sub(startTime)
remaining := timeoutSecs - delta
if remaining <= 0 {
return nil, fmt.Errorf("failed to connect to proxy after %v: %v", timeoutSecs, err)
}
logDelta := now.Sub(lastLogTime)
logDeltaSecs := logDelta / time.Second
if logDeltaSecs >= 1 {
h.Logger().WithError(err).WithFields(logrus.Fields{
"attempt": fmt.Sprintf("%d", attempt),
"proxy-network": scheme,
"proxy-address": address,
"remaining-time-secs": fmt.Sprintf("%2.2f", remaining.Seconds()),
}).Warning("Retrying proxy connection")
lastLogTime = now
}
time.Sleep(time.Duration(100) * time.Millisecond)
}
}
func (h *hyper) connect() error {
if h.client != nil {
return nil
}
u, err := url.Parse(h.state.URL)
if err != nil {
return err
}
if u.Scheme == "" {
return fmt.Errorf("URL scheme cannot be empty")
}
address := u.Host
if address == "" {
if u.Path == "" {
return fmt.Errorf("URL host and path cannot be empty")
}
address = u.Path
}
conn, err := h.connectProxyRetry(u.Scheme, address)
if err != nil {
return err
}
h.client = proxyClient.NewClient(conn)
return nil
}
func (h *hyper) disconnect() {
if h.client == nil {
return
}
h.client.Close()
h.client = nil
}
func (h *hyper) register() error {
if err := h.connect(); err != nil {
return err
}
defer h.disconnect()
registerVMOptions := &proxyClient.RegisterVMOptions{
Console: h.pod.hypervisor.getPodConsole(h.pod.id),
NumIOStreams: 0,
}
_, err := h.client.RegisterVM(h.pod.id, h.sockets[0].HostPath,
h.sockets[1].HostPath, registerVMOptions)
return err
}
func (h *hyper) unregister() error {
if err := h.connect(); err != nil {
return err
}
defer h.disconnect()
h.client.UnregisterVM(h.pod.id)
return nil
}
func (h *hyper) attach() (string, error) {
if err := h.connect(); err != nil {
return "", err
}
defer h.disconnect()
numTokens := 1
attachVMOptions := &proxyClient.AttachVMOptions{
NumIOStreams: numTokens,
}
attachVMReturn, err := h.client.AttachVM(h.pod.id, attachVMOptions)
if err != nil {
return "", err
}
if len(attachVMReturn.IO.Tokens) != numTokens {
return "", fmt.Errorf("%d tokens retrieved out of %d expected",
len(attachVMReturn.IO.Tokens), numTokens)
}
return attachVMReturn.IO.Tokens[0], nil
}
func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) {
if err := h.connect(); err != nil {
return nil, err
}
defer h.disconnect()
attachVMOptions := &proxyClient.AttachVMOptions{
NumIOStreams: 0,
}
if _, err := h.client.AttachVM(h.pod.id, attachVMOptions); err != nil {
return nil, err
}
var tokens []string
if proxyCmd.token != "" {
tokens = append(tokens, proxyCmd.token)
}
return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message)
}

169
virtcontainers/hyperstart_agent_test.go Normal file
View File

@@ -0,0 +1,169 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"net"
"reflect"
"testing"
"github.com/containers/virtcontainers/pkg/hyperstart"
"github.com/vishvananda/netlink"
)
var testRouteDest = "192.168.10.1/32"
var testRouteGateway = "192.168.0.0"
var testRouteDeviceName = "test_eth0"
var testRouteDestIPv6 = "2001:db8::/32"
func TestHyperstartGenerateSocketsSuccessful(t *testing.T) {
config := HyperConfig{
SockCtlName: "ctlSock",
SockTtyName: "ttySock",
}
pod := Pod{
id: testPodID,
}
h := &hyper{}
h.generateSockets(pod, config)
expectedSockets := []Socket{
{
DeviceID: fmt.Sprintf(defaultDeviceIDTemplate, 0),
ID: fmt.Sprintf(defaultIDTemplate, 0),
HostPath: config.SockCtlName,
Name: fmt.Sprintf(defaultChannelTemplate, 0),
},
{
DeviceID: fmt.Sprintf(defaultDeviceIDTemplate, 1),
ID: fmt.Sprintf(defaultIDTemplate, 1),
HostPath: config.SockTtyName,
Name: fmt.Sprintf(defaultChannelTemplate, 1),
},
}
if !reflect.DeepEqual(expectedSockets, h.sockets) {
t.Fatalf("Expecting %+v, Got %+v", expectedSockets, h.sockets)
}
}
func TestHyperstartGenerateSocketsSuccessfulNoPathProvided(t *testing.T) {
config := HyperConfig{}
pod := Pod{
id: testPodID,
}
h := &hyper{}
h.generateSockets(pod, config)
expectedSockets := []Socket{
{
DeviceID: fmt.Sprintf(defaultDeviceIDTemplate, 0),
ID: fmt.Sprintf(defaultIDTemplate, 0),
HostPath: fmt.Sprintf(defaultSockPathTemplates[0], runStoragePath, pod.id),
Name: fmt.Sprintf(defaultChannelTemplate, 0),
},
{
DeviceID: fmt.Sprintf(defaultDeviceIDTemplate, 1),
ID: fmt.Sprintf(defaultIDTemplate, 1),
HostPath: fmt.Sprintf(defaultSockPathTemplates[1], runStoragePath, pod.id),
Name: fmt.Sprintf(defaultChannelTemplate, 1),
},
}
if !reflect.DeepEqual(expectedSockets, h.sockets) {
t.Fatalf("Expecting %+v, Got %+v", expectedSockets, h.sockets)
}
}
func testProcessHyperRoute(t *testing.T, route netlink.Route, deviceName string, expected *hyperstart.Route) {
h := &hyper{}
hyperRoute := h.processHyperRoute(route, deviceName)
if expected == nil {
if hyperRoute != nil {
t.Fatalf("Expecting route to be nil, Got %+v", hyperRoute)
} else {
return
}
}
// At this point, we know that "expected" != nil.
if !reflect.DeepEqual(*expected, *hyperRoute) {
t.Fatalf("Expecting %+v, Got %+v", *expected, *hyperRoute)
}
}
func TestProcessHyperRouteEmptyGWSuccessful(t *testing.T) {
expected := &hyperstart.Route{
Dest: testRouteDest,
Gateway: "",
Device: testRouteDeviceName,
}
_, dest, err := net.ParseCIDR(testRouteDest)
if err != nil {
t.Fatal(err)
}
route := netlink.Route{
Dst: dest,
Gw: net.IP{},
}
testProcessHyperRoute(t, route, testRouteDeviceName, expected)
}
func TestProcessHyperRouteEmptyDestSuccessful(t *testing.T) {
expected := &hyperstart.Route{
Dest: defaultRouteLabel,
Gateway: testRouteGateway,
Device: testRouteDeviceName,
}
_, dest, err := net.ParseCIDR(defaultRouteDest)
if err != nil {
t.Fatal(err)
}
route := netlink.Route{
Dst: dest,
Gw: net.ParseIP(testRouteGateway),
}
testProcessHyperRoute(t, route, testRouteDeviceName, expected)
}
func TestProcessHyperRouteDestIPv6Failure(t *testing.T) {
_, dest, err := net.ParseCIDR(testRouteDestIPv6)
if err != nil {
t.Fatal(err)
}
route := netlink.Route{
Dst: dest,
Gw: net.IP{},
}
testProcessHyperRoute(t, route, testRouteDeviceName, nil)
}

500
virtcontainers/hypervisor.go Normal file
View File

@@ -0,0 +1,500 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"bufio"
"fmt"
"os"
"runtime"
"strconv"
"strings"
)
// HypervisorType describes an hypervisor type.
type HypervisorType string
const (
// QemuHypervisor is the QEMU hypervisor.
QemuHypervisor HypervisorType = "qemu"
// MockHypervisor is a mock hypervisor for testing purposes
MockHypervisor HypervisorType = "mock"
)
const (
procMemInfo = "/proc/meminfo"
procCPUInfo = "/proc/cpuinfo"
)
const (
defaultVCPUs = 1
// 2 GiB
defaultMemSzMiB = 2048
defaultBridges = 1
defaultBlockDriver = VirtioSCSI
)
// In some architectures the maximum number of vCPUs depends on the number of physical cores.
var defaultMaxQemuVCPUs = maxQemuVCPUs()
// deviceType describes a virtualized device type.
type deviceType int
const (
// ImgDev is the image device type.
imgDev deviceType = iota
// FsDev is the filesystem device type.
fsDev
// NetDev is the network device type.
netDev
// SerialDev is the serial device type.
serialDev
// BlockDev is the block device type.
blockDev
// ConsoleDev is the console device type.
consoleDev
// SerialPortDev is the serial port device type.
serialPortDev
// VFIODevice is VFIO device type
vfioDev
// vhostuserDev is a Vhost-user device type
vhostuserDev
// CPUDevice is CPU device type
cpuDev
)
// Set sets an hypervisor type based on the input string.
func (hType *HypervisorType) Set(value string) error {
switch value {
case "qemu":
*hType = QemuHypervisor
return nil
case "mock":
*hType = MockHypervisor
return nil
default:
return fmt.Errorf("Unknown hypervisor type %s", value)
}
}
// String converts an hypervisor type to a string.
func (hType *HypervisorType) String() string {
switch *hType {
case QemuHypervisor:
return string(QemuHypervisor)
case MockHypervisor:
return string(MockHypervisor)
default:
return ""
}
}
// newHypervisor returns an hypervisor from and hypervisor type.
func newHypervisor(hType HypervisorType) (hypervisor, error) {
switch hType {
case QemuHypervisor:
return &qemu{}, nil
case MockHypervisor:
return &mockHypervisor{}, nil
default:
return nil, fmt.Errorf("Unknown hypervisor type %s", hType)
}
}
//Generic function for creating a named-id for passing on the hypervisor commandline
func makeNameID(namedType string, id string) string {
nameID := fmt.Sprintf("%s-%s", namedType, id)
if len(nameID) > maxDevIDSize {
nameID = string(nameID[:maxDevIDSize])
}
return nameID
}
// Param is a key/value representation for hypervisor and kernel parameters.
type Param struct {
Key string
Value string
}
// HypervisorConfig is the hypervisor configuration.
type HypervisorConfig struct {
// KernelPath is the guest kernel host path.
KernelPath string
// ImagePath is the guest image host path.
ImagePath string
// FirmwarePath is the bios host path
FirmwarePath string
// MachineAccelerators are machine specific accelerators
MachineAccelerators string
// HypervisorPath is the hypervisor executable host path.
HypervisorPath string
// DisableBlockDeviceUse disallows a block device from being used.
DisableBlockDeviceUse bool
// BlockDeviceDriver specifies the driver to be used for block device
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
BlockDeviceDriver string
// KernelParams are additional guest kernel parameters.
KernelParams []Param
// HypervisorParams are additional hypervisor parameters.
HypervisorParams []Param
// HypervisorMachineType specifies the type of machine being
// emulated.
HypervisorMachineType string
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available.
Debug bool
// DefaultVCPUs specifies default number of vCPUs for the VM.
DefaultVCPUs uint32
//DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM.
DefaultMaxVCPUs uint32
// DefaultMem specifies default memory size in MiB for the VM.
// Pod configuration VMConfig.Memory overwrites this.
DefaultMemSz uint32
// DefaultBridges specifies default number of bridges for the VM.
// Bridges can be used to hot plug devices
DefaultBridges uint32
// MemPrealloc specifies if the memory should be pre-allocated
MemPrealloc bool
// HugePages specifies if the memory should be pre-allocated from huge pages
HugePages bool
// Realtime Used to enable/disable realtime
Realtime bool
// Mlock is used to control memory locking when Realtime is enabled
// Realtime=true and Mlock=false, allows for swapping out of VM memory
// enabling higher density
Mlock bool
// DisableNestingChecks is used to override customizations performed
// when running on top of another VMM.
DisableNestingChecks bool
// customAssets is a map of assets.
// Each value in that map takes precedence over the configured assets.
// For example, if there is a value for the "kernel" key in this map,
// it will be used for the pod's kernel path instead of KernelPath.
customAssets map[assetType]*asset
}
func (conf *HypervisorConfig) valid() (bool, error) {
if conf.KernelPath == "" {
return false, fmt.Errorf("Missing kernel path")
}
if conf.ImagePath == "" {
return false, fmt.Errorf("Missing image path")
}
if conf.DefaultVCPUs == 0 {
conf.DefaultVCPUs = defaultVCPUs
}
if conf.DefaultMemSz == 0 {
conf.DefaultMemSz = defaultMemSzMiB
}
if conf.DefaultBridges == 0 {
conf.DefaultBridges = defaultBridges
}
if conf.BlockDeviceDriver == "" {
conf.BlockDeviceDriver = defaultBlockDriver
}
if conf.DefaultMaxVCPUs == 0 {
conf.DefaultMaxVCPUs = defaultMaxQemuVCPUs
}
return true, nil
}
// AddKernelParam allows the addition of new kernel parameters to an existing
// hypervisor configuration.
func (conf *HypervisorConfig) AddKernelParam(p Param) error {
if p.Key == "" {
return fmt.Errorf("Empty kernel parameter")
}
conf.KernelParams = append(conf.KernelParams, p)
return nil
}
func (conf *HypervisorConfig) addCustomAsset(a *asset) error {
if a == nil || a.path == "" {
// We did not get a custom asset, we will use the default one.
return nil
}
if !a.valid() {
return fmt.Errorf("Invalid %s at %s", a.kind, a.path)
}
virtLog.Debugf("Using custom %v asset %s", a.kind, a.path)
if conf.customAssets == nil {
conf.customAssets = make(map[assetType]*asset)
}
conf.customAssets[a.kind] = a
return nil
}
func (conf *HypervisorConfig) assetPath(t assetType) (string, error) {
// Custom assets take precedence over the configured ones
a, ok := conf.customAssets[t]
if ok {
return a.path, nil
}
// We could not find a custom asset for the given type, let's
// fall back to the configured ones.
switch t {
case kernelAsset:
return conf.KernelPath, nil
case imageAsset:
return conf.ImagePath, nil
case hypervisorAsset:
return conf.HypervisorPath, nil
case firmwareAsset:
return conf.FirmwarePath, nil
default:
return "", fmt.Errorf("Unknown asset type %v", t)
}
}
func (conf *HypervisorConfig) isCustomAsset(t assetType) bool {
_, ok := conf.customAssets[t]
if ok {
return true
}
return false
}
// KernelAssetPath returns the guest kernel path
func (conf *HypervisorConfig) KernelAssetPath() (string, error) {
return conf.assetPath(kernelAsset)
}
// CustomKernelAsset returns true if the kernel asset is a custom one, false otherwise.
func (conf *HypervisorConfig) CustomKernelAsset() bool {
return conf.isCustomAsset(kernelAsset)
}
// ImageAssetPath returns the guest image path
func (conf *HypervisorConfig) ImageAssetPath() (string, error) {
return conf.assetPath(imageAsset)
}
// CustomImageAsset returns true if the image asset is a custom one, false otherwise.
func (conf *HypervisorConfig) CustomImageAsset() bool {
return conf.isCustomAsset(imageAsset)
}
// HypervisorAssetPath returns the VM hypervisor path
func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) {
return conf.assetPath(hypervisorAsset)
}
// CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise.
func (conf *HypervisorConfig) CustomHypervisorAsset() bool {
return conf.isCustomAsset(hypervisorAsset)
}
// FirmwareAssetPath returns the guest firmware path
func (conf *HypervisorConfig) FirmwareAssetPath() (string, error) {
return conf.assetPath(firmwareAsset)
}
// CustomFirmwareAsset returns true if the firmware asset is a custom one, false otherwise.
func (conf *HypervisorConfig) CustomFirmwareAsset() bool {
return conf.isCustomAsset(firmwareAsset)
}
func appendParam(params []Param, parameter string, value string) []Param {
return append(params, Param{parameter, value})
}
// SerializeParams converts []Param to []string
func SerializeParams(params []Param, delim string) []string {
var parameters []string
for _, p := range params {
if p.Key == "" && p.Value == "" {
continue
} else if p.Key == "" {
parameters = append(parameters, fmt.Sprintf("%s", p.Value))
} else if p.Value == "" {
parameters = append(parameters, fmt.Sprintf("%s", p.Key))
} else if delim == "" {
parameters = append(parameters, fmt.Sprintf("%s", p.Key))
parameters = append(parameters, fmt.Sprintf("%s", p.Value))
} else {
parameters = append(parameters, fmt.Sprintf("%s%s%s", p.Key, delim, p.Value))
}
}
return parameters
}
// DeserializeParams converts []string to []Param
func DeserializeParams(parameters []string) []Param {
var params []Param
for _, param := range parameters {
if param == "" {
continue
}
p := strings.SplitN(param, "=", 2)
if len(p) == 2 {
params = append(params, Param{Key: p[0], Value: p[1]})
} else {
params = append(params, Param{Key: p[0], Value: ""})
}
}
return params
}
func getHostMemorySizeKb(memInfoPath string) (uint64, error) {
f, err := os.Open(memInfoPath)
if err != nil {
return 0, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
// Expected format: ["MemTotal:", "1234", "kB"]
parts := strings.Fields(scanner.Text())
// Sanity checks: Skip malformed entries.
if len(parts) < 3 || parts[0] != "MemTotal:" || parts[2] != "kB" {
continue
}
sizeKb, err := strconv.ParseUint(parts[1], 0, 64)
if err != nil {
continue
}
return sizeKb, nil
}
// Handle errors that may have occurred during the reading of the file.
if err := scanner.Err(); err != nil {
return 0, err
}
return 0, fmt.Errorf("unable get MemTotal from %s", memInfoPath)
}
// RunningOnVMM checks if the system is running inside a VM.
func RunningOnVMM(cpuInfoPath string) (bool, error) {
if runtime.GOARCH == "arm64" {
virtLog.Debugf("Unable to know if the system is running inside a VM")
return false, nil
}
flagsField := "flags"
f, err := os.Open(cpuInfoPath)
if err != nil {
return false, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
// Expected format: ["flags", ":", ...] or ["flags:", ...]
fields := strings.Fields(scanner.Text())
if len(fields) < 2 {
continue
}
if !strings.HasPrefix(fields[0], flagsField) {
continue
}
for _, field := range fields[1:] {
if field == "hypervisor" {
return true, nil
}
}
// As long as we have been able to analyze the fields from
// "flags", there is no reason to check what comes next from
// /proc/cpuinfo, because we already know we are not running
// on a VMM.
return false, nil
}
if err := scanner.Err(); err != nil {
return false, err
}
return false, fmt.Errorf("Couldn't find %q from %q output", flagsField, cpuInfoPath)
}
// hypervisor is the virtcontainers hypervisor interface.
// The default hypervisor implementation is Qemu.
type hypervisor interface {
init(pod *Pod) error
createPod(podConfig PodConfig) error
startPod() error
waitPod(timeout int) error
stopPod() error
pausePod() error
resumePod() error
addDevice(devInfo interface{}, devType deviceType) error
hotplugAddDevice(devInfo interface{}, devType deviceType) error
hotplugRemoveDevice(devInfo interface{}, devType deviceType) error
getPodConsole(podID string) string
capabilities() capabilities
}

502
virtcontainers/hypervisor_test.go Normal file
View File

@@ -0,0 +1,502 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
)
func testSetHypervisorType(t *testing.T, value string, expected HypervisorType) {
var hypervisorType HypervisorType
err := (&hypervisorType).Set(value)
if err != nil {
t.Fatal(err)
}
if hypervisorType != expected {
t.Fatal()
}
}
func TestSetQemuHypervisorType(t *testing.T) {
testSetHypervisorType(t, "qemu", QemuHypervisor)
}
func TestSetMockHypervisorType(t *testing.T) {
testSetHypervisorType(t, "mock", MockHypervisor)
}
func TestSetUnknownHypervisorType(t *testing.T) {
var hypervisorType HypervisorType
err := (&hypervisorType).Set("unknown")
if err == nil {
t.Fatal()
}
if hypervisorType == QemuHypervisor ||
hypervisorType == MockHypervisor {
t.Fatal()
}
}
func testStringFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected string) {
hypervisorTypeStr := (&hypervisorType).String()
if hypervisorTypeStr != expected {
t.Fatal()
}
}
func TestStringFromQemuHypervisorType(t *testing.T) {
hypervisorType := QemuHypervisor
testStringFromHypervisorType(t, hypervisorType, "qemu")
}
func TestStringFromMockHypervisorType(t *testing.T) {
hypervisorType := MockHypervisor
testStringFromHypervisorType(t, hypervisorType, "mock")
}
func TestStringFromUnknownHypervisorType(t *testing.T) {
var hypervisorType HypervisorType
testStringFromHypervisorType(t, hypervisorType, "")
}
func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected hypervisor) {
hy, err := newHypervisor(hypervisorType)
if err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(hy, expected) == false {
t.Fatal()
}
}
func TestNewHypervisorFromQemuHypervisorType(t *testing.T) {
hypervisorType := QemuHypervisor
expectedHypervisor := &qemu{}
testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor)
}
func TestNewHypervisorFromMockHypervisorType(t *testing.T) {
hypervisorType := MockHypervisor
expectedHypervisor := &mockHypervisor{}
testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor)
}
func TestNewHypervisorFromUnknownHypervisorType(t *testing.T) {
var hypervisorType HypervisorType
hy, err := newHypervisor(hypervisorType)
if err == nil {
t.Fatal()
}
if hy != nil {
t.Fatal()
}
}
func testHypervisorConfigValid(t *testing.T, hypervisorConfig *HypervisorConfig, expected bool) {
ret, _ := hypervisorConfig.valid()
if ret != expected {
t.Fatal()
}
}
func TestHypervisorConfigNoKernelPath(t *testing.T) {
hypervisorConfig := &HypervisorConfig{
KernelPath: "",
ImagePath: fmt.Sprintf("%s/%s", testDir, testImage),
HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
}
testHypervisorConfigValid(t, hypervisorConfig, false)
}
func TestHypervisorConfigNoImagePath(t *testing.T) {
hypervisorConfig := &HypervisorConfig{
KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel),
ImagePath: "",
HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
}
testHypervisorConfigValid(t, hypervisorConfig, false)
}
func TestHypervisorConfigNoHypervisorPath(t *testing.T) {
hypervisorConfig := &HypervisorConfig{
KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel),
ImagePath: fmt.Sprintf("%s/%s", testDir, testImage),
HypervisorPath: "",
}
testHypervisorConfigValid(t, hypervisorConfig, true)
}
func TestHypervisorConfigIsValid(t *testing.T) {
hypervisorConfig := &HypervisorConfig{
KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel),
ImagePath: fmt.Sprintf("%s/%s", testDir, testImage),
HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
}
testHypervisorConfigValid(t, hypervisorConfig, true)
}
func TestHypervisorConfigDefaults(t *testing.T) {
hypervisorConfig := &HypervisorConfig{
KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel),
ImagePath: fmt.Sprintf("%s/%s", testDir, testImage),
HypervisorPath: "",
}
testHypervisorConfigValid(t, hypervisorConfig, true)
hypervisorConfigDefaultsExpected := &HypervisorConfig{
KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel),
ImagePath: fmt.Sprintf("%s/%s", testDir, testImage),
HypervisorPath: "",
DefaultVCPUs: defaultVCPUs,
DefaultMemSz: defaultMemSzMiB,
DefaultBridges: defaultBridges,
BlockDeviceDriver: defaultBlockDriver,
DefaultMaxVCPUs: defaultMaxQemuVCPUs,
}
if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false {
t.Fatal()
}
}
func TestAppendParams(t *testing.T) {
paramList := []Param{
{
Key: "param1",
Value: "value1",
},
}
expectedParams := []Param{
{
Key: "param1",
Value: "value1",
},
{
Key: "param2",
Value: "value2",
},
}
paramList = appendParam(paramList, "param2", "value2")
if reflect.DeepEqual(paramList, expectedParams) == false {
t.Fatal()
}
}
func testSerializeParams(t *testing.T, params []Param, delim string, expected []string) {
result := SerializeParams(params, delim)
if reflect.DeepEqual(result, expected) == false {
t.Fatal()
}
}
func TestSerializeParamsNoParamNoValue(t *testing.T) {
params := []Param{
{
Key: "",
Value: "",
},
}
var expected []string
testSerializeParams(t, params, "", expected)
}
func TestSerializeParamsNoParam(t *testing.T) {
params := []Param{
{
Value: "value1",
},
}
expected := []string{"value1"}
testSerializeParams(t, params, "", expected)
}
func TestSerializeParamsNoValue(t *testing.T) {
params := []Param{
{
Key: "param1",
},
}
expected := []string{"param1"}
testSerializeParams(t, params, "", expected)
}
func TestSerializeParamsNoDelim(t *testing.T) {
params := []Param{
{
Key: "param1",
Value: "value1",
},
}
expected := []string{"param1", "value1"}
testSerializeParams(t, params, "", expected)
}
func TestSerializeParams(t *testing.T) {
params := []Param{
{
Key: "param1",
Value: "value1",
},
}
expected := []string{"param1=value1"}
testSerializeParams(t, params, "=", expected)
}
func testDeserializeParams(t *testing.T, parameters []string, expected []Param) {
result := DeserializeParams(parameters)
if reflect.DeepEqual(result, expected) == false {
t.Fatal()
}
}
func TestDeserializeParamsNil(t *testing.T) {
var parameters []string
var expected []Param
testDeserializeParams(t, parameters, expected)
}
func TestDeserializeParamsNoParamNoValue(t *testing.T) {
parameters := []string{
"",
}
var expected []Param
testDeserializeParams(t, parameters, expected)
}
func TestDeserializeParamsNoValue(t *testing.T) {
parameters := []string{
"param1",
}
expected := []Param{
{
Key: "param1",
},
}
testDeserializeParams(t, parameters, expected)
}
func TestDeserializeParams(t *testing.T) {
parameters := []string{
"param1=value1",
}
expected := []Param{
{
Key: "param1",
Value: "value1",
},
}
testDeserializeParams(t, parameters, expected)
}
func TestAddKernelParamValid(t *testing.T) {
var config HypervisorConfig
expected := []Param{
{"foo", "bar"},
}
err := config.AddKernelParam(expected[0])
if err != nil || reflect.DeepEqual(config.KernelParams, expected) == false {
t.Fatal()
}
}
func TestAddKernelParamInvalid(t *testing.T) {
var config HypervisorConfig
invalid := []Param{
{"", "bar"},
}
err := config.AddKernelParam(invalid[0])
if err == nil {
t.Fatal()
}
}
func TestGetHostMemorySizeKb(t *testing.T) {
type testData struct {
contents string
expectedResult int
expectError bool
}
data := []testData{
{
`
MemTotal: 1 kB
MemFree: 2 kB
SwapTotal: 3 kB
SwapFree: 4 kB
`,
1024,
false,
},
{
`
MemFree: 2 kB
SwapTotal: 3 kB
SwapFree: 4 kB
`,
0,
true,
},
}
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
file := filepath.Join(dir, "meminfo")
if _, err := getHostMemorySizeKb(file); err == nil {
t.Fatalf("expected failure as file %q does not exist", file)
}
for _, d := range data {
if err := ioutil.WriteFile(file, []byte(d.contents), os.FileMode(0640)); err != nil {
t.Fatal(err)
}
defer os.Remove(file)
hostMemKb, err := getHostMemorySizeKb(file)
if (d.expectError && err == nil) || (!d.expectError && err != nil) {
t.Fatalf("got %d, input %v", hostMemKb, d)
}
if reflect.DeepEqual(hostMemKb, d.expectedResult) {
t.Fatalf("got %d, input %v", hostMemKb, d)
}
}
}
var dataFlagsFieldWithoutHypervisor = []byte(`
fpu_exception : yes
cpuid level : 20
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology eagerfpu pni pclmulqdq vmx ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch tpr_shadow vnmi ept vpid fsgsbase bmi1 hle avx2 smep bmi2 erms rtm rdseed adx smap xsaveopt
bugs :
bogomips : 4589.35
`)
var dataFlagsFieldWithHypervisor = []byte(`
fpu_exception : yes
cpuid level : 20
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology eagerfpu pni pclmulqdq vmx ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch tpr_shadow vnmi ept vpid fsgsbase bmi1 hle avx2 smep bmi2 erms rtm rdseed adx smap xsaveopt
bugs :
bogomips : 4589.35
`)
var dataWithoutFlagsField = []byte(`
fpu_exception : yes
cpuid level : 20
wp : yes
bugs :
bogomips : 4589.35
`)
func testRunningOnVMMSuccessful(t *testing.T, cpuInfoContent []byte, expectedErr bool, expected bool) {
f, err := ioutil.TempFile("", "cpuinfo")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
defer f.Close()
n, err := f.Write(cpuInfoContent)
if err != nil {
t.Fatal(err)
}
if n != len(cpuInfoContent) {
t.Fatalf("Only %d bytes written out of %d expected", n, len(cpuInfoContent))
}
running, err := RunningOnVMM(f.Name())
if !expectedErr && err != nil {
t.Fatalf("This test should succeed: %v", err)
} else if expectedErr && err == nil {
t.Fatalf("This test should fail")
}
if running != expected {
t.Fatalf("Expecting running on VMM = %t, Got %t", expected, running)
}
}
func TestRunningOnVMMFalseSuccessful(t *testing.T) {
testRunningOnVMMSuccessful(t, dataFlagsFieldWithoutHypervisor, false, false)
}
func TestRunningOnVMMTrueSuccessful(t *testing.T) {
testRunningOnVMMSuccessful(t, dataFlagsFieldWithHypervisor, false, true)
}
func TestRunningOnVMMNoFlagsFieldFailure(t *testing.T) {
testRunningOnVMMSuccessful(t, dataWithoutFlagsField, true, false)
}
func TestRunningOnVMMNotExistingCPUInfoPathFailure(t *testing.T) {
f, err := ioutil.TempFile("", "cpuinfo")
if err != nil {
t.Fatal(err)
}
filePath := f.Name()
f.Close()
os.Remove(filePath)
if _, err := RunningOnVMM(filePath); err == nil {
t.Fatalf("Should fail because %q file path does not exist", filePath)
}
}

119
virtcontainers/implementation.go Normal file
View File

@@ -0,0 +1,119 @@
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Description: The true virtcontainers function of the same name.
// This indirection is required to allow an alternative implemenation to be
// used for testing purposes.
package virtcontainers
import (
"syscall"
"github.com/sirupsen/logrus"
)
// VCImpl is the official virtcontainers function of the same name.
type VCImpl struct {
}
// SetLogger implements the VC function of the same name.
func (impl *VCImpl) SetLogger(logger logrus.FieldLogger) {
SetLogger(logger)
}
// CreatePod implements the VC function of the same name.
func (impl *VCImpl) CreatePod(podConfig PodConfig) (VCPod, error) {
return CreatePod(podConfig)
}
// DeletePod implements the VC function of the same name.
func (impl *VCImpl) DeletePod(podID string) (VCPod, error) {
return DeletePod(podID)
}
// StartPod implements the VC function of the same name.
func (impl *VCImpl) StartPod(podID string) (VCPod, error) {
return StartPod(podID)
}
// StopPod implements the VC function of the same name.
func (impl *VCImpl) StopPod(podID string) (VCPod, error) {
return StopPod(podID)
}
// RunPod implements the VC function of the same name.
func (impl *VCImpl) RunPod(podConfig PodConfig) (VCPod, error) {
return RunPod(podConfig)
}
// ListPod implements the VC function of the same name.
func (impl *VCImpl) ListPod() ([]PodStatus, error) {
return ListPod()
}
// StatusPod implements the VC function of the same name.
func (impl *VCImpl) StatusPod(podID string) (PodStatus, error) {
return StatusPod(podID)
}
// PausePod implements the VC function of the same name.
func (impl *VCImpl) PausePod(podID string) (VCPod, error) {
return PausePod(podID)
}
// ResumePod implements the VC function of the same name.
func (impl *VCImpl) ResumePod(podID string) (VCPod, error) {
return ResumePod(podID)
}
// CreateContainer implements the VC function of the same name.
func (impl *VCImpl) CreateContainer(podID string, containerConfig ContainerConfig) (VCPod, VCContainer, error) {
return CreateContainer(podID, containerConfig)
}
// DeleteContainer implements the VC function of the same name.
func (impl *VCImpl) DeleteContainer(podID, containerID string) (VCContainer, error) {
return DeleteContainer(podID, containerID)
}
// StartContainer implements the VC function of the same name.
func (impl *VCImpl) StartContainer(podID, containerID string) (VCContainer, error) {
return StartContainer(podID, containerID)
}
// StopContainer implements the VC function of the same name.
func (impl *VCImpl) StopContainer(podID, containerID string) (VCContainer, error) {
return StopContainer(podID, containerID)
}
// EnterContainer implements the VC function of the same name.
func (impl *VCImpl) EnterContainer(podID, containerID string, cmd Cmd) (VCPod, VCContainer, *Process, error) {
return EnterContainer(podID, containerID, cmd)
}
// StatusContainer implements the VC function of the same name.
func (impl *VCImpl) StatusContainer(podID, containerID string) (ContainerStatus, error) {
return StatusContainer(podID, containerID)
}
// KillContainer implements the VC function of the same name.
func (impl *VCImpl) KillContainer(podID, containerID string, signal syscall.Signal, all bool) error {
return KillContainer(podID, containerID, signal, all)
}
// ProcessListContainer implements the VC function of the same name.
func (impl *VCImpl) ProcessListContainer(podID, containerID string, options ProcessListOptions) (ProcessList, error) {
return ProcessListContainer(podID, containerID, options)
}

68
virtcontainers/interfaces.go Normal file
View File

@@ -0,0 +1,68 @@
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package virtcontainers
import (
"syscall"
"github.com/sirupsen/logrus"
)
// VC is the Virtcontainers interface
type VC interface {
SetLogger(logger logrus.FieldLogger)
CreatePod(podConfig PodConfig) (VCPod, error)
DeletePod(podID string) (VCPod, error)
ListPod() ([]PodStatus, error)
PausePod(podID string) (VCPod, error)
ResumePod(podID string) (VCPod, error)
RunPod(podConfig PodConfig) (VCPod, error)
StartPod(podID string) (VCPod, error)
StatusPod(podID string) (PodStatus, error)
StopPod(podID string) (VCPod, error)
CreateContainer(podID string, containerConfig ContainerConfig) (VCPod, VCContainer, error)
DeleteContainer(podID, containerID string) (VCContainer, error)
EnterContainer(podID, containerID string, cmd Cmd) (VCPod, VCContainer, *Process, error)
KillContainer(podID, containerID string, signal syscall.Signal, all bool) error
StartContainer(podID, containerID string) (VCContainer, error)
StatusContainer(podID, containerID string) (ContainerStatus, error)
StopContainer(podID, containerID string) (VCContainer, error)
ProcessListContainer(podID, containerID string, options ProcessListOptions) (ProcessList, error)
}
// VCPod is the Pod interface
// (required since virtcontainers.Pod only contains private fields)
type VCPod interface {
Annotations(key string) (string, error)
GetAllContainers() []VCContainer
GetAnnotations() map[string]string
GetContainer(containerID string) VCContainer
ID() string
SetAnnotations(annotations map[string]string) error
}
// VCContainer is the Container interface
// (required since virtcontainers.Container only contains private fields)
type VCContainer interface {
GetAnnotations() map[string]string
GetPid() int
GetToken() string
ID() string
Pod() VCPod
Process() Process
SetPid(pid int) error
}

833
virtcontainers/kata_agent.go Normal file
View File

@@ -0,0 +1,833 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
vcAnnotations "github.com/containers/virtcontainers/pkg/annotations"
ns "github.com/containers/virtcontainers/pkg/nsenter"
"github.com/containers/virtcontainers/pkg/uuid"
kataclient "github.com/kata-containers/agent/protocols/client"
"github.com/kata-containers/agent/protocols/grpc"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
var (
defaultKataSockPathTemplate = "%s/%s/kata.sock"
defaultKataChannel = "agent.channel.0"
defaultKataDeviceID = "channel0"
defaultKataID = "charch0"
errorMissingProxy = errors.New("Missing proxy pointer")
errorMissingOCISpec = errors.New("Missing OCI specification")
kataHostSharedDir = "/run/kata-containers/shared/pods/"
kataGuestSharedDir = "/run/kata-containers/shared/containers/"
mountGuest9pTag = "kataShared"
type9pFs = "9p"
devPath = "/dev"
vsockSocketScheme = "vsock"
kataBlkDevType = "blk"
)
// KataAgentConfig is a structure storing information needed
// to reach the Kata Containers agent.
type KataAgentConfig struct {
GRPCSocket string
}
type kataVSOCK struct {
contextID uint32
port uint32
}
func (s *kataVSOCK) String() string {
return fmt.Sprintf("%s://%d:%d", vsockSocketScheme, s.contextID, s.port)
}
// KataAgentState is the structure describing the data stored from this
// agent implementation.
type KataAgentState struct {
ProxyPid int
URL string
}
type kataAgent struct {
shim shim
proxy proxy
client *kataclient.AgentClient
state KataAgentState
vmSocket interface{}
}
func (k *kataAgent) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "kata_agent")
}
func parseVSOCKAddr(sock string) (uint32, uint32, error) {
sp := strings.Split(sock, ":")
if len(sp) != 3 {
return 0, 0, fmt.Errorf("Invalid vsock address: %s", sock)
}
if sp[0] != vsockSocketScheme {
return 0, 0, fmt.Errorf("Invalid vsock URL scheme: %s", sp[0])
}
cid, err := strconv.ParseUint(sp[1], 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("Invalid vsock cid: %s", sp[1])
}
port, err := strconv.ParseUint(sp[2], 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("Invalid vsock port: %s", sp[2])
}
return uint32(cid), uint32(port), nil
}
func (k *kataAgent) generateVMSocket(pod Pod, c KataAgentConfig) error {
cid, port, err := parseVSOCKAddr(c.GRPCSocket)
if err != nil {
// We need to generate a host UNIX socket path for the emulated serial port.
k.vmSocket = Socket{
DeviceID: defaultKataDeviceID,
ID: defaultKataID,
HostPath: fmt.Sprintf(defaultKataSockPathTemplate, runStoragePath, pod.id),
Name: defaultKataChannel,
}
} else {
// We want to go through VSOCK. The VM VSOCK endpoint will be our gRPC.
k.vmSocket = kataVSOCK{
contextID: cid,
port: port,
}
}
return nil
}
func (k *kataAgent) init(pod *Pod, config interface{}) (err error) {
switch c := config.(type) {
case KataAgentConfig:
if err := k.generateVMSocket(*pod, c); err != nil {
return err
}
default:
return fmt.Errorf("Invalid config type")
}
k.proxy, err = newProxy(pod.config.ProxyType)
if err != nil {
return err
}
k.shim, err = newShim(pod.config.ShimType)
if err != nil {
return err
}
// Fetch agent runtime info.
if err := pod.storage.fetchAgentState(pod.id, &k.state); err != nil {
k.Logger().Debug("Could not retrieve anything from storage")
}
return nil
}
func (k *kataAgent) agentURL() (string, error) {
switch s := k.vmSocket.(type) {
case Socket:
return s.HostPath, nil
case kataVSOCK:
return s.String(), nil
default:
return "", fmt.Errorf("Invalid socket type")
}
}
func (k *kataAgent) capabilities() capabilities {
var caps capabilities
// add all capabilities supported by agent
caps.setBlockDeviceSupport()
return caps
}
func (k *kataAgent) createPod(pod *Pod) error {
switch s := k.vmSocket.(type) {
case Socket:
err := pod.hypervisor.addDevice(s, serialPortDev)
if err != nil {
return err
}
case kataVSOCK:
// TODO Add an hypervisor vsock
default:
return fmt.Errorf("Invalid config type")
}
// Adding the shared volume.
// This volume contains all bind mounted container bundles.
sharedVolume := Volume{
MountTag: mountGuest9pTag,
HostPath: filepath.Join(kataHostSharedDir, pod.id),
}
if err := os.MkdirAll(sharedVolume.HostPath, dirMode); err != nil {
return err
}
return pod.hypervisor.addDevice(sharedVolume, fsDev)
}
func cmdToKataProcess(cmd Cmd) (process *grpc.Process, err error) {
var i uint64
var extraGids []uint32
// Number of bits used to store user+group values in
// the gRPC "User" type.
const grpcUserBits = 32
// User can contain only the "uid" or it can contain "uid:gid".
parsedUser := strings.Split(cmd.User, ":")
if len(parsedUser) > 2 {
return nil, fmt.Errorf("cmd.User %q format is wrong", cmd.User)
}
i, err = strconv.ParseUint(parsedUser[0], 10, grpcUserBits)
if err != nil {
return nil, err
}
uid := uint32(i)
var gid uint32
if len(parsedUser) > 1 {
i, err = strconv.ParseUint(parsedUser[1], 10, grpcUserBits)
if err != nil {
return nil, err
}
gid = uint32(i)
}
if cmd.PrimaryGroup != "" {
i, err = strconv.ParseUint(cmd.PrimaryGroup, 10, grpcUserBits)
if err != nil {
return nil, err
}
gid = uint32(i)
}
for _, g := range cmd.SupplementaryGroups {
var extraGid uint64
extraGid, err = strconv.ParseUint(g, 10, grpcUserBits)
if err != nil {
return nil, err
}
extraGids = append(extraGids, uint32(extraGid))
}
process = &grpc.Process{
Terminal: cmd.Interactive,
User: grpc.User{
UID: uid,
GID: gid,
AdditionalGids: extraGids,
},
Args: cmd.Args,
Env: cmdEnvsToStringSlice(cmd.Envs),
Cwd: cmd.WorkDir,
}
return process, nil
}
func cmdEnvsToStringSlice(ev []EnvVar) []string {
var env []string
for _, e := range ev {
pair := []string{e.Var, e.Value}
env = append(env, strings.Join(pair, "="))
}
return env
}
func (k *kataAgent) exec(pod *Pod, c Container, cmd Cmd) (*Process, error) {
var kataProcess *grpc.Process
kataProcess, err := cmdToKataProcess(cmd)
if err != nil {
return nil, err
}
req := &grpc.ExecProcessRequest{
ContainerId: c.id,
ExecId: uuid.Generate().String(),
Process: kataProcess,
}
if _, err := k.sendReq(req); err != nil {
return nil, err
}
enterNSList := []ns.Namespace{
{
PID: c.process.Pid,
Type: ns.NSTypeNet,
},
{
PID: c.process.Pid,
Type: ns.NSTypePID,
},
}
return prepareAndStartShim(pod, k.shim, c.id, req.ExecId,
k.state.URL, cmd, []ns.NSType{}, enterNSList)
}
func (k *kataAgent) generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*grpc.Interface, []*grpc.Route, error) {
if networkNS.NetNsPath == "" {
return nil, nil, nil
}
var routes []*grpc.Route
var ifaces []*grpc.Interface
for _, endpoint := range networkNS.Endpoints {
var ipAddresses []*grpc.IPAddress
for _, addr := range endpoint.Properties().Addrs {
// Skip IPv6 because not supported
if addr.IP.To4() == nil {
// Skip IPv6 because not supported
k.Logger().WithFields(logrus.Fields{
"unsupported-address-type": "ipv6",
"address": addr,
}).Warn("unsupported address")
continue
}
// Skip localhost interface
if addr.IP.IsLoopback() {
continue
}
netMask, _ := addr.Mask.Size()
ipAddress := grpc.IPAddress{
Family: grpc.IPFamily_v4,
Address: addr.IP.String(),
Mask: fmt.Sprintf("%d", netMask),
}
ipAddresses = append(ipAddresses, &ipAddress)
}
ifc := grpc.Interface{
IPAddresses: ipAddresses,
Device: endpoint.Name(),
Name: endpoint.Name(),
Mtu: uint64(endpoint.Properties().Iface.MTU),
HwAddr: endpoint.HardwareAddr(),
}
ifaces = append(ifaces, &ifc)
for _, route := range endpoint.Properties().Routes {
var r grpc.Route
if route.Dst != nil {
r.Dest = route.Dst.String()
if route.Dst.IP.To4() == nil {
// Skip IPv6 because not supported
k.Logger().WithFields(logrus.Fields{
"unsupported-route-type": "ipv6",
"destination": r.Dest,
}).Warn("unsupported route")
continue
}
}
if route.Gw != nil {
r.Gateway = route.Gw.String()
}
if route.Src != nil {
r.Source = route.Src.String()
}
r.Device = endpoint.Name()
r.Scope = uint32(route.Scope)
routes = append(routes, &r)
}
}
return ifaces, routes, nil
}
func (k *kataAgent) startPod(pod Pod) error {
if k.proxy == nil {
return errorMissingProxy
}
// Get agent socket path to provide it to the proxy.
agentURL, err := k.agentURL()
if err != nil {
return err
}
proxyParams := proxyParams{
agentURL: agentURL,
}
// Start the proxy here
pid, uri, err := k.proxy.start(pod, proxyParams)
if err != nil {
return err
}
// Fill agent state with proxy information, and store them.
k.state.ProxyPid = pid
k.state.URL = uri
if err := pod.storage.storeAgentState(pod.id, k.state); err != nil {
return err
}
k.Logger().WithField("proxy-pid", pid).Info("proxy started")
hostname := pod.config.Hostname
if len(hostname) > maxHostnameLen {
hostname = hostname[:maxHostnameLen]
}
//
// Setup network interfaces and routes
//
interfaces, routes, err := k.generateInterfacesAndRoutes(pod.networkNS)
if err != nil {
return err
}
for _, ifc := range interfaces {
// send update interface request
ifcReq := &grpc.UpdateInterfaceRequest{
Interface: ifc,
}
resultingInterface, err := k.sendReq(ifcReq)
if err != nil {
k.Logger().WithFields(logrus.Fields{
"interface-requested": fmt.Sprintf("%+v", ifc),
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
}).WithError(err).Error("update interface request failed")
return err
}
}
if routes != nil {
routesReq := &grpc.UpdateRoutesRequest{
Routes: &grpc.Routes{
Routes: routes,
},
}
resultingRoutes, err := k.sendReq(routesReq)
if err != nil {
k.Logger().WithFields(logrus.Fields{
"routes-requested": fmt.Sprintf("%+v", routes),
"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
}).WithError(err).Error("update routes request failed")
return err
}
}
// We mount the shared directory in a predefined location
// in the guest.
// This is where at least some of the host config files
// (resolv.conf, etc...) and potentially all container
// rootfs will reside.
sharedVolume := &grpc.Storage{
Source: mountGuest9pTag,
MountPoint: kataGuestSharedDir,
Fstype: type9pFs,
Options: []string{"trans=virtio", "nodev"},
}
req := &grpc.CreateSandboxRequest{
Hostname: hostname,
Storages: []*grpc.Storage{sharedVolume},
SandboxPidns: false,
}
_, err = k.sendReq(req)
return err
}
func (k *kataAgent) stopPod(pod Pod) error {
if k.proxy == nil {
return errorMissingProxy
}
req := &grpc.DestroySandboxRequest{}
if _, err := k.sendReq(req); err != nil {
return err
}
return k.proxy.stop(pod, k.state.ProxyPid)
}
func appendStorageFromMounts(storage []*grpc.Storage, mounts []*Mount) []*grpc.Storage {
for _, m := range mounts {
s := &grpc.Storage{
Source: m.Source,
MountPoint: m.Destination,
Fstype: m.Type,
Options: m.Options,
}
storage = append(storage, s)
}
return storage
}
func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts []Mount) error {
ociMounts := spec.Mounts
for index, m := range ociMounts {
for _, guestMount := range guestMounts {
if guestMount.Destination != m.Destination {
continue
}
k.Logger().Debugf("Replacing OCI mount (%s) source %s with %s", m.Destination, m.Source, guestMount.Source)
ociMounts[index].Source = guestMount.Source
}
}
return nil
}
func constraintGRPCSpec(grpcSpec *grpc.Spec) {
// Disable Hooks since they have been handled on the host and there is
// no reason to send them to the agent. It would make no sense to try
// to apply them on the guest.
grpcSpec.Hooks = nil
// Disable Seccomp since they cannot be handled properly by the agent
// until we provide a guest image with libseccomp support. More details
// here: https://github.com/kata-containers/agent/issues/104
grpcSpec.Linux.Seccomp = nil
// TODO: Remove this constraint as soon as the agent properly handles
// resources provided through the specification.
grpcSpec.Linux.Resources = nil
// Disable network namespace since it is already handled on the host by
// virtcontainers. The network is a complex part which cannot be simply
// passed to the agent.
// Every other namespaces's paths have to be emptied. This way, there
// is no confusion from the agent, trying to find an existing namespace
// on the guest.
var tmpNamespaces []grpc.LinuxNamespace
for _, ns := range grpcSpec.Linux.Namespaces {
switch ns.Type {
case specs.NetworkNamespace:
default:
ns.Path = ""
tmpNamespaces = append(tmpNamespaces, ns)
}
}
grpcSpec.Linux.Namespaces = tmpNamespaces
// Handle /dev/shm mount
for idx, mnt := range grpcSpec.Mounts {
if mnt.Destination == "/dev/shm" {
grpcSpec.Mounts[idx].Type = "tmpfs"
grpcSpec.Mounts[idx].Source = "shm"
grpcSpec.Mounts[idx].Options = []string{"noexec", "nosuid", "nodev", "mode=1777", "size=65536k"}
break
}
}
}
func (k *kataAgent) createContainer(pod *Pod, c *Container) (*Process, error) {
ociSpecJSON, ok := c.config.Annotations[vcAnnotations.ConfigJSONKey]
if !ok {
return nil, errorMissingOCISpec
}
var ctrStorages []*grpc.Storage
var ctrDevices []*grpc.Device
// The rootfs storage volume represents the container rootfs
// mount point inside the guest.
// It can be a block based device (when using block based container
// overlay on the host) mount or a 9pfs one (for all other overlay
// implementations).
rootfs := &grpc.Storage{}
// This is the guest absolute root path for that container.
rootPathParent := filepath.Join(kataGuestSharedDir, c.id)
rootPath := filepath.Join(rootPathParent, rootfsDir)
if c.state.Fstype != "" {
// This is a block based device rootfs.
// driveName is the predicted virtio-block guest name (the vd* in /dev/vd*).
driveName, err := getVirtDriveName(c.state.BlockIndex)
if err != nil {
return nil, err
}
virtPath := filepath.Join(devPath, driveName)
// Create a new device with empty ContainerPath so that we get
// the device being waited for by the agent inside the VM,
// without trying to match and update it into the OCI spec list
// of actual devices. The device corresponding to the rootfs is
// a very specific case.
rootfsDevice := &grpc.Device{
Type: kataBlkDevType,
VmPath: virtPath,
ContainerPath: "",
}
ctrDevices = append(ctrDevices, rootfsDevice)
rootfs.Source = virtPath
rootfs.MountPoint = rootPathParent
rootfs.Fstype = c.state.Fstype
if c.state.Fstype == "xfs" {
rootfs.Options = []string{"nouuid"}
}
// Add rootfs to the list of container storage.
// We only need to do this for block based rootfs, as we
// want the agent to mount it into the right location
// (kataGuestSharedDir/ctrID/
ctrStorages = append(ctrStorages, rootfs)
} else {
// This is not a block based device rootfs.
// We are going to bind mount it into the 9pfs
// shared drive between the host and the guest.
// With 9pfs we don't need to ask the agent to
// mount the rootfs as the shared directory
// (kataGuestSharedDir) is already mounted in the
// guest. We only need to mount the rootfs from
// the host and it will show up in the guest.
if err := bindMountContainerRootfs(kataHostSharedDir, pod.id, c.id, c.rootFs, false); err != nil {
bindUnmountAllRootfs(kataHostSharedDir, *pod)
return nil, err
}
}
ociSpec := &specs.Spec{}
if err := json.Unmarshal([]byte(ociSpecJSON), ociSpec); err != nil {
return nil, err
}
// Handle container mounts
newMounts, err := c.mountSharedDirMounts(kataHostSharedDir, kataGuestSharedDir)
if err != nil {
bindUnmountAllRootfs(kataHostSharedDir, *pod)
return nil, err
}
// We replace all OCI mount sources that match our container mount
// with the right source path (The guest one).
if err := k.replaceOCIMountSource(ociSpec, newMounts); err != nil {
return nil, err
}
grpcSpec, err := grpc.OCItoGRPC(ociSpec)
if err != nil {
return nil, err
}
// We need to give the OCI spec our absolute rootfs path in the guest.
grpcSpec.Root.Path = rootPath
// We need to constraint the spec to make sure we're not passing
// irrelevant information to the agent.
constraintGRPCSpec(grpcSpec)
// Append container mounts for block devices passed with --device.
for _, device := range c.devices {
d, ok := device.(*BlockDevice)
if !ok {
continue
}
kataDevice := &grpc.Device{
Type: kataBlkDevType,
VmPath: d.VirtPath,
ContainerPath: d.DeviceInfo.ContainerPath,
}
ctrDevices = append(ctrDevices, kataDevice)
}
req := &grpc.CreateContainerRequest{
ContainerId: c.id,
ExecId: c.id,
Storages: ctrStorages,
Devices: ctrDevices,
OCI: grpcSpec,
}
if _, err := k.sendReq(req); err != nil {
return nil, err
}
createNSList := []ns.NSType{ns.NSTypePID}
enterNSList := []ns.Namespace{
{
Path: pod.networkNS.NetNsPath,
Type: ns.NSTypeNet,
},
}
return prepareAndStartShim(pod, k.shim, c.id, req.ExecId,
k.state.URL, c.config.Cmd, createNSList, enterNSList)
}
func (k *kataAgent) startContainer(pod Pod, c *Container) error {
req := &grpc.StartContainerRequest{
ContainerId: c.id,
}
_, err := k.sendReq(req)
return err
}
func (k *kataAgent) stopContainer(pod Pod, c Container) error {
req := &grpc.RemoveContainerRequest{
ContainerId: c.id,
}
if _, err := k.sendReq(req); err != nil {
return err
}
if err := c.unmountHostMounts(); err != nil {
return err
}
if err := bindUnmountContainerRootfs(kataHostSharedDir, pod.id, c.id); err != nil {
return err
}
return nil
}
func (k *kataAgent) killContainer(pod Pod, c Container, signal syscall.Signal, all bool) error {
req := &grpc.SignalProcessRequest{
ContainerId: c.id,
ExecId: c.process.Token,
Signal: uint32(signal),
}
_, err := k.sendReq(req)
return err
}
func (k *kataAgent) processListContainer(pod Pod, c Container, options ProcessListOptions) (ProcessList, error) {
return nil, nil
}
func (k *kataAgent) connect() error {
if k.client != nil {
return nil
}
client, err := kataclient.NewAgentClient(k.state.URL)
if err != nil {
return err
}
k.client = client
return nil
}
func (k *kataAgent) disconnect() error {
if k.client == nil {
return nil
}
if err := k.client.Close(); err != nil {
return err
}
k.client = nil
return nil
}
func (k *kataAgent) sendReq(request interface{}) (interface{}, error) {
if err := k.connect(); err != nil {
return nil, err
}
defer k.disconnect()
switch req := request.(type) {
case *grpc.ExecProcessRequest:
_, err := k.client.ExecProcess(context.Background(), req)
return nil, err
case *grpc.CreateSandboxRequest:
_, err := k.client.CreateSandbox(context.Background(), req)
return nil, err
case *grpc.DestroySandboxRequest:
_, err := k.client.DestroySandbox(context.Background(), req)
return nil, err
case *grpc.CreateContainerRequest:
_, err := k.client.CreateContainer(context.Background(), req)
return nil, err
case *grpc.StartContainerRequest:
_, err := k.client.StartContainer(context.Background(), req)
return nil, err
case *grpc.RemoveContainerRequest:
_, err := k.client.RemoveContainer(context.Background(), req)
return nil, err
case *grpc.SignalProcessRequest:
_, err := k.client.SignalProcess(context.Background(), req)
return nil, err
case *grpc.UpdateRoutesRequest:
_, err := k.client.UpdateRoutes(context.Background(), req)
return nil, err
case *grpc.UpdateInterfaceRequest:
ifc, err := k.client.UpdateInterface(context.Background(), req)
return ifc, err
default:
return nil, fmt.Errorf("Unknown gRPC type %T", req)
}
}

338
virtcontainers/kata_agent_test.go Normal file
View File

@@ -0,0 +1,338 @@
//
// Copyright (c) 2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io/ioutil"
"net"
"os"
"reflect"
"testing"
"github.com/containers/virtcontainers/pkg/mock"
gpb "github.com/gogo/protobuf/types"
pb "github.com/kata-containers/agent/protocols/grpc"
"github.com/stretchr/testify/assert"
"github.com/vishvananda/netlink"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
var testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock"
func proxyHandlerDiscard(c net.Conn) {
buf := make([]byte, 1024)
c.Read(buf)
}
func testGenerateKataProxySockDir() (string, error) {
dir, err := ioutil.TempDir("", "kata-proxy-test")
if err != nil {
return "", err
}
return dir, nil
}
func TestKataAgentConnect(t *testing.T) {
proxy := mock.ProxyUnixMock{
ClientHandler: proxyHandlerDiscard,
}
sockDir, err := testGenerateKataProxySockDir()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(sockDir)
testKataProxyURL := fmt.Sprintf(testKataProxyURLTempl, sockDir)
if err := proxy.Start(testKataProxyURL); err != nil {
t.Fatal(err)
}
defer proxy.Stop()
k := &kataAgent{
state: KataAgentState{
URL: testKataProxyURL,
},
}
if err := k.connect(); err != nil {
t.Fatal(err)
}
if k.client == nil {
t.Fatal("Kata agent client is not properly initialized")
}
}
func TestKataAgentDisconnect(t *testing.T) {
proxy := mock.ProxyUnixMock{
ClientHandler: proxyHandlerDiscard,
}
sockDir, err := testGenerateKataProxySockDir()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(sockDir)
testKataProxyURL := fmt.Sprintf(testKataProxyURLTempl, sockDir)
if err := proxy.Start(testKataProxyURL); err != nil {
t.Fatal(err)
}
defer proxy.Stop()
k := &kataAgent{
state: KataAgentState{
URL: testKataProxyURL,
},
}
if err := k.connect(); err != nil {
t.Fatal(err)
}
if err := k.disconnect(); err != nil {
t.Fatal(err)
}
if k.client != nil {
t.Fatal("Kata agent client pointer should be nil")
}
}
type gRPCProxy struct{}
var emptyResp = &gpb.Empty{}
func (p *gRPCProxy) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) ExecProcess(ctx context.Context, req *pb.ExecProcessRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) SignalProcess(ctx context.Context, req *pb.SignalProcessRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest) (*pb.WaitProcessResponse, error) {
return &pb.WaitProcessResponse{}, nil
}
func (p *gRPCProxy) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) WriteStdin(ctx context.Context, req *pb.WriteStreamRequest) (*pb.WriteStreamResponse, error) {
return &pb.WriteStreamResponse{}, nil
}
func (p *gRPCProxy) ReadStdout(ctx context.Context, req *pb.ReadStreamRequest) (*pb.ReadStreamResponse, error) {
return &pb.ReadStreamResponse{}, nil
}
func (p *gRPCProxy) ReadStderr(ctx context.Context, req *pb.ReadStreamRequest) (*pb.ReadStreamResponse, error) {
return &pb.ReadStreamResponse{}, nil
}
func (p *gRPCProxy) CloseStdin(ctx context.Context, req *pb.CloseStdinRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) TtyWinResize(ctx context.Context, req *pb.TtyWinResizeRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) DestroySandbox(ctx context.Context, req *pb.DestroySandboxRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func (p *gRPCProxy) AddInterface(ctx context.Context, req *pb.AddInterfaceRequest) (*pb.Interface, error) {
return nil, nil
}
func (p *gRPCProxy) RemoveInterface(ctx context.Context, req *pb.RemoveInterfaceRequest) (*pb.Interface, error) {
return nil, nil
}
func (p *gRPCProxy) UpdateInterface(ctx context.Context, req *pb.UpdateInterfaceRequest) (*pb.Interface, error) {
return nil, nil
}
func (p *gRPCProxy) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
return nil, nil
}
func (p *gRPCProxy) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {
return emptyResp, nil
}
func gRPCRegister(s *grpc.Server, srv interface{}) {
switch g := srv.(type) {
case *gRPCProxy:
pb.RegisterAgentServiceServer(s, g)
}
}
var reqList = []interface{}{
&pb.CreateSandboxRequest{},
&pb.DestroySandboxRequest{},
&pb.ExecProcessRequest{},
&pb.CreateContainerRequest{},
&pb.StartContainerRequest{},
&pb.RemoveContainerRequest{},
&pb.SignalProcessRequest{},
}
func TestKataAgentSendReq(t *testing.T) {
impl := &gRPCProxy{}
proxy := mock.ProxyGRPCMock{
GRPCImplementer: impl,
GRPCRegister: gRPCRegister,
}
sockDir, err := testGenerateKataProxySockDir()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(sockDir)
testKataProxyURL := fmt.Sprintf(testKataProxyURLTempl, sockDir)
if err := proxy.Start(testKataProxyURL); err != nil {
t.Fatal(err)
}
defer proxy.Stop()
k := &kataAgent{
state: KataAgentState{
URL: testKataProxyURL,
},
}
for _, req := range reqList {
if _, err := k.sendReq(req); err != nil {
t.Fatal(err)
}
}
}
func TestGenerateInterfacesAndRoutes(t *testing.T) {
impl := &gRPCProxy{}
proxy := mock.ProxyGRPCMock{
GRPCImplementer: impl,
GRPCRegister: gRPCRegister,
}
sockDir, err := testGenerateKataProxySockDir()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(sockDir)
testKataProxyURL := fmt.Sprintf(testKataProxyURLTempl, sockDir)
if err := proxy.Start(testKataProxyURL); err != nil {
t.Fatal(err)
}
defer proxy.Stop()
k := &kataAgent{
state: KataAgentState{
URL: testKataProxyURL,
},
}
//
//Create a couple of addresses
//
address1 := &net.IPNet{IP: net.IPv4(172, 17, 0, 2), Mask: net.CIDRMask(16, 32)}
address2 := &net.IPNet{IP: net.IPv4(182, 17, 0, 2), Mask: net.CIDRMask(16, 32)}
addrs := []netlink.Addr{
{IPNet: address1, Label: "phyaddr1"},
{IPNet: address2, Label: "phyaddr2"},
}
// Create a couple of routes:
dst2 := &net.IPNet{IP: net.IPv4(172, 17, 0, 0), Mask: net.CIDRMask(16, 32)}
src2 := net.IPv4(172, 17, 0, 2)
gw2 := net.IPv4(172, 17, 0, 1)
routes := []netlink.Route{
{LinkIndex: 329, Dst: nil, Src: nil, Gw: net.IPv4(172, 17, 0, 1), Scope: netlink.Scope(254)},
{LinkIndex: 329, Dst: dst2, Src: src2, Gw: gw2},
}
networkInfo := NetworkInfo{
Iface: NetlinkIface{
LinkAttrs: netlink.LinkAttrs{MTU: 1500},
Type: "",
},
Addrs: addrs,
Routes: routes,
}
ep0 := &PhysicalEndpoint{
IfaceName: "eth0",
HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(),
EndpointProperties: networkInfo,
}
endpoints := []Endpoint{ep0}
nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints}
resInterfaces, resRoutes, err := k.generateInterfacesAndRoutes(nns)
//
// Build expected results:
//
expectedAddresses := []*pb.IPAddress{
{Family: 0, Address: "172.17.0.2", Mask: "16"},
{Family: 0, Address: "182.17.0.2", Mask: "16"},
}
expectedInterfaces := []*pb.Interface{
{Device: "eth0", Name: "eth0", IPAddresses: expectedAddresses, Mtu: 1500, HwAddr: "02:00:ca:fe:00:04"},
}
expectedRoutes := []*pb.Route{
{Dest: "", Gateway: "172.17.0.1", Device: "eth0", Source: "", Scope: uint32(254)},
{Dest: "172.17.0.0/16", Gateway: "172.17.0.1", Device: "eth0", Source: "172.17.0.2"},
}
assert.Nil(t, err, "unexpected failure when calling generateKataInterfacesAndRoutes")
assert.True(t, reflect.DeepEqual(resInterfaces, expectedInterfaces),
"Interfaces returned didn't match: got %+v, expecting %+v", resInterfaces, expectedInterfaces)
assert.True(t, reflect.DeepEqual(resRoutes, expectedRoutes),
"Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes)
}

70
virtcontainers/kata_proxy.go Normal file
View File

@@ -0,0 +1,70 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"os/exec"
"syscall"
)
// This is the Kata Containers implementation of the proxy interface.
// This is pretty simple since it provides the same interface to both
// runtime and shim as if they were talking directly to the agent.
type kataProxy struct {
}
// start is kataProxy start implementation for proxy interface.
func (p *kataProxy) start(pod Pod, params proxyParams) (int, string, error) {
if pod.agent == nil {
return -1, "", fmt.Errorf("No agent")
}
if params.agentURL == "" {
return -1, "", fmt.Errorf("AgentURL cannot be empty")
}
config, err := newProxyConfig(pod.config)
if err != nil {
return -1, "", err
}
// construct the socket path the proxy instance will use
proxyURL, err := defaultProxyURL(pod, SocketTypeUNIX)
if err != nil {
return -1, "", err
}
args := []string{config.Path, "-listen-socket", proxyURL, "-mux-socket", params.agentURL}
if config.Debug {
args = append(args, "-log", "debug")
args = append(args, "-agent-logs-socket", pod.hypervisor.getPodConsole(pod.id))
}
cmd := exec.Command(args[0], args[1:]...)
if err := cmd.Start(); err != nil {
return -1, "", err
}
return cmd.Process.Pid, proxyURL, nil
}
// stop is kataProxy stop implementation for proxy interface.
func (p *kataProxy) stop(pod Pod, pid int) error {
// Signal the proxy with SIGTERM.
return syscall.Kill(pid, syscall.SIGTERM)
}

72
virtcontainers/kata_shim.go Normal file
View File

@@ -0,0 +1,72 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
)
type kataShim struct{}
// KataShimConfig is the structure providing specific configuration
// for kataShim implementation.
type KataShimConfig struct {
Path string
Debug bool
}
// start is the ccShim start implementation.
// It starts the cc-shim binary with URL and token flags provided by
// the proxy.
func (s *kataShim) start(pod Pod, params ShimParams) (int, error) {
if pod.config == nil {
return -1, fmt.Errorf("Pod config cannot be nil")
}
config, ok := newShimConfig(*(pod.config)).(ShimConfig)
if !ok {
return -1, fmt.Errorf("Wrong shim config type, should be KataShimConfig type")
}
if config.Path == "" {
return -1, fmt.Errorf("Shim path cannot be empty")
}
if params.URL == "" {
return -1, fmt.Errorf("URL cannot be empty")
}
if params.Container == "" {
return -1, fmt.Errorf("Container cannot be empty")
}
if params.Token == "" {
return -1, fmt.Errorf("Process token cannot be empty")
}
args := []string{config.Path, "-agent", params.URL, "-container", params.Container, "-exec-id", params.Token}
if params.Terminal {
args = append(args, "-terminal")
}
if config.Debug {
args = append(args, "-log", "debug")
}
return startShim(args, params)
}

316
virtcontainers/kata_shim_test.go Normal file
View File

@@ -0,0 +1,316 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
. "github.com/containers/virtcontainers/pkg/mock"
)
// These tests don't care about the format of the container ID
const testKataContainer = "testContainer"
var testKataShimPath = "/usr/bin/virtcontainers/bin/test/kata-shim"
var testKataShimProxyURL = "foo:///foo/kata-containers/proxy.sock"
func getMockKataShimBinPath() string {
if DefaultMockKataShimBinPath == "" {
return testKataShimPath
}
return DefaultMockKataShimBinPath
}
func testKataShimStart(t *testing.T, pod Pod, params ShimParams, expectFail bool) {
s := &kataShim{}
pid, err := s.start(pod, params)
if expectFail {
if err == nil || pid != -1 {
t.Fatalf("This test should fail (pod %+v, params %+v, expectFail %t)",
pod, params, expectFail)
}
} else {
if err != nil {
t.Fatalf("This test should pass (pod %+v, params %+v, expectFail %t): %s",
pod, params, expectFail, err)
}
if pid == -1 {
t.Fatalf("This test should pass (pod %+v, params %+v, expectFail %t)",
pod, params, expectFail)
}
}
}
func TestKataShimStartNilPodConfigFailure(t *testing.T) {
testKataShimStart(t, Pod{}, ShimParams{}, true)
}
func TestKataShimStartNilShimConfigFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{},
}
testKataShimStart(t, pod, ShimParams{}, true)
}
func TestKataShimStartShimPathEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{},
},
}
testKataShimStart(t, pod, ShimParams{}, true)
}
func TestKataShimStartShimTypeInvalid(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: "foo",
ShimConfig: ShimConfig{},
},
}
testKataShimStart(t, pod, ShimParams{}, true)
}
func TestKataShimStartParamsTokenEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: getMockKataShimBinPath(),
},
},
}
testKataShimStart(t, pod, ShimParams{}, true)
}
func TestKataShimStartParamsURLEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: getMockKataShimBinPath(),
},
},
}
params := ShimParams{
Token: "testToken",
}
testKataShimStart(t, pod, params, true)
}
func TestKataShimStartParamsContainerEmptyFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: getMockKataShimBinPath(),
},
},
}
params := ShimParams{
Token: "testToken",
URL: "unix://is/awesome",
}
testKataShimStart(t, pod, params, true)
}
func TestKataShimStartParamsInvalidCommand(t *testing.T) {
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
cmd := filepath.Join(dir, "does-not-exist")
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: cmd,
},
},
}
params := ShimParams{
Token: "testToken",
URL: "http://foo",
}
testKataShimStart(t, pod, params, true)
}
func startKataShimStartWithoutConsoleSuccessful(t *testing.T, detach bool) (*os.File, *os.File, *os.File, Pod, ShimParams, error) {
saveStdout := os.Stdout
rStdout, wStdout, err := os.Pipe()
if err != nil {
return nil, nil, nil, Pod{}, ShimParams{}, err
}
os.Stdout = wStdout
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: getMockKataShimBinPath(),
},
},
}
params := ShimParams{
Container: testContainer,
Token: "testToken",
URL: testKataShimProxyURL,
Detach: detach,
}
return rStdout, wStdout, saveStdout, pod, params, nil
}
func TestKataShimStartSuccessful(t *testing.T) {
rStdout, wStdout, saveStdout, pod, params, err := startKataShimStartWithoutConsoleSuccessful(t, false)
if err != nil {
t.Fatal(err)
}
defer func() {
os.Stdout = saveStdout
rStdout.Close()
wStdout.Close()
}()
testKataShimStart(t, pod, params, false)
bufStdout := make([]byte, 1024)
if _, err := rStdout.Read(bufStdout); err != nil {
t.Fatal(err)
}
if !strings.Contains(string(bufStdout), ShimStdoutOutput) {
t.Fatalf("Substring %q not found in %q", ShimStdoutOutput, string(bufStdout))
}
}
func TestKataShimStartDetachSuccessful(t *testing.T) {
rStdout, wStdout, saveStdout, pod, params, err := startKataShimStartWithoutConsoleSuccessful(t, true)
if err != nil {
t.Fatal(err)
}
defer func() {
os.Stdout = saveStdout
wStdout.Close()
rStdout.Close()
}()
testKataShimStart(t, pod, params, false)
readCh := make(chan error)
go func() {
bufStdout := make([]byte, 1024)
n, err := rStdout.Read(bufStdout)
if err != nil && err != io.EOF {
readCh <- err
return
}
if n > 0 {
readCh <- fmt.Errorf("Not expecting to read anything, Got %q", string(bufStdout))
return
}
readCh <- nil
}()
select {
case err := <-readCh:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Duration(20) * time.Millisecond):
return
}
}
func TestKataShimStartWithConsoleNonExistingFailure(t *testing.T) {
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: getMockKataShimBinPath(),
},
},
}
params := ShimParams{
Token: "testToken",
URL: testKataShimProxyURL,
Console: testWrongConsolePath,
}
testKataShimStart(t, pod, params, true)
}
func TestKataShimStartWithConsoleSuccessful(t *testing.T) {
cleanUp()
master, console, err := newConsole()
t.Logf("Console created for tests:%s\n", console)
if err != nil {
t.Fatal(err)
}
pod := Pod{
config: &PodConfig{
ShimType: KataShimType,
ShimConfig: ShimConfig{
Path: getMockKataShimBinPath(),
},
},
}
params := ShimParams{
Container: testContainer,
Token: "testToken",
URL: testKataShimProxyURL,
Console: console,
}
testKataShimStart(t, pod, params, false)
master.Close()
}

73
virtcontainers/mock_hypervisor.go Normal file
View File

@@ -0,0 +1,73 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
type mockHypervisor struct {
}
func (m *mockHypervisor) init(pod *Pod) error {
valid, err := pod.config.HypervisorConfig.valid()
if valid == false || err != nil {
return err
}
return nil
}
func (m *mockHypervisor) capabilities() capabilities {
return capabilities{}
}
func (m *mockHypervisor) createPod(podConfig PodConfig) error {
return nil
}
func (m *mockHypervisor) startPod() error {
return nil
}
func (m *mockHypervisor) waitPod(timeout int) error {
return nil
}
func (m *mockHypervisor) stopPod() error {
return nil
}
func (m *mockHypervisor) pausePod() error {
return nil
}
func (m *mockHypervisor) resumePod() error {
return nil
}
func (m *mockHypervisor) addDevice(devInfo interface{}, devType deviceType) error {
return nil
}
func (m *mockHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) error {
return nil
}
func (m *mockHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) error {
return nil
}
func (m *mockHypervisor) getPodConsole(podID string) string {
return ""
}

104
virtcontainers/mock_hypervisor_test.go Normal file
View File

@@ -0,0 +1,104 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
"testing"
)
func TestMockHypervisorInit(t *testing.T) {
var m *mockHypervisor
pod := &Pod{
config: &PodConfig{
HypervisorConfig: HypervisorConfig{
KernelPath: "",
ImagePath: "",
HypervisorPath: "",
},
},
}
// wrong config
if err := m.init(pod); err == nil {
t.Fatal()
}
pod.config.HypervisorConfig = HypervisorConfig{
KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel),
ImagePath: fmt.Sprintf("%s/%s", testDir, testImage),
HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
}
// right config
if err := m.init(pod); err != nil {
t.Fatal(err)
}
}
func TestMockHypervisorCreatePod(t *testing.T) {
var m *mockHypervisor
config := PodConfig{}
if err := m.createPod(config); err != nil {
t.Fatal(err)
}
}
func TestMockHypervisorStartPod(t *testing.T) {
var m *mockHypervisor
if err := m.startPod(); err != nil {
t.Fatal(err)
}
}
func TestMockHypervisorWaitPod(t *testing.T) {
var m *mockHypervisor
if err := m.waitPod(0); err != nil {
t.Fatal(err)
}
}
func TestMockHypervisorStopPod(t *testing.T) {
var m *mockHypervisor
if err := m.stopPod(); err != nil {
t.Fatal(err)
}
}
func TestMockHypervisorAddDevice(t *testing.T) {
var m *mockHypervisor
if err := m.addDevice(nil, imgDev); err != nil {
t.Fatal(err)
}
}
func TestMockHypervisorGetPodConsole(t *testing.T) {
var m *mockHypervisor
expected := ""
if result := m.getPodConsole("testPodID"); result != expected {
t.Fatalf("Got %s\nExpecting %s", result, expected)
}
}

350
virtcontainers/mount.go Normal file
View File

@@ -0,0 +1,350 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/sirupsen/logrus"
)
var rootfsDir = "rootfs"
func mountLogger() *logrus.Entry {
return virtLog.WithField("subsystem", "mount")
}
// These mounts need to be created by the agent within the VM
var systemMounts = []string{"/proc", "/dev", "/dev/pts", "/dev/shm", "/dev/mqueue", "/sys", "/sys/fs/cgroup"}
var systemMountPrefixes = []string{"/proc", "/dev", "/sys"}
func isSystemMount(m string) bool {
for _, p := range systemMountPrefixes {
if m == p || strings.HasPrefix(m, p+"/") {
return true
}
}
return false
}
func major(dev uint64) int {
return int((dev >> 8) & 0xfff)
}
func minor(dev uint64) int {
return int((dev & 0xff) | ((dev >> 12) & 0xfff00))
}
type device struct {
major int
minor int
mountPoint string
}
var errMountPointNotFound = errors.New("Mount point not found")
// getDeviceForPath gets the underlying device containing the file specified by path.
// The device type constitutes the major-minor number of the device and the dest mountPoint for the device
//
// eg. if /dev/sda1 is mounted on /a/b/c, a call to getDeviceForPath("/a/b/c/file") would return
//
// device {
// major : major(/dev/sda1)
// manor : minor(/dev/sda1)
// mountPoint: /a/b/c
// }
func getDeviceForPath(path string) (device, error) {
if path == "" {
return device{}, fmt.Errorf("Path cannot be empty")
}
stat := syscall.Stat_t{}
err := syscall.Stat(path, &stat)
if err != nil {
return device{}, err
}
// stat.Dev points to the underlying device containing the file
major := major(stat.Dev)
minor := minor(stat.Dev)
path, err = filepath.Abs(path)
if err != nil {
return device{}, err
}
mountPoint := path
if path == "/" {
return device{
major: major,
minor: minor,
mountPoint: mountPoint,
}, nil
}
// We get the mount point by recursively peforming stat on the path
// The point where the device changes indicates the mountpoint
for {
if mountPoint == "/" {
return device{}, errMountPointNotFound
}
parentStat := syscall.Stat_t{}
parentDir := filepath.Dir(path)
err := syscall.Lstat(parentDir, &parentStat)
if err != nil {
return device{}, err
}
if parentStat.Dev != stat.Dev {
break
}
mountPoint = parentDir
stat = parentStat
path = parentDir
}
dev := device{
major: major,
minor: minor,
mountPoint: mountPoint,
}
return dev, nil
}
const (
procMountsFile = "/proc/mounts"
fieldsPerLine = 6
)
const (
procDeviceIndex = iota
procPathIndex
procTypeIndex
)
func getDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) {
if mountPoint == "" {
err = fmt.Errorf("Mount point cannot be empty")
return
}
var file *os.File
file, err = os.Open(procMountsFile)
if err != nil {
return
}
defer file.Close()
reader := bufio.NewReader(file)
for {
var line string
line, err = reader.ReadString('\n')
if err == io.EOF {
err = fmt.Errorf("Mount %s not found", mountPoint)
return
}
fields := strings.Fields(line)
if len(fields) != fieldsPerLine {
err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
return
}
if mountPoint == fields[procPathIndex] {
devicePath = fields[procDeviceIndex]
fsType = fields[procTypeIndex]
return
}
}
}
var blockFormatTemplate = "/sys/dev/block/%d:%d/dm"
var checkStorageDriver = isDeviceMapper
// isDeviceMapper checks if the device with the major and minor numbers is a devicemapper block device
func isDeviceMapper(major, minor int) (bool, error) {
//Check if /sys/dev/block/${major}-${minor}/dm exists
sysPath := fmt.Sprintf(blockFormatTemplate, major, minor)
_, err := os.Stat(sysPath)
if err == nil {
return true, nil
} else if os.IsNotExist(err) {
return false, nil
}
return false, err
}
// getVirtBlockDriveName returns the disk name format for virtio-blk
// Reference: https://github.com/torvalds/linux/blob/master/drivers/block/virtio_blk.c @c0aa3e0916d7e531e69b02e426f7162dfb1c6c0
func getVirtDriveName(index int) (string, error) {
if index < 0 {
return "", fmt.Errorf("Index cannot be negative for drive")
}
// Prefix used for virtio-block devices
const prefix = "vd"
//Refer to DISK_NAME_LEN: https://github.com/torvalds/linux/blob/08c521a2011ff492490aa9ed6cc574be4235ce2b/include/linux/genhd.h#L61
diskNameLen := 32
base := 26
suffLen := diskNameLen - len(prefix)
diskLetters := make([]byte, suffLen)
var i int
for i = 0; i < suffLen && index >= 0; i++ {
letter := byte('a' + (index % base))
diskLetters[i] = letter
index = index/base - 1
}
if index >= 0 {
return "", fmt.Errorf("Index not supported")
}
diskName := prefix + reverseString(string(diskLetters[:i]))
return diskName, nil
}
const mountPerm = os.FileMode(0755)
// bindMount bind mounts a source in to a destination. This will
// do some bookkeeping:
// * evaluate all symlinks
// * ensure the source exists
// * recursively create the destination
func bindMount(source, destination string, readonly bool) error {
if source == "" {
return fmt.Errorf("source must be specified")
}
if destination == "" {
return fmt.Errorf("destination must be specified")
}
absSource, err := filepath.EvalSymlinks(source)
if err != nil {
return fmt.Errorf("Could not resolve symlink for source %v", source)
}
if err := ensureDestinationExists(absSource, destination); err != nil {
return fmt.Errorf("Could not create destination mount point %v: %v", destination, err)
} else if err := syscall.Mount(absSource, destination, "bind", syscall.MS_BIND, ""); err != nil {
return fmt.Errorf("Could not bind mount %v to %v: %v", absSource, destination, err)
}
// For readonly bind mounts, we need to remount with the readonly flag.
// This is needed as only very recent versions of libmount/util-linux support "bind,ro"
if readonly {
return syscall.Mount(absSource, destination, "bind", uintptr(syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY), "")
}
return nil
}
// bindMountContainerRootfs bind mounts a container rootfs into a 9pfs shared
// directory between the guest and the host.
func bindMountContainerRootfs(sharedDir, podID, cID, cRootFs string, readonly bool) error {
rootfsDest := filepath.Join(sharedDir, podID, cID, rootfsDir)
return bindMount(cRootFs, rootfsDest, readonly)
}
// Mount describes a container mount.
type Mount struct {
Source string
Destination string
// Type specifies the type of filesystem to mount.
Type string
// Options list all the mount options of the filesystem.
Options []string
// HostPath used to store host side bind mount path
HostPath string
// ReadOnly specifies if the mount should be read only or not
ReadOnly bool
}
func bindUnmountContainerRootfs(sharedDir, podID, cID string) error {
rootfsDest := filepath.Join(sharedDir, podID, cID, rootfsDir)
syscall.Unmount(rootfsDest, 0)
return nil
}
func bindUnmountAllRootfs(sharedDir string, pod Pod) {
for _, c := range pod.containers {
c.unmountHostMounts()
if c.state.Fstype == "" {
// Need to check for error returned by this call.
// See: https://github.com/containers/virtcontainers/issues/295
bindUnmountContainerRootfs(sharedDir, pod.id, c.id)
}
}
}
const maxSCSIDevices = 65535
// getSCSIIdLun gets the SCSI id and lun, based on the index of the drive being inserted.
// qemu code suggests that scsi-id can take values from 0 to 255 inclusive, while lun can
// take values from 0 to 16383 inclusive. But lun values over 255 do not seem to follow
// consistent SCSI addressing. Hence we limit to 255.
func getSCSIIdLun(index int) (int, int, error) {
if index < 0 {
return -1, -1, fmt.Errorf("Index cannot be negative")
}
if index > maxSCSIDevices {
return -1, -1, fmt.Errorf("Index cannot be greater than %d, maximum of %d devices are supported", maxSCSIDevices, maxSCSIDevices)
}
return index / 256, index % 256, nil
}
func getSCSIAddress(index int) (string, error) {
scsiID, lun, err := getSCSIIdLun(index)
if err != nil {
return "", err
}
return fmt.Sprintf("%d:%d", scsiID, lun), nil
}

334
virtcontainers/mount_test.go Normal file
View File

@@ -0,0 +1,334 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"bytes"
"fmt"
"github.com/stretchr/testify/assert"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
)
func TestIsSystemMount(t *testing.T) {
tests := []struct {
mnt string
expected bool
}{
{"/sys", true},
{"/sys/", true},
{"/sys//", true},
{"/sys/fs", true},
{"/sys/fs/", true},
{"/sys/fs/cgroup", true},
{"/sysfoo", false},
{"/home", false},
{"/dev/block/", true},
}
for _, test := range tests {
result := isSystemMount(test.mnt)
if result != test.expected {
t.Fatalf("Expected result for path %s : %v, got %v", test.mnt, test.expected, result)
}
}
}
func TestMajorMinorNumber(t *testing.T) {
devices := []string{"/dev/zero", "/dev/net/tun"}
for _, device := range devices {
cmdStr := fmt.Sprintf("ls -l %s | awk '{print $5$6}'", device)
cmd := exec.Command("sh", "-c", cmdStr)
output, err := cmd.Output()
if err != nil {
t.Fatal(err)
}
data := bytes.Split(output, []byte(","))
if len(data) < 2 {
t.Fatal()
}
majorStr := strings.TrimSpace(string(data[0]))
minorStr := strings.TrimSpace(string(data[1]))
majorNo, err := strconv.Atoi(majorStr)
if err != nil {
t.Fatal(err)
}
minorNo, err := strconv.Atoi(minorStr)
if err != nil {
t.Fatal(err)
}
stat := syscall.Stat_t{}
err = syscall.Stat(device, &stat)
if err != nil {
t.Fatal(err)
}
// Get major and minor numbers for the device itself. Note the use of stat.Rdev instead of Dev.
major := major(stat.Rdev)
minor := minor(stat.Rdev)
if minor != minorNo {
t.Fatalf("Expected minor number for device %s: %d, Got :%d", device, minorNo, minor)
}
if major != majorNo {
t.Fatalf("Expected major number for device %s : %d, Got :%d", device, majorNo, major)
}
}
}
func TestGetDeviceForPathRoot(t *testing.T) {
dev, err := getDeviceForPath("/")
if err != nil {
t.Fatal(err)
}
expected := "/"
if dev.mountPoint != expected {
t.Fatalf("Expected %s mountpoint, got %s", expected, dev.mountPoint)
}
}
func TestGetDeviceForPathValidMount(t *testing.T) {
dev, err := getDeviceForPath("/proc")
if err != nil {
t.Fatal(err)
}
expected := "/proc"
if dev.mountPoint != expected {
t.Fatalf("Expected %s mountpoint, got %s", expected, dev.mountPoint)
}
}
func TestGetDeviceForPathEmptyPath(t *testing.T) {
_, err := getDeviceForPath("")
if err == nil {
t.Fatal()
}
}
func TestGetDeviceForPath(t *testing.T) {
dev, err := getDeviceForPath("///")
if err != nil {
t.Fatal(err)
}
if dev.mountPoint != "/" {
t.Fatal(err)
}
_, err = getDeviceForPath("/../../.././././../.")
if err != nil {
t.Fatal(err)
}
_, err = getDeviceForPath("/root/file with spaces")
if err == nil {
t.Fatal()
}
}
func TestGetDeviceForPathBindMount(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledAsNonRoot)
}
source := filepath.Join(testDir, "testDeviceDirSrc")
dest := filepath.Join(testDir, "testDeviceDirDest")
syscall.Unmount(dest, 0)
os.Remove(source)
os.Remove(dest)
err := os.MkdirAll(source, mountPerm)
if err != nil {
t.Fatal(err)
}
defer os.Remove(source)
err = os.MkdirAll(dest, mountPerm)
if err != nil {
t.Fatal(err)
}
defer os.Remove(dest)
err = bindMount(source, dest, false)
if err != nil {
t.Fatal(err)
}
defer syscall.Unmount(dest, 0)
destFile := filepath.Join(dest, "test")
_, err = os.Create(destFile)
if err != nil {
fmt.Println("Could not create test file:", err)
t.Fatal(err)
}
defer os.Remove(destFile)
sourceDev, _ := getDeviceForPath(source)
destDev, _ := getDeviceForPath(destFile)
if sourceDev != destDev {
t.Fatal()
}
}
func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) {
_, _, err := getDevicePathAndFsType("")
if err == nil {
t.Fatal()
}
}
func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) {
path, fstype, err := getDevicePathAndFsType("/proc")
if err != nil {
t.Fatal(err)
}
if path != "proc" || fstype != "proc" {
t.Fatal(err)
}
}
func TestIsDeviceMapper(t *testing.T) {
// known major, minor for /dev/tty
major := 5
minor := 0
isDM, err := isDeviceMapper(major, minor)
if err != nil {
t.Fatal(err)
}
if isDM {
t.Fatal()
}
// fake the block device format
blockFormatTemplate = "/sys/dev/char/%d:%d"
isDM, err = isDeviceMapper(major, minor)
if err != nil {
t.Fatal(err)
}
if !isDM {
t.Fatal()
}
}
func TestGetVirtDriveNameInvalidIndex(t *testing.T) {
_, err := getVirtDriveName(-1)
if err == nil {
t.Fatal(err)
}
}
func TestGetVirtDriveName(t *testing.T) {
tests := []struct {
index int
expectedDrive string
}{
{0, "vda"},
{25, "vdz"},
{27, "vdab"},
{704, "vdaac"},
{18277, "vdzzz"},
}
for _, test := range tests {
driveName, err := getVirtDriveName(test.index)
if err != nil {
t.Fatal(err)
}
if driveName != test.expectedDrive {
t.Fatalf("Incorrect drive Name: Got: %s, Expecting :%s", driveName, test.expectedDrive)
}
}
}
func TestGetSCSIIdLun(t *testing.T) {
tests := []struct {
index int
expectedScsiID int
expectedLun int
}{
{0, 0, 0},
{1, 0, 1},
{2, 0, 2},
{255, 0, 255},
{256, 1, 0},
{257, 1, 1},
{258, 1, 2},
{512, 2, 0},
{513, 2, 1},
}
for _, test := range tests {
scsiID, lun, err := getSCSIIdLun(test.index)
assert.Nil(t, err)
if scsiID != test.expectedScsiID && lun != test.expectedLun {
t.Fatalf("Expecting scsi-id:lun %d:%d, Got %d:%d", test.expectedScsiID, test.expectedLun, scsiID, lun)
}
}
_, _, err := getSCSIIdLun(maxSCSIDevices + 1)
assert.NotNil(t, err)
}
func TestGetSCSIAddress(t *testing.T) {
tests := []struct {
index int
expectedSCSIAddress string
}{
{0, "0:0"},
{200, "0:200"},
{255, "0:255"},
{258, "1:2"},
{512, "2:0"},
}
for _, test := range tests {
scsiAddr, err := getSCSIAddress(test.index)
assert.Nil(t, err)
assert.Equal(t, scsiAddr, test.expectedSCSIAddress)
}
}

1394
virtcontainers/network.go Normal file
View File

File diff suppressed because it is too large Load Diff

410
virtcontainers/network_test.go Normal file
View File

@@ -0,0 +1,410 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"net"
"os"
"reflect"
"testing"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)
func testNetworkModelSet(t *testing.T, value string, expected NetworkModel) {
var netModel NetworkModel
err := netModel.Set(value)
if err != nil {
t.Fatal(err)
}
if netModel != expected {
t.Fatal()
}
}
func TestNoopNetworkModelSet(t *testing.T) {
testNetworkModelSet(t, "noop", NoopNetworkModel)
}
func TestCNINetworkModelSet(t *testing.T) {
testNetworkModelSet(t, "CNI", CNINetworkModel)
}
func TestCNMNetworkModelSet(t *testing.T) {
testNetworkModelSet(t, "CNM", CNMNetworkModel)
}
func TestNetworkModelSetFailure(t *testing.T) {
var netModel NetworkModel
err := netModel.Set("wrong-value")
if err == nil {
t.Fatal(err)
}
}
func testNetworkModelString(t *testing.T, netModel *NetworkModel, expected string) {
result := netModel.String()
if result != expected {
t.Fatal()
}
}
func TestNoopNetworkModelString(t *testing.T) {
netModel := NoopNetworkModel
testNetworkModelString(t, &netModel, string(NoopNetworkModel))
}
func TestCNINetworkModelString(t *testing.T) {
netModel := CNINetworkModel
testNetworkModelString(t, &netModel, string(CNINetworkModel))
}
func TestCNMNetworkModelString(t *testing.T) {
netModel := CNMNetworkModel
testNetworkModelString(t, &netModel, string(CNMNetworkModel))
}
func TestWrongNetworkModelString(t *testing.T) {
var netModel NetworkModel
testNetworkModelString(t, &netModel, "")
}
func testNewNetworkFromNetworkModel(t *testing.T, netModel NetworkModel, expected interface{}) {
result := newNetwork(netModel)
if reflect.DeepEqual(result, expected) == false {
t.Fatal()
}
}
func TestNewNoopNetworkFromNetworkModel(t *testing.T) {
testNewNetworkFromNetworkModel(t, NoopNetworkModel, &noopNetwork{})
}
func TestNewCNINetworkFromNetworkModel(t *testing.T) {
testNewNetworkFromNetworkModel(t, CNINetworkModel, &cni{})
}
func TestNewCNMNetworkFromNetworkModel(t *testing.T) {
testNewNetworkFromNetworkModel(t, CNMNetworkModel, &cnm{})
}
func TestNewUnknownNetworkFromNetworkModel(t *testing.T) {
var netModel NetworkModel
testNewNetworkFromNetworkModel(t, netModel, &noopNetwork{})
}
func TestCreateDeleteNetNS(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip(testDisabledAsNonRoot)
}
netNSPath, err := createNetNS()
if err != nil {
t.Fatal(err)
}
if netNSPath == "" {
t.Fatal()
}
_, err = os.Stat(netNSPath)
if err != nil {
t.Fatal(err)
}
err = deleteNetNS(netNSPath, true)
if err != nil {
t.Fatal(err)
}
}
func testEndpointTypeSet(t *testing.T, value string, expected EndpointType) {
//var netModel NetworkModel
var endpointType EndpointType
err := endpointType.Set(value)
if err != nil {
t.Fatal(err)
}
if endpointType != expected {
t.Fatal()
}
}
func TestPhysicalEndpointTypeSet(t *testing.T) {
testEndpointTypeSet(t, "physical", PhysicalEndpointType)
}
func TestVirtualEndpointTypeSet(t *testing.T) {
testEndpointTypeSet(t, "virtual", VirtualEndpointType)
}
func TestEndpointTypeSetFailure(t *testing.T) {
var endpointType EndpointType
err := endpointType.Set("wrong-value")
if err == nil {
t.Fatal(err)
}
}
func testEndpointTypeString(t *testing.T, endpointType *EndpointType, expected string) {
result := endpointType.String()
if result != expected {
t.Fatal()
}
}
func TestPhysicalEndpointTypeString(t *testing.T) {
endpointType := PhysicalEndpointType
testEndpointTypeString(t, &endpointType, string(PhysicalEndpointType))
}
func TestVirtualEndpointTypeString(t *testing.T) {
endpointType := VirtualEndpointType
testEndpointTypeString(t, &endpointType, string(VirtualEndpointType))
}
func TestIncorrectEndpointTypeString(t *testing.T) {
var endpointType EndpointType
testEndpointTypeString(t, &endpointType, "")
}
func TestCreateVhostUserEndpoint(t *testing.T) {
macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48}
ifcName := "vhost-deadbeef"
socket := "/tmp/vhu_192.168.0.1"
netinfo := NetworkInfo{
Iface: NetlinkIface{
LinkAttrs: netlink.LinkAttrs{
HardwareAddr: macAddr,
Name: ifcName,
},
},
}
expected := &VhostUserEndpoint{
SocketPath: socket,
HardAddr: macAddr.String(),
IfaceName: ifcName,
EndpointType: VhostUserEndpointType,
}
result, err := createVhostUserEndpoint(netinfo, socket)
if err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(result, expected) == false {
t.Fatalf("\n\tGot %v\n\tExpecting %v", result, expected)
}
}
func TestCreateVirtualNetworkEndpoint(t *testing.T) {
macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04}
expected := &VirtualEndpoint{
NetPair: NetworkInterfacePair{
ID: "uniqueTestID-4",
Name: "br4",
VirtIface: NetworkInterface{
Name: "eth4",
HardAddr: macAddr.String(),
},
TAPIface: NetworkInterface{
Name: "tap4",
},
NetInterworkingModel: DefaultNetInterworkingModel,
},
EndpointType: VirtualEndpointType,
}
result, err := createVirtualNetworkEndpoint(4, "", DefaultNetInterworkingModel)
if err != nil {
t.Fatal(err)
}
// the resulting ID will be random - so let's overwrite to test the rest of the flow
result.NetPair.ID = "uniqueTestID-4"
if reflect.DeepEqual(result, expected) == false {
t.Fatal()
}
}
func TestCreateVirtualNetworkEndpointChooseIfaceName(t *testing.T) {
macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04}
expected := &VirtualEndpoint{
NetPair: NetworkInterfacePair{
ID: "uniqueTestID-4",
Name: "br4",
VirtIface: NetworkInterface{
Name: "eth1",
HardAddr: macAddr.String(),
},
TAPIface: NetworkInterface{
Name: "tap4",
},
NetInterworkingModel: DefaultNetInterworkingModel,
},
EndpointType: VirtualEndpointType,
}
result, err := createVirtualNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel)
if err != nil {
t.Fatal(err)
}
// the resulting ID will be random - so let's overwrite to test the rest of the flow
result.NetPair.ID = "uniqueTestID-4"
if reflect.DeepEqual(result, expected) == false {
t.Fatal()
}
}
func TestCreateVirtualNetworkEndpointInvalidArgs(t *testing.T) {
type endpointValues struct {
idx int
ifName string
}
// all elements are expected to result in failure
failingValues := []endpointValues{
{-1, "bar"},
{-1, ""},
}
for _, d := range failingValues {
result, err := createVirtualNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel)
if err == nil {
t.Fatalf("expected invalid endpoint for %v, got %v", d, result)
}
}
}
func TestIsPhysicalIface(t *testing.T) {
testNetIface := "testIface0"
testMTU := 1500
testMACAddr := "00:00:00:00:00:01"
hwAddr, err := net.ParseMAC(testMACAddr)
if err != nil {
t.Fatal(err)
}
link := &netlink.Bridge{
LinkAttrs: netlink.LinkAttrs{
Name: testNetIface,
MTU: testMTU,
HardwareAddr: hwAddr,
TxQLen: -1,
},
}
n, err := ns.NewNS()
if err != nil {
t.Fatal(err)
}
defer n.Close()
netnsHandle, err := netns.GetFromPath(n.Path())
if err != nil {
t.Fatal(err)
}
defer netnsHandle.Close()
netlinkHandle, err := netlink.NewHandleAt(netnsHandle)
if err != nil {
t.Fatal(err)
}
defer netlinkHandle.Delete()
if err := netlinkHandle.LinkAdd(link); err != nil {
t.Fatal(err)
}
var isPhysical bool
err = doNetNS(n.Path(), func(_ ns.NetNS) error {
var err error
isPhysical, err = isPhysicalIface(testNetIface)
return err
})
if err != nil {
t.Fatal(err)
}
if isPhysical == true {
t.Fatalf("Got %+v\nExpecting %+v", isPhysical, false)
}
}
func TestNetInterworkingModelIsValid(t *testing.T) {
tests := []struct {
name string
n NetInterworkingModel
want bool
}{
{"Invalid Model", NetXConnectInvalidModel, false},
{"Default Model", NetXConnectDefaultModel, true},
{"Bridged Model", NetXConnectBridgedModel, true},
{"Macvtap Model", NetXConnectMacVtapModel, true},
{"Enlightened Model", NetXConnectEnlightenedModel, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.n.IsValid(); got != tt.want {
t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want)
}
})
}
}
func TestNetInterworkingModelSetModel(t *testing.T) {
var n NetInterworkingModel
tests := []struct {
name string
modelName string
wantErr bool
}{
{"Invalid Model", "Invalid", true},
{"default Model", "default", false},
{"bridged Model", "bridged", false},
{"macvtap Model", "macvtap", false},
{"enlightened Model", "enlightened", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr {
t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

48
virtcontainers/no_proxy.go Normal file
View File

@@ -0,0 +1,48 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"fmt"
)
// This is the no proxy implementation of the proxy interface. This
// is a generic implementation for any case (basically any agent),
// where no actual proxy is needed. This happens when the combination
// of the VM and the agent can handle multiple connections without
// additional component to handle the multiplexing. Both the runtime
// and the shim will connect to the agent through the VM, bypassing
// the proxy model.
// That's why this implementation is very generic, and all it does
// is to provide both shim and runtime the correct URL to connect
// directly to the VM.
type noProxy struct {
}
// start is noProxy start implementation for proxy interface.
func (p *noProxy) start(pod Pod, params proxyParams) (int, string, error) {
if params.agentURL == "" {
return -1, "", fmt.Errorf("AgentURL cannot be empty")
}
return 0, params.agentURL, nil
}
// stop is noProxy stop implementation for proxy interface.
func (p *noProxy) stop(pod Pod, pid int) error {
return nil
}

53
virtcontainers/no_proxy_test.go Normal file
View File

@@ -0,0 +1,53 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"testing"
)
var testNoProxyVMURL = "vmURL"
func TestNoProxyStart(t *testing.T) {
pod := Pod{
agent: newAgent(NoopAgentType),
}
p := &noProxy{}
agentURL := "agentURL"
pid, vmURL, err := p.start(pod, proxyParams{agentURL: agentURL})
if err != nil {
t.Fatal(err)
}
if vmURL != agentURL {
t.Fatalf("Got URL %q, expecting %q", vmURL, agentURL)
}
if pid != 0 {
t.Fatal("Failure since returned PID should be 0")
}
}
func TestNoProxyStop(t *testing.T) {
p := &noProxy{}
if err := p.stop(Pod{}, 0); err != nil {
t.Fatal(err)
}
}

81
virtcontainers/noop_agent.go Normal file
View File

@@ -0,0 +1,81 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"syscall"
)
// noopAgent a.k.a. NO-OP Agent is an empty Agent implementation, for testing and
// mocking purposes.
type noopAgent struct {
}
// init initializes the Noop agent, i.e. it does nothing.
func (n *noopAgent) init(pod *Pod, config interface{}) error {
return nil
}
// createPod is the Noop agent pod creation implementation. It does nothing.
func (n *noopAgent) createPod(pod *Pod) error {
return nil
}
// capabilities returns empty capabilities, i.e no capabilties are supported.
func (n *noopAgent) capabilities() capabilities {
return capabilities{}
}
// exec is the Noop agent command execution implementation. It does nothing.
func (n *noopAgent) exec(pod *Pod, c Container, cmd Cmd) (*Process, error) {
return nil, nil
}
// startPod is the Noop agent Pod starting implementation. It does nothing.
func (n *noopAgent) startPod(pod Pod) error {
return nil
}
// stopPod is the Noop agent Pod stopping implementation. It does nothing.
func (n *noopAgent) stopPod(pod Pod) error {
return nil
}
// createContainer is the Noop agent Container creation implementation. It does nothing.
func (n *noopAgent) createContainer(pod *Pod, c *Container) (*Process, error) {
return &Process{}, nil
}
// startContainer is the Noop agent Container starting implementation. It does nothing.
func (n *noopAgent) startContainer(pod Pod, c *Container) error {
return nil
}
// stopContainer is the Noop agent Container stopping implementation. It does nothing.
func (n *noopAgent) stopContainer(pod Pod, c Container) error {
return nil
}
// killContainer is the Noop agent Container signaling implementation. It does nothing.
func (n *noopAgent) killContainer(pod Pod, c Container, signal syscall.Signal, all bool) error {
return nil
}
// processListContainer is the Noop agent Container ps implementation. It does nothing.
func (n *noopAgent) processListContainer(pod Pod, c Container, options ProcessListOptions) (ProcessList, error) {
return nil, nil
}

125
virtcontainers/noop_agent_test.go Normal file
View File

@@ -0,0 +1,125 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"testing"
)
func testCreateNoopContainer() (*Pod, *Container, error) {
contID := "100"
config := newTestPodConfigNoop()
p, err := CreatePod(config)
if err != nil {
return nil, nil, err
}
contConfig := newTestContainerConfigNoop(contID)
p, c, err := CreateContainer(p.ID(), contConfig)
if err != nil {
return nil, nil, err
}
return p.(*Pod), c.(*Container), nil
}
func TestNoopAgentInit(t *testing.T) {
n := &noopAgent{}
pod := &Pod{}
err := n.init(pod, nil)
if err != nil {
t.Fatal(err)
}
}
func TestNoopAgentExec(t *testing.T) {
n := &noopAgent{}
cmd := Cmd{}
pod, container, err := testCreateNoopContainer()
if err != nil {
t.Fatal(err)
}
if _, err = n.exec(pod, *container, cmd); err != nil {
t.Fatal(err)
}
}
func TestNoopAgentStartPod(t *testing.T) {
n := &noopAgent{}
pod := Pod{}
err := n.startPod(pod)
if err != nil {
t.Fatal(err)
}
}
func TestNoopAgentStopPod(t *testing.T) {
n := &noopAgent{}
pod := Pod{}
err := n.stopPod(pod)
if err != nil {
t.Fatal(err)
}
}
func TestNoopAgentCreateContainer(t *testing.T) {
n := &noopAgent{}
pod, container, err := testCreateNoopContainer()
if err != nil {
t.Fatal(err)
}
if err := n.startPod(*pod); err != nil {
t.Fatal(err)
}
if _, err := n.createContainer(pod, container); err != nil {
t.Fatal(err)
}
}
func TestNoopAgentStartContainer(t *testing.T) {
n := &noopAgent{}
pod, container, err := testCreateNoopContainer()
if err != nil {
t.Fatal(err)
}
err = n.startContainer(*pod, container)
if err != nil {
t.Fatal(err)
}
}
func TestNoopAgentStopContainer(t *testing.T) {
n := &noopAgent{}
pod, container, err := testCreateNoopContainer()
if err != nil {
t.Fatal(err)
}
err = n.stopContainer(*pod, *container)
if err != nil {
t.Fatal(err)
}
}

48
virtcontainers/noop_network.go Normal file
View File

@@ -0,0 +1,48 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
// noopNetwork a.k.a. NO-OP Network is an empty network implementation, for
// testing and mocking purposes.
type noopNetwork struct {
}
// init initializes the network, setting a new network namespace for the Noop network.
// It does nothing.
func (n *noopNetwork) init(config NetworkConfig) (string, bool, error) {
return "", true, nil
}
// run runs a callback in the specified network namespace for
// the Noop network.
// It does nothing.
func (n *noopNetwork) run(networkNSPath string, cb func() error) error {
return cb()
}
// add adds all needed interfaces inside the network namespace the Noop network.
// It does nothing.
func (n *noopNetwork) add(pod Pod, config NetworkConfig, netNsPath string, netNsCreated bool) (NetworkNamespace, error) {
return NetworkNamespace{}, nil
}
// remove unbridges and deletes TAP interfaces. It also removes virtual network
// interfaces and deletes the network namespace for the Noop network.
// It does nothing.
func (n *noopNetwork) remove(pod Pod, networkNS NetworkNamespace) error {
return nil
}

35
virtcontainers/noop_proxy.go Normal file
View File

@@ -0,0 +1,35 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
// This is a dummy proxy implementation of the proxy interface, only
// used for testing purpose.
type noopProxy struct{}
var noopProxyURL = "noopProxyURL"
// register is the proxy start implementation for testing purpose.
// It does nothing.
func (p *noopProxy) start(pod Pod, params proxyParams) (int, string, error) {
return 0, noopProxyURL, nil
}
// stop is the proxy stop implementation for testing purpose.
// It does nothing.
func (p *noopProxy) stop(pod Pod, pid int) error {
return nil
}

25
virtcontainers/noop_shim.go Normal file
View File

@@ -0,0 +1,25 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
type noopShim struct{}
// start is the noopShim start implementation for testing purpose.
// It does nothing.
func (s *noopShim) start(pod Pod, params ShimParams) (int, error) {
return 1000, nil
}

37
virtcontainers/noop_shim_test.go Normal file
View File

@@ -0,0 +1,37 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"testing"
)
func TestNoopShimStart(t *testing.T) {
s := &noopShim{}
pod := Pod{}
params := ShimParams{}
expected := 1000
pid, err := s.start(pod, params)
if err != nil {
t.Fatal(err)
}
if pid != expected {
t.Fatalf("PID should be %d", expected)
}
}

40
virtcontainers/nsenter.go Normal file
View File

@@ -0,0 +1,40 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
// nsenter is a spawner implementation for the nsenter util-linux command.
type nsenter struct {
ContConfig ContainerConfig
}
const (
// NsenterCmd is the command used to start nsenter.
nsenterCmd = "nsenter"
)
// formatArgs is the spawner command formatting implementation for nsenter.
func (n *nsenter) formatArgs(args []string) ([]string, error) {
var newArgs []string
pid := "-1"
// TODO: Retrieve container PID from container ID
newArgs = append(newArgs, nsenterCmd+" --target "+pid+" --mount --uts --ipc --net --pid")
newArgs = append(newArgs, args...)
return newArgs, nil
}

43
virtcontainers/nsenter_test.go Normal file
View File

@@ -0,0 +1,43 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package virtcontainers
import (
"strings"
"testing"
)
func testNsEnterFormatArgs(t *testing.T, args []string, expected string) {
nsenter := &nsenter{}
cmd, err := nsenter.formatArgs(args)
if err != nil {
t.Fatal(err)
}
if strings.Join(cmd, " ") != expected {
t.Fatal()
}
}
func TestNsEnterFormatArgsHello(t *testing.T) {
expectedCmd := "nsenter --target -1 --mount --uts --ipc --net --pid echo hello"
args := []string{"echo", "hello"}
testNsEnterFormatArgs(t, args, expectedCmd)
}

62
virtcontainers/pkg/annotations/annotations.go Normal file
View File

@@ -0,0 +1,62 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package annotations
const (
vcAnnotationsPrefix = "com.github.containers.virtcontainers."
// KernelPath is a pod annotation for passing a per container path pointing at the kernel needed to boot the container VM.
KernelPath = vcAnnotationsPrefix + "KernelPath"
// ImagePath is a pod annotation for passing a per container path pointing at the guest image that will run in the container VM.
ImagePath = vcAnnotationsPrefix + "ImagePath"
// HypervisorPath is a pod annotation for passing a per container path pointing at the hypervisor that will run the container VM.
HypervisorPath = vcAnnotationsPrefix + "HypervisorPath"
// FirmwarePath is a pod annotation for passing a per container path pointing at the guest firmware that will run the container VM.
FirmwarePath = vcAnnotationsPrefix + "FirmwarePath"
// KernelHash is a pod annotation for passing a container kernel image SHA-512 hash value.
KernelHash = vcAnnotationsPrefix + "KernelHash"
// ImageHash is an pod annotation for passing a container guest image SHA-512 hash value.
ImageHash = vcAnnotationsPrefix + "ImageHash"
// HypervisorHash is an pod annotation for passing a container hypervisor binary SHA-512 hash value.
HypervisorHash = vcAnnotationsPrefix + "HypervisorHash"
// FirmwareHash is an pod annotation for passing a container guest firmware SHA-512 hash value.
FirmwareHash = vcAnnotationsPrefix + "FirmwareHash"
// AssetHashType is the hash type used for assets verification
AssetHashType = vcAnnotationsPrefix + "AssetHashType"
// ConfigJSONKey is the annotation key to fetch the OCI configuration.
ConfigJSONKey = vcAnnotationsPrefix + "pkg.oci.config"
// BundlePathKey is the annotation key to fetch the OCI configuration file path.
BundlePathKey = vcAnnotationsPrefix + "pkg.oci.bundle_path"
// ContainerTypeKey is the annotation key to fetch container type.
ContainerTypeKey = vcAnnotationsPrefix + "pkg.oci.container_type"
)
const (
// SHA512 is the SHA-512 (64) hash algorithm
SHA512 string = "sha512"
)

36
virtcontainers/pkg/annotations/dockershim/annotations.go Normal file
View File

@@ -0,0 +1,36 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dockershim
const (
// Copied from k8s.io/pkg/kubelet/dockershim/docker_service.go,
// used to identify whether a docker container is a sandbox or
// a regular container, will be removed after defining those as
// public fields in dockershim.
// ContainerTypeLabelKey is the container type (podsandbox or container) annotation
ContainerTypeLabelKey = "io.kubernetes.docker.type"
// ContainerTypeLabelSandbox represents a pod sandbox container
ContainerTypeLabelSandbox = "podsandbox"
// ContainerTypeLabelContainer represents a container running within a pod
ContainerTypeLabelContainer = "container"
// SandboxIDLabelKey is the sandbox ID annotation
SandboxIDLabelKey = "io.kubernetes.sandbox.id"
)

6
virtcontainers/pkg/cni/OWNERS Normal file
View File

@@ -0,0 +1,6 @@
reviewers:
- virtcontainers-maintainers
approvers:
- mcastelino
- sboeuf

167
virtcontainers/pkg/cni/cni.go Normal file
View File

@@ -0,0 +1,167 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package cni
import (
"fmt"
"sort"
"github.com/containernetworking/cni/libcni"
types "github.com/containernetworking/cni/pkg/types"
)
// CNI default values to find plugins and configurations.
const (
LocalNetName = "lo"
DefNetName = "net"
PluginConfDir = "/etc/cni/net.d"
PluginBinDir = "/opt/cni/bin"
)
var confExtensions = []string{".conf"}
// NetworkPlugin is the CNI network plugin handler.
type NetworkPlugin struct {
loNetwork *cniNetwork
defNetwork *cniNetwork
}
type cniNetwork struct {
name string
networkConfig *libcni.NetworkConfig
cniConfig libcni.CNI
}
// NewNetworkPlugin initialize the CNI network plugin and returns
// a handler to it.
func NewNetworkPlugin() (*NetworkPlugin, error) {
return NewNetworkPluginWithArgs(PluginConfDir, PluginBinDir)
}
// NewNetworkPluginWithArgs initialize the CNI network plugin, specifying the
// configuration and binary directories, and it returns a handler to it.
func NewNetworkPluginWithArgs(confDir, binDir string) (*NetworkPlugin, error) {
var err error
plugin := &NetworkPlugin{}
plugin.loNetwork, err = getLoNetwork(confDir, binDir)
if err != nil {
return nil, err
}
plugin.defNetwork, err = getDefNetwork(confDir, binDir)
if err != nil {
return nil, err
}
return plugin, nil
}
func getNetwork(confDir, binDir, defaultName string, local bool) (*cniNetwork, error) {
confFiles, err := libcni.ConfFiles(confDir, confExtensions)
if err != nil || confFiles == nil {
return nil, fmt.Errorf("Invalid configuration directory %s", confDir)
}
if len(confFiles) == 0 {
return nil, fmt.Errorf("Could not find networks in %s", confDir)
}
if local == true {
sort.Sort(sort.Reverse(sort.StringSlice(confFiles)))
} else {
sort.Sort(sort.StringSlice(confFiles))
}
for _, confFile := range confFiles {
conf, err := libcni.ConfFromFile(confFile)
if err != nil {
continue
}
cninet := &libcni.CNIConfig{
Path: []string{binDir},
}
name := defaultName
if conf.Network.Name != "" {
name = conf.Network.Name
}
network := &cniNetwork{
name: name,
networkConfig: conf,
cniConfig: cninet,
}
return network, nil
}
return nil, fmt.Errorf("No valid networks found in %s", confDir)
}
func getLoNetwork(confDir, binDir string) (*cniNetwork, error) {
return getNetwork(confDir, binDir, LocalNetName, true)
}
func getDefNetwork(confDir, binDir string) (*cniNetwork, error) {
return getNetwork(confDir, binDir, DefNetName, false)
}
func buildRuntimeConf(podID, podNetNSPath, ifName string) *libcni.RuntimeConf {
return &libcni.RuntimeConf{
ContainerID: podID,
NetNS: podNetNSPath,
IfName: ifName,
}
}
// AddNetwork calls the CNI plugin to create a network between the host and the network namespace.
func (plugin *NetworkPlugin) AddNetwork(podID, netNSPath, ifName string) (types.Result, error) {
rt := buildRuntimeConf(podID, netNSPath, ifName)
_, err := plugin.loNetwork.cniConfig.AddNetwork(plugin.loNetwork.networkConfig, rt)
if err != nil {
return nil, err
}
ifaceResult, err := plugin.defNetwork.cniConfig.AddNetwork(plugin.defNetwork.networkConfig, rt)
if err != nil {
return nil, err
}
return ifaceResult, nil
}
// RemoveNetwork calls the CNI plugin to remove a specific network previously created between
// the host and the network namespace.
func (plugin *NetworkPlugin) RemoveNetwork(podID, netNSPath, ifName string) error {
rt := buildRuntimeConf(podID, netNSPath, ifName)
err := plugin.defNetwork.cniConfig.DelNetwork(plugin.defNetwork.networkConfig, rt)
if err != nil {
return err
}
err = plugin.loNetwork.cniConfig.DelNetwork(plugin.loNetwork.networkConfig, rt)
if err != nil {
return err
}
return nil
}

408
virtcontainers/pkg/cni/cni_test.go Normal file
View File

@@ -0,0 +1,408 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package cni
import (
"fmt"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/containernetworking/cni/libcni"
"github.com/containernetworking/plugins/pkg/ns"
)
const (
dirMode = os.FileMode(0750)
)
var testDirBase = "../../utils/supportfiles/cni"
var testConfDir = testDirBase + "/net.d"
var testBinDir = testDirBase + "/bin"
var testWrongConfDir = testDirBase + "/wrong"
var testDefFile = "10-test_network.conf"
var testLoFile = "99-test_loopback.conf"
var testWrongFile = "100-test_error.conf"
var testLoFileContent = []byte(`{
"cniVersion": "0.3.0",
"name": "testlonetwork",
"type": "loopback"
}`)
var testLoFileContentNoName = []byte(`{
"cniVersion": "0.3.0",
"type": "loopback"
}`)
var testDefFileContent = []byte(`{
"cniVersion": "0.3.0",
"name": "testdefnetwork",
"type": "cni-bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.88.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
}`)
var testDefFileContentNoName = []byte(`{
"cniVersion": "0.3.0",
"type": "cni-bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.88.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
}`)
var testWrongFileContent = []byte(`{
"cniVersion "0.3.0",
"type": "loopback"
}`)
func createLoNetwork(t *testing.T) {
loFile := filepath.Join(testConfDir, testLoFile)
f, err := os.Create(loFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
_, err = f.Write(testLoFileContent)
if err != nil {
t.Fatal(err)
}
}
func createLoNetworkNoName(t *testing.T) {
loFile := filepath.Join(testConfDir, testLoFile)
f, err := os.Create(loFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
_, err = f.Write(testLoFileContentNoName)
if err != nil {
t.Fatal(err)
}
}
func createDefNetwork(t *testing.T) {
defFile := filepath.Join(testConfDir, testDefFile)
f, err := os.Create(defFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
_, err = f.Write(testDefFileContent)
if err != nil {
t.Fatal(err)
}
}
func createDefNetworkNoName(t *testing.T) {
defFile := filepath.Join(testConfDir, testDefFile)
f, err := os.Create(defFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
_, err = f.Write(testDefFileContentNoName)
if err != nil {
t.Fatal(err)
}
}
func createWrongNetwork(t *testing.T) {
wrongFile := filepath.Join(testConfDir, testWrongFile)
f, err := os.Create(wrongFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
_, err = f.Write(testWrongFileContent)
if err != nil {
t.Fatal(err)
}
}
func removeLoNetwork(t *testing.T) {
loFile := filepath.Join(testConfDir, testLoFile)
err := os.Remove(loFile)
if err != nil {
t.Fatal(err)
}
}
func removeDefNetwork(t *testing.T) {
defFile := filepath.Join(testConfDir, testDefFile)
err := os.Remove(defFile)
if err != nil {
t.Fatal(err)
}
}
func removeWrongNetwork(t *testing.T) {
wrongFile := filepath.Join(testConfDir, testWrongFile)
err := os.Remove(wrongFile)
if err != nil {
t.Fatal(err)
}
}
func TestNewNetworkPluginSuccessful(t *testing.T) {
createLoNetwork(t)
defer removeLoNetwork(t)
createDefNetwork(t)
defer removeDefNetwork(t)
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err != nil {
t.Fatal(err)
}
if netPlugin.loNetwork == nil {
t.Fatal("Invalid local network")
}
if netPlugin.defNetwork == nil {
t.Fatal("Invalid default network")
}
if netPlugin.loNetwork.name != "testlonetwork" {
t.Fatal("Invalid local network name")
}
if netPlugin.defNetwork.name != "testdefnetwork" {
t.Fatal("Invalid default network name")
}
}
func TestNewNetworkPluginSuccessfulNoName(t *testing.T) {
createLoNetworkNoName(t)
defer removeLoNetwork(t)
createDefNetworkNoName(t)
defer removeDefNetwork(t)
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err != nil {
t.Fatal(err)
}
if netPlugin.loNetwork == nil {
t.Fatal("Invalid local network")
}
if netPlugin.defNetwork == nil {
t.Fatal("Invalid default network")
}
if netPlugin.loNetwork.name != "lo" {
t.Fatal("Invalid local network name")
}
if netPlugin.defNetwork.name != "net" {
t.Fatal("Invalid default network name")
}
}
func TestNewNetworkPluginFailureNoNetwork(t *testing.T) {
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err == nil || netPlugin != nil {
t.Fatal("Should fail because no available network")
}
}
func TestNewNetworkPluginFailureNoConfDir(t *testing.T) {
netPlugin, err := NewNetworkPluginWithArgs(testWrongConfDir, testBinDir)
if err == nil || netPlugin != nil {
t.Fatal("Should fail because configuration directory does not exist")
}
}
func TestNewNetworkPluginFailureWrongNetwork(t *testing.T) {
createWrongNetwork(t)
defer removeWrongNetwork(t)
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err == nil || netPlugin != nil {
t.Fatal("Should fail because of wrong network definition")
}
}
func TestBuildRuntimeConf(t *testing.T) {
expected := libcni.RuntimeConf{
ContainerID: "testPodID",
NetNS: "testPodNetNSPath",
IfName: "testIfName",
}
runtimeConf := buildRuntimeConf("testPodID", "testPodNetNSPath", "testIfName")
if reflect.DeepEqual(*runtimeConf, expected) == false {
t.Fatal("Runtime configuration different from expected one")
}
}
func TestAddNetworkSuccessful(t *testing.T) {
createLoNetworkNoName(t)
defer removeLoNetwork(t)
createDefNetworkNoName(t)
defer removeDefNetwork(t)
netNsHandle, err := ns.NewNS()
if err != nil {
t.Fatal(err)
}
defer netNsHandle.Close()
testNetNsPath := netNsHandle.Path()
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err != nil {
t.Fatal(err)
}
_, err = netPlugin.AddNetwork("testPodID", testNetNsPath, "testIfName")
if err != nil {
t.Fatal(err)
}
}
func TestAddNetworkFailureUnknownNetNs(t *testing.T) {
createLoNetworkNoName(t)
defer removeLoNetwork(t)
createDefNetworkNoName(t)
defer removeDefNetwork(t)
const invalidNetNsPath = "/this/path/does/not/exist"
// ensure it really is invalid
_, err := os.Stat(invalidNetNsPath)
if err == nil {
t.Fatalf("directory %v unexpectedly exists", invalidNetNsPath)
}
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err != nil {
t.Fatal(err)
}
_, err = netPlugin.AddNetwork("testPodID", invalidNetNsPath, "testIfName")
if err == nil {
t.Fatalf("Should fail because netns %s does not exist", invalidNetNsPath)
}
}
func TestRemoveNetworkSuccessful(t *testing.T) {
createLoNetworkNoName(t)
defer removeLoNetwork(t)
createDefNetworkNoName(t)
defer removeDefNetwork(t)
netNsHandle, err := ns.NewNS()
if err != nil {
t.Fatal(err)
}
defer netNsHandle.Close()
testNetNsPath := netNsHandle.Path()
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err != nil {
t.Fatal(err)
}
_, err = netPlugin.AddNetwork("testPodID", testNetNsPath, "testIfName")
if err != nil {
t.Fatal(err)
}
err = netPlugin.RemoveNetwork("testPodID", testNetNsPath, "testIfName")
if err != nil {
t.Fatal(err)
}
}
func TestRemoveNetworkSuccessfulNetworkDoesNotExist(t *testing.T) {
createLoNetworkNoName(t)
defer removeLoNetwork(t)
createDefNetworkNoName(t)
defer removeDefNetwork(t)
netNsHandle, err := ns.NewNS()
if err != nil {
t.Fatal(err)
}
defer netNsHandle.Close()
testNetNsPath := netNsHandle.Path()
netPlugin, err := NewNetworkPluginWithArgs(testConfDir, testBinDir)
if err != nil {
t.Fatal(err)
}
err = netPlugin.RemoveNetwork("testPodID", testNetNsPath, "testIfName")
if err != nil {
// CNI specification says that no error should be returned
// in case we try to tear down a non-existing network.
t.Fatalf("Should pass because network not previously added: %s", err)
}
}
func TestMain(m *testing.M) {
err := os.MkdirAll(testConfDir, dirMode)
if err != nil {
fmt.Println("Could not create test configuration directory:", err)
os.Exit(1)
}
_, err = os.Stat(testBinDir)
if err != nil {
fmt.Println("Test binary directory should exist:", err)
os.RemoveAll(testConfDir)
os.Exit(1)
}
ret := m.Run()
os.RemoveAll(testConfDir)
os.Exit(ret)
}

164
virtcontainers/pkg/ethtool/ethtool.go Normal file
View File

@@ -0,0 +1,164 @@
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package ethtool
import (
"bytes"
"syscall"
"unsafe"
)
// Maximum size of an interface name
const (
IFNAMSIZ = 16
)
// ioctl ethtool request
const (
SIOCETHTOOL = 0x8946
)
// ethtool stats related constants.
const (
ethGstringLen = 32
ethtoolGDrvInfo = 0x00000003
)
// maxGtrings maximum number of stats entries that ethtool can
// retrieve currently.
const (
maxGstrings = 1000
)
type ifreq struct {
ifrName [IFNAMSIZ]byte
ifrData uintptr
}
type ethtoolDrvInfo struct {
cmd uint32
driver [32]byte
version [32]byte
fwVersion [32]byte
busInfo [32]byte
eromVersion [32]byte
reserved2 [12]byte
nPrivFlags uint32
nStats uint32
testinfoLen uint32
eedumpLen uint32
regdumpLen uint32
}
type ethtoolGStrings struct {
cmd uint32
stringSet uint32
len uint32
data [maxGstrings * ethGstringLen]byte
}
type ethtoolStats struct {
cmd uint32
nStats uint32
data [maxGstrings]uint64
}
// Ethtool file descriptor.
type Ethtool struct {
fd int
}
// DriverName returns the driver name of the given interface.
func (e *Ethtool) DriverName(intf string) (string, error) {
info, err := e.getDriverInfo(intf)
if err != nil {
return "", err
}
return string(bytes.Trim(info.driver[:], "\x00")), nil
}
// BusInfo returns the bus info of the given interface.
func (e *Ethtool) BusInfo(intf string) (string, error) {
info, err := e.getDriverInfo(intf)
if err != nil {
return "", err
}
return string(bytes.Trim(info.busInfo[:], "\x00")), nil
}
func (e *Ethtool) getDriverInfo(intf string) (ethtoolDrvInfo, error) {
drvinfo := ethtoolDrvInfo{
cmd: ethtoolGDrvInfo,
}
var name [IFNAMSIZ]byte
copy(name[:], []byte(intf))
ifr := ifreq{
ifrName: name,
ifrData: uintptr(unsafe.Pointer(&drvinfo)),
}
_, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), SIOCETHTOOL, uintptr(unsafe.Pointer(&ifr)))
if ep != 0 {
return ethtoolDrvInfo{}, syscall.Errno(ep)
}
return drvinfo, nil
}
// Close closes the ethtool file descriptor.
func (e *Ethtool) Close() {
syscall.Close(e.fd)
}
// NewEthtool opens a ethtool socket.
func NewEthtool() (*Ethtool, error) {
fd, _, err := syscall.RawSyscall(syscall.SYS_SOCKET, syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
if err != 0 {
return nil, syscall.Errno(err)
}
return &Ethtool{
fd: int(fd),
}, nil
}
// BusInfo returns the bus information of the network device.
func BusInfo(intf string) (string, error) {
e, err := NewEthtool()
if err != nil {
return "", err
}
defer e.Close()
return e.BusInfo(intf)
}
// DriverName returns the driver name of the network interface.
func DriverName(intf string) (string, error) {
e, err := NewEthtool()
if err != nil {
return "", err
}
defer e.Close()
return e.DriverName(intf)
}

67
virtcontainers/pkg/ethtool/ethtool_test.go Normal file
View File

@@ -0,0 +1,67 @@
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package ethtool
import (
"net"
"testing"
)
func TestDriverName(t *testing.T) {
intfs, err := net.Interfaces()
if err != nil {
t.Fatal(err)
}
// we expected to have at least one success
success := false
for _, intf := range intfs {
_, err := DriverName(intf.Name)
if err == nil {
success = true
}
}
if !success {
t.Fatal("Unable to retrieve driver from any interface of this system.")
}
}
func TestBusInfo(t *testing.T) {
intfs, err := net.Interfaces()
if err != nil {
t.Fatal(err)
}
// we expected to have at least one success
success := false
for _, intf := range intfs {
_, err := BusInfo(intf.Name)
if err == nil {
success = true
}
}
if !success {
t.Fatal("Unable to retrieve bus info from any interface of this system.")
}
}

545
virtcontainers/pkg/hyperstart/hyperstart.go Normal file
View File

@@ -0,0 +1,545 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package hyperstart
import (
"encoding/binary"
"encoding/json"
"fmt"
"math"
"net"
"sync"
"time"
"github.com/sirupsen/logrus"
)
// Control command IDs
// Need to be in sync with hyperstart/src/api.h
const (
Version = "version"
StartPod = "startpod"
DestroyPod = "destroypod"
ExecCmd = "execcmd"
Ready = "ready"
Ack = "ack"
Error = "error"
WinSize = "winsize"
Ping = "ping"
FinishPod = "finishpod"
Next = "next"
WriteFile = "writefile"
ReadFile = "readfile"
NewContainer = "newcontainer"
KillContainer = "killcontainer"
OnlineCPUMem = "onlinecpumem"
SetupInterface = "setupinterface"
SetupRoute = "setuproute"
RemoveContainer = "removecontainer"
PsContainer = "pscontainer"
)
// CodeList is the map making the relation between a string command
// and its corresponding code.
var CodeList = map[string]uint32{
Version: VersionCode,
StartPod: StartPodCode,
DestroyPod: DestroyPodCode,
ExecCmd: ExecCmdCode,
Ready: ReadyCode,
Ack: AckCode,
Error: ErrorCode,
WinSize: WinsizeCode,
Ping: PingCode,
Next: NextCode,
WriteFile: WriteFileCode,
ReadFile: ReadFileCode,
NewContainer: NewContainerCode,
KillContainer: KillContainerCode,
OnlineCPUMem: OnlineCPUMemCode,
SetupInterface: SetupInterfaceCode,
SetupRoute: SetupRouteCode,
RemoveContainer: RemoveContainerCode,
PsContainer: PsContainerCode,
}
// Values related to the communication on control channel.
const (
CtlHdrSize = 8
CtlHdrLenOffset = 4
)
// Values related to the communication on tty channel.
const (
TtyHdrSize = 12
TtyHdrLenOffset = 8
)
type connState struct {
sync.Mutex
opened bool
}
func (c *connState) close() {
c.Lock()
defer c.Unlock()
c.opened = false
}
func (c *connState) open() {
c.Lock()
defer c.Unlock()
c.opened = true
}
func (c *connState) closed() bool {
c.Lock()
defer c.Unlock()
return !c.opened
}
// Hyperstart is the base structure for hyperstart.
type Hyperstart struct {
ctlSerial, ioSerial string
sockType string
ctl, io net.Conn
ctlState, ioState connState
// ctl access is arbitrated by ctlMutex. We can only allow a single
// "transaction" (write command + read answer) at a time
ctlMutex sync.Mutex
ctlMulticast *multicast
ctlChDone chan interface{}
}
var hyperLog = logrus.FieldLogger(logrus.New())
// SetLogger sets the logger for hyperstart package.
func SetLogger(logger logrus.FieldLogger) {
hyperLog = logger.WithField("source", "virtcontainers/hyperstart")
}
// NewHyperstart returns a new hyperstart structure.
func NewHyperstart(ctlSerial, ioSerial, sockType string) *Hyperstart {
return &Hyperstart{
ctlSerial: ctlSerial,
ioSerial: ioSerial,
sockType: sockType,
}
}
// GetCtlSock returns the internal CTL sock.
func (h *Hyperstart) GetCtlSock() net.Conn {
return h.ctl
}
// GetIoSock returns the internal IO sock.
func (h *Hyperstart) GetIoSock() net.Conn {
return h.io
}
// GetCtlSockPath returns the internal CTL sock path.
func (h *Hyperstart) GetCtlSockPath() string {
return h.ctlSerial
}
// GetIoSockPath returns the internal IO sock path.
func (h *Hyperstart) GetIoSockPath() string {
return h.ioSerial
}
// GetSockType returns the internal sock type.
func (h *Hyperstart) GetSockType() string {
return h.sockType
}
// OpenSocketsNoMulticast opens both CTL and IO sockets, without
// starting the multicast.
func (h *Hyperstart) OpenSocketsNoMulticast() error {
var err error
h.ctl, err = net.Dial(h.sockType, h.ctlSerial)
if err != nil {
return err
}
h.ctlState.open()
h.io, err = net.Dial(h.sockType, h.ioSerial)
if err != nil {
h.ctl.Close()
return err
}
h.ioState.open()
return nil
}
// OpenSockets opens both CTL and IO sockets.
func (h *Hyperstart) OpenSockets() error {
if err := h.OpenSocketsNoMulticast(); err != nil {
return err
}
h.ctlChDone = make(chan interface{})
h.ctlMulticast = startCtlMonitor(h.ctl, h.ctlChDone)
return nil
}
// CloseSockets closes both CTL and IO sockets.
func (h *Hyperstart) CloseSockets() error {
if !h.ctlState.closed() {
if h.ctlChDone != nil {
// Wait for the CTL channel to be terminated.
select {
case <-h.ctlChDone:
break
case <-time.After(time.Duration(3) * time.Second):
return fmt.Errorf("CTL channel did not end as expected")
}
}
err := h.ctl.Close()
if err != nil {
return err
}
h.ctlState.close()
}
if !h.ioState.closed() {
err := h.io.Close()
if err != nil {
return err
}
h.ioState.close()
}
h.ctlMulticast = nil
return nil
}
// SetDeadline sets a timeout for CTL connection.
func (h *Hyperstart) SetDeadline(t time.Time) error {
err := h.ctl.SetDeadline(t)
if err != nil {
return err
}
return nil
}
// IsStarted returns about connection status.
func (h *Hyperstart) IsStarted() bool {
ret := false
timeoutDuration := 1 * time.Second
if h.ctlState.closed() {
return ret
}
h.SetDeadline(time.Now().Add(timeoutDuration))
_, err := h.SendCtlMessage(Ping, nil)
if err == nil {
ret = true
}
h.SetDeadline(time.Time{})
if ret == false {
h.CloseSockets()
}
return ret
}
// FormatMessage formats hyperstart messages.
func FormatMessage(payload interface{}) ([]byte, error) {
var payloadSlice []byte
var err error
if payload != nil {
switch p := payload.(type) {
case string:
payloadSlice = []byte(p)
default:
payloadSlice, err = json.Marshal(p)
if err != nil {
return nil, err
}
}
}
return payloadSlice, nil
}
// ReadCtlMessage reads an hyperstart message from conn and returns a decoded message.
//
// This is a low level function, for a full and safe transaction on the
// hyperstart control serial link, use SendCtlMessage.
func ReadCtlMessage(conn net.Conn) (*DecodedMessage, error) {
needRead := CtlHdrSize
length := 0
read := 0
buf := make([]byte, 512)
res := []byte{}
for read < needRead {
want := needRead - read
if want > 512 {
want = 512
}
nr, err := conn.Read(buf[:want])
if err != nil {
return nil, err
}
res = append(res, buf[:nr]...)
read = read + nr
if length == 0 && read >= CtlHdrSize {
length = int(binary.BigEndian.Uint32(res[CtlHdrLenOffset:CtlHdrSize]))
if length > CtlHdrSize {
needRead = length
}
}
}
return &DecodedMessage{
Code: binary.BigEndian.Uint32(res[:CtlHdrLenOffset]),
Message: res[CtlHdrSize:],
}, nil
}
// WriteCtlMessage writes an hyperstart message to conn.
//
// This is a low level function, for a full and safe transaction on the
// hyperstart control serial link, use SendCtlMessage.
func (h *Hyperstart) WriteCtlMessage(conn net.Conn, m *DecodedMessage) error {
length := len(m.Message) + CtlHdrSize
// XXX: Support sending messages by chunks to support messages over
// 10240 bytes. That limit is from hyperstart src/init.c,
// hyper_channel_ops, rbuf_size.
if length > 10240 {
return fmt.Errorf("message too long %d", length)
}
msg := make([]byte, length)
binary.BigEndian.PutUint32(msg[:], uint32(m.Code))
binary.BigEndian.PutUint32(msg[CtlHdrLenOffset:], uint32(length))
copy(msg[CtlHdrSize:], m.Message)
_, err := conn.Write(msg)
if err != nil {
return err
}
return nil
}
// ReadIoMessageWithConn returns data coming from the specified IO channel.
func ReadIoMessageWithConn(conn net.Conn) (*TtyMessage, error) {
needRead := TtyHdrSize
length := 0
read := 0
buf := make([]byte, 512)
res := []byte{}
for read < needRead {
want := needRead - read
if want > 512 {
want = 512
}
nr, err := conn.Read(buf[:want])
if err != nil {
return nil, err
}
res = append(res, buf[:nr]...)
read = read + nr
if length == 0 && read >= TtyHdrSize {
length = int(binary.BigEndian.Uint32(res[TtyHdrLenOffset:TtyHdrSize]))
if length > TtyHdrSize {
needRead = length
}
}
}
return &TtyMessage{
Session: binary.BigEndian.Uint64(res[:TtyHdrLenOffset]),
Message: res[TtyHdrSize:],
}, nil
}
// ReadIoMessage returns data coming from the IO channel.
func (h *Hyperstart) ReadIoMessage() (*TtyMessage, error) {
return ReadIoMessageWithConn(h.io)
}
// SendIoMessageWithConn sends data to the specified IO channel.
func SendIoMessageWithConn(conn net.Conn, ttyMsg *TtyMessage) error {
length := len(ttyMsg.Message) + TtyHdrSize
// XXX: Support sending messages by chunks to support messages over
// 10240 bytes. That limit is from hyperstart src/init.c,
// hyper_channel_ops, rbuf_size.
if length > 10240 {
return fmt.Errorf("message too long %d", length)
}
msg := make([]byte, length)
binary.BigEndian.PutUint64(msg[:], ttyMsg.Session)
binary.BigEndian.PutUint32(msg[TtyHdrLenOffset:], uint32(length))
copy(msg[TtyHdrSize:], ttyMsg.Message)
n, err := conn.Write(msg)
if err != nil {
return err
}
if n != length {
return fmt.Errorf("%d bytes written out of %d expected", n, length)
}
return nil
}
// SendIoMessage sends data to the IO channel.
func (h *Hyperstart) SendIoMessage(ttyMsg *TtyMessage) error {
return SendIoMessageWithConn(h.io, ttyMsg)
}
// CodeFromCmd translates a string command to its corresponding code.
func (h *Hyperstart) CodeFromCmd(cmd string) (uint32, error) {
_, ok := CodeList[cmd]
if ok == false {
return math.MaxUint32, fmt.Errorf("unknown command '%s'", cmd)
}
return CodeList[cmd], nil
}
// CheckReturnedCode ensures we did not receive an ERROR code.
func (h *Hyperstart) CheckReturnedCode(recvMsg *DecodedMessage, expectedCode uint32) error {
if recvMsg.Code != expectedCode {
if recvMsg.Code == ErrorCode {
return fmt.Errorf("ERROR received from VM agent, control msg received : %s", recvMsg.Message)
}
return fmt.Errorf("CMD ID received %d not matching expected %d, control msg received : %s", recvMsg.Code, expectedCode, recvMsg.Message)
}
return nil
}
// WaitForReady waits for a READY message on CTL channel.
func (h *Hyperstart) WaitForReady() error {
if h.ctlMulticast == nil {
return fmt.Errorf("No multicast available for CTL channel")
}
channel, err := h.ctlMulticast.listen("", "", replyType)
if err != nil {
return err
}
msg := <-channel
err = h.CheckReturnedCode(msg, ReadyCode)
if err != nil {
return err
}
return nil
}
// WaitForPAE waits for a PROCESSASYNCEVENT message on CTL channel.
func (h *Hyperstart) WaitForPAE(containerID, processID string) (*PAECommand, error) {
if h.ctlMulticast == nil {
return nil, fmt.Errorf("No multicast available for CTL channel")
}
channel, err := h.ctlMulticast.listen(containerID, processID, eventType)
if err != nil {
return nil, err
}
msg := <-channel
var paeData PAECommand
err = json.Unmarshal(msg.Message, paeData)
if err != nil {
return nil, err
}
return &paeData, nil
}
// SendCtlMessage sends a message to the CTL channel.
//
// This function does a full transaction over the CTL channel: it will rely on the
// multicaster to register a listener reading over the CTL channel. Then it writes
// a command and waits for the multicaster to send hyperstart's answer back before
// it can return.
// Several concurrent calls to SendCtlMessage are allowed, the function ensuring
// proper serialization of the communication by making the listener registration
// and the command writing an atomic operation protected by a mutex.
// Waiting for the reply from multicaster doesn't need to be protected by this mutex.
func (h *Hyperstart) SendCtlMessage(cmd string, data []byte) (*DecodedMessage, error) {
if h.ctlMulticast == nil {
return nil, fmt.Errorf("No multicast available for CTL channel")
}
h.ctlMutex.Lock()
channel, err := h.ctlMulticast.listen("", "", replyType)
if err != nil {
h.ctlMutex.Unlock()
return nil, err
}
code, err := h.CodeFromCmd(cmd)
if err != nil {
h.ctlMutex.Unlock()
return nil, err
}
msgSend := &DecodedMessage{
Code: code,
Message: data,
}
err = h.WriteCtlMessage(h.ctl, msgSend)
if err != nil {
h.ctlMutex.Unlock()
return nil, err
}
h.ctlMutex.Unlock()
msgRecv := <-channel
err = h.CheckReturnedCode(msgRecv, AckCode)
if err != nil {
return nil, err
}
return msgRecv, nil
}

583
virtcontainers/pkg/hyperstart/hyperstart_test.go Normal file
View File

@@ -0,0 +1,583 @@
//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package hyperstart_test
import (
"math"
"net"
"reflect"
"testing"
"time"
. "github.com/containers/virtcontainers/pkg/hyperstart"
"github.com/containers/virtcontainers/pkg/hyperstart/mock"
)
const (
testSockType = "unix"
testSequence = uint64(100)
testMessage = "test_message"
)
func connectHyperstartNoMulticast(h *Hyperstart) error {
return h.OpenSocketsNoMulticast()
}
func connectHyperstart(h *Hyperstart) error {
return h.OpenSockets()
}
func disconnectHyperstart(h *Hyperstart) {
h.CloseSockets()
}
func connectMockHyperstart(t *testing.T, multiCast bool) (*mock.Hyperstart, *Hyperstart, error) {
mockHyper := mock.NewHyperstart(t)
mockHyper.Start()
ctlSock, ioSock := mockHyper.GetSocketPaths()
h := NewHyperstart(ctlSock, ioSock, testSockType)
var err error
if multiCast {
err = connectHyperstart(h)
} else {
err = connectHyperstartNoMulticast(h)
}
if err != nil {
mockHyper.Stop()
return nil, nil, err
}
return mockHyper, h, nil
}
func TestNewHyperstart(t *testing.T) {
ctlSock := "/tmp/test_hyper.sock"
ioSock := "/tmp/test_tty.sock"
sockType := "test_unix"
h := NewHyperstart(ctlSock, ioSock, sockType)
resultCtlSockPath := h.GetCtlSockPath()
resultIoSockPath := h.GetIoSockPath()
resultSockType := h.GetSockType()
if resultCtlSockPath != ctlSock {
t.Fatalf("CTL sock result %s should be the same than %s", resultCtlSockPath, ctlSock)
}
if resultIoSockPath != ioSock {
t.Fatalf("IO sock result %s should be the same than %s", resultIoSockPath, ioSock)
}
if resultSockType != sockType {
t.Fatalf("Sock type result %s should be the same than %s", resultSockType, sockType)
}
}
func TestOpenSockets(t *testing.T) {
mockHyper := mock.NewHyperstart(t)
mockHyper.Start()
ctlSock, ioSock := mockHyper.GetSocketPaths()
h := NewHyperstart(ctlSock, ioSock, testSockType)
err := h.OpenSockets()
if err != nil {
mockHyper.Stop()
t.Fatal()
}
mockHyper.Stop()
disconnectHyperstart(h)
}
func TestCloseSockets(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
mockHyper.Stop()
err = h.CloseSockets()
if err != nil {
t.Fatal()
}
}
func TestSetDeadline(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, false)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
timeoutDuration := 1 * time.Second
err = h.SetDeadline(time.Now().Add(timeoutDuration))
if err != nil {
t.Fatal()
}
mockHyper.SendMessage(ReadyCode, []byte{})
buf := make([]byte, 512)
_, err = h.GetCtlSock().Read(buf)
if err != nil {
t.Fatal()
}
err = h.SetDeadline(time.Now().Add(timeoutDuration))
if err != nil {
t.Fatal()
}
time.Sleep(timeoutDuration)
_, err = h.GetCtlSock().Read(buf)
netErr, ok := err.(net.Error)
if ok && netErr.Timeout() == false {
t.Fatal()
}
}
func TestIsStartedFalse(t *testing.T) {
h := &Hyperstart{}
if h.IsStarted() == true {
t.Fatal()
}
}
func TestIsStartedTrue(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
if h.IsStarted() == false {
t.Fatal()
}
}
func testFormatMessage(t *testing.T, payload interface{}, expected []byte) {
res, err := FormatMessage(payload)
if err != nil {
t.Fatal()
}
if reflect.DeepEqual(res, expected) == false {
t.Fatal()
}
}
func TestFormatMessageFromString(t *testing.T) {
payload := testMessage
expectedOut := []byte(payload)
testFormatMessage(t, payload, expectedOut)
}
type TestStruct struct {
FieldString string `json:"fieldString"`
FieldInt int `json:"fieldInt"`
}
func TestFormatMessageFromStruct(t *testing.T) {
payload := TestStruct{
FieldString: "test_string",
FieldInt: 100,
}
expectedOut := []byte("{\"fieldString\":\"test_string\",\"fieldInt\":100}")
testFormatMessage(t, payload, expectedOut)
}
func TestReadCtlMessage(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, false)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
expected := &DecodedMessage{
Code: ReadyCode,
Message: []byte{},
}
mockHyper.SendMessage(int(expected.Code), expected.Message)
reply, err := ReadCtlMessage(h.GetCtlSock())
if err != nil {
t.Fatal()
}
if reflect.DeepEqual(reply, expected) == false {
t.Fatal()
}
}
func TestWriteCtlMessage(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, false)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
msg := DecodedMessage{
Code: PingCode,
Message: []byte{},
}
err = h.WriteCtlMessage(h.GetCtlSock(), &msg)
if err != nil {
t.Fatal()
}
for {
reply, err := ReadCtlMessage(h.GetCtlSock())
if err != nil {
t.Fatal()
}
if reply.Code == NextCode {
continue
}
err = h.CheckReturnedCode(reply, AckCode)
if err != nil {
t.Fatal()
}
break
}
msgs := mockHyper.GetLastMessages()
if msgs == nil {
t.Fatal()
}
if msgs[0].Code != msg.Code || string(msgs[0].Message) != string(msg.Message) {
t.Fatal()
}
}
func TestReadIoMessage(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
mockHyper.SendIo(testSequence, []byte(testMessage))
msg, err := h.ReadIoMessage()
if err != nil {
t.Fatal()
}
if msg.Session != testSequence || string(msg.Message) != testMessage {
t.Fatal()
}
}
func TestReadIoMessageWithConn(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
mockHyper.SendIo(testSequence, []byte(testMessage))
msg, err := ReadIoMessageWithConn(h.GetIoSock())
if err != nil {
t.Fatal()
}
if msg.Session != testSequence || string(msg.Message) != testMessage {
t.Fatal()
}
}
func TestSendIoMessage(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
msg := &TtyMessage{
Session: testSequence,
Message: []byte(testMessage),
}
err = h.SendIoMessage(msg)
if err != nil {
t.Fatal()
}
buf := make([]byte, 512)
n, seqRecv := mockHyper.ReadIo(buf)
if seqRecv != testSequence || string(buf[TtyHdrSize:n]) != testMessage {
t.Fatal()
}
}
func TestSendIoMessageWithConn(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
msg := &TtyMessage{
Session: testSequence,
Message: []byte(testMessage),
}
err = SendIoMessageWithConn(h.GetIoSock(), msg)
if err != nil {
t.Fatal()
}
buf := make([]byte, 512)
n, seqRecv := mockHyper.ReadIo(buf)
if seqRecv != testSequence || string(buf[TtyHdrSize:n]) != testMessage {
t.Fatal()
}
}
func testCodeFromCmd(t *testing.T, cmd string, expected uint32) {
h := &Hyperstart{}
code, err := h.CodeFromCmd(cmd)
if err != nil || code != expected {
t.Fatal()
}
}
func TestCodeFromCmdVersion(t *testing.T) {
testCodeFromCmd(t, Version, VersionCode)
}
func TestCodeFromCmdStartPod(t *testing.T) {
testCodeFromCmd(t, StartPod, StartPodCode)
}
func TestCodeFromCmdDestroyPod(t *testing.T) {
testCodeFromCmd(t, DestroyPod, DestroyPodCode)
}
func TestCodeFromCmdExecCmd(t *testing.T) {
testCodeFromCmd(t, ExecCmd, ExecCmdCode)
}
func TestCodeFromCmdReady(t *testing.T) {
testCodeFromCmd(t, Ready, ReadyCode)
}
func TestCodeFromCmdAck(t *testing.T) {
testCodeFromCmd(t, Ack, AckCode)
}
func TestCodeFromCmdError(t *testing.T) {
testCodeFromCmd(t, Error, ErrorCode)
}
func TestCodeFromCmdWinSize(t *testing.T) {
testCodeFromCmd(t, WinSize, WinsizeCode)
}
func TestCodeFromCmdPing(t *testing.T) {
testCodeFromCmd(t, Ping, PingCode)
}
func TestCodeFromCmdNext(t *testing.T) {
testCodeFromCmd(t, Next, NextCode)
}
func TestCodeFromCmdWriteFile(t *testing.T) {
testCodeFromCmd(t, WriteFile, WriteFileCode)
}
func TestCodeFromCmdReadFile(t *testing.T) {
testCodeFromCmd(t, ReadFile, ReadFileCode)
}
func TestCodeFromCmdNewContainer(t *testing.T) {
testCodeFromCmd(t, NewContainer, NewContainerCode)
}
func TestCodeFromCmdKillContainer(t *testing.T) {
testCodeFromCmd(t, KillContainer, KillContainerCode)
}
func TestCodeFromCmdOnlineCPUMem(t *testing.T) {
testCodeFromCmd(t, OnlineCPUMem, OnlineCPUMemCode)
}
func TestCodeFromCmdSetupInterface(t *testing.T) {
testCodeFromCmd(t, SetupInterface, SetupInterfaceCode)
}
func TestCodeFromCmdSetupRoute(t *testing.T) {
testCodeFromCmd(t, SetupRoute, SetupRouteCode)
}
func TestCodeFromCmdRemoveContainer(t *testing.T) {
testCodeFromCmd(t, RemoveContainer, RemoveContainerCode)
}
func TestCodeFromCmdUnknown(t *testing.T) {
h := &Hyperstart{}
code, err := h.CodeFromCmd("unknown")
if err == nil || code != math.MaxUint32 {
t.Fatal()
}
}
func testCheckReturnedCode(t *testing.T, recvMsg *DecodedMessage, refCode uint32) {
h := &Hyperstart{}
err := h.CheckReturnedCode(recvMsg, refCode)
if err != nil {
t.Fatal()
}
}
func TestCheckReturnedCodeList(t *testing.T) {
for _, code := range CodeList {
recvMsg := DecodedMessage{Code: code}
testCheckReturnedCode(t, &recvMsg, code)
}
}
func testCheckReturnedCodeFailure(t *testing.T, recvMsg *DecodedMessage, refCode uint32) {
h := &Hyperstart{}
err := h.CheckReturnedCode(recvMsg, refCode)
if err == nil {
t.Fatal()
}
}
func TestCheckReturnedCodeListWrong(t *testing.T) {
for _, code := range CodeList {
msg := DecodedMessage{Code: code}
if code != ReadyCode {
testCheckReturnedCodeFailure(t, &msg, ReadyCode)
} else {
testCheckReturnedCodeFailure(t, &msg, PingCode)
}
}
}
func TestWaitForReady(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
mockHyper.SendMessage(int(ReadyCode), []byte{})
err = h.WaitForReady()
if err != nil {
t.Fatal()
}
}
func TestWaitForReadyError(t *testing.T) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
mockHyper.SendMessage(int(ErrorCode), []byte{})
err = h.WaitForReady()
if err == nil {
t.Fatal()
}
}
var cmdList = []string{
Version,
StartPod,
DestroyPod,
ExecCmd,
Ready,
Ack,
Error,
WinSize,
Ping,
Next,
NewContainer,
KillContainer,
OnlineCPUMem,
SetupInterface,
SetupRoute,
RemoveContainer,
}
func testSendCtlMessage(t *testing.T, cmd string) {
mockHyper, h, err := connectMockHyperstart(t, true)
if err != nil {
t.Fatal()
}
defer disconnectHyperstart(h)
defer mockHyper.Stop()
msg, err := h.SendCtlMessage(cmd, []byte{})
if err != nil {
t.Fatal()
}
if msg.Code != AckCode {
t.Fatal()
}
}
func TestSendCtlMessage(t *testing.T) {
for _, cmd := range cmdList {
testSendCtlMessage(t, cmd)
}
}

408
virtcontainers/pkg/hyperstart/mock/hyperstart.go Normal file
View File

@@ -0,0 +1,408 @@
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mock
import (
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"os"
"path/filepath"
"sync"
"testing"
hyper "github.com/containers/virtcontainers/pkg/hyperstart"
"github.com/stretchr/testify/assert"
)
// Control command string IDs
const (
Version = "version"
StartPod = "startpod"
DestroyPod = "destroypod"
ExecCmd = "execcmd"
Ready = "ready"
Ack = "ack"
Error = "error"
WinSize = "winsize"
Ping = "ping"
FinishPod = "finishpod"
Next = "next"
WriteFile = "writefile"
ReadFile = "readfile"
NewContainer = "newcontainer"
KillContainer = "killcontainer"
RemoveContainer = "removecontainer"
OnlineCPUMem = "onlinecpumem"
SetupInterface = "setupinterface"
SetupRoute = "setuproute"
)
var codeList = map[int]string{
hyper.VersionCode: Version,
hyper.StartPodCode: StartPod,
hyper.DestroyPodCode: DestroyPod,
hyper.ExecCmdCode: ExecCmd,
hyper.ReadyCode: Ready,
hyper.AckCode: Ack,
hyper.ErrorCode: Error,
hyper.WinsizeCode: WinSize,
hyper.PingCode: Ping,
hyper.NextCode: Next,
hyper.WriteFileCode: WriteFile,
hyper.ReadFileCode: ReadFile,
hyper.NewContainerCode: NewContainer,
hyper.KillContainerCode: KillContainer,
hyper.OnlineCPUMemCode: OnlineCPUMem,
hyper.SetupInterfaceCode: SetupInterface,
hyper.SetupRouteCode: SetupRoute,
hyper.RemoveContainerCode: RemoveContainer,
}
// Hyperstart is an object mocking the hyperstart agent.
type Hyperstart struct {
t *testing.T
ctlSocketPath, ioSocketPath string
ctlListener, ioListener *net.UnixListener
ctl, io net.Conn
// Start() will launch two goroutines to accept connections on the ctl
// and io sockets. Those goroutine will exit once the first connection
// is accepted or when the listening socket is closed. wgConnected can
// be used to make sure we've accepted connections to both sockets
wgConnected sync.WaitGroup
// We then have two other goroutines to handle communication on those
// sockets.
wg sync.WaitGroup
// Keep the list of messages received by hyperstart, older first, for
// later inspection with GetLastMessages()
lastMessages []hyper.DecodedMessage
}
func newMessageList() []hyper.DecodedMessage {
return make([]hyper.DecodedMessage, 0, 10)
}
// NewHyperstart creates a new hyperstart instance.
func NewHyperstart(t *testing.T) *Hyperstart {
dir := os.TempDir()
ctlSocketPath := filepath.Join(dir, "mock.hyper."+nextSuffix()+".0.sock")
ioSocketPath := filepath.Join(dir, "mock.hyper."+nextSuffix()+".1.sock")
return &Hyperstart{
t: t,
ctlSocketPath: ctlSocketPath,
ioSocketPath: ioSocketPath,
lastMessages: newMessageList(),
}
}
// GetSocketPaths returns the ctl and io socket paths, respectively
func (h *Hyperstart) GetSocketPaths() (string, string) {
return h.ctlSocketPath, h.ioSocketPath
}
// GetLastMessages returns list of messages received by hyperstart, older
// first. This function only returns the messages:
// - since Start on the first invocation
// - since the last GetLastMessages for subsequent invocations
func (h *Hyperstart) GetLastMessages() []hyper.DecodedMessage {
msgs := h.lastMessages
h.lastMessages = newMessageList()
return msgs
}
func (h *Hyperstart) log(s string) {
h.logf("%s\n", s)
}
func (h *Hyperstart) logf(format string, args ...interface{}) {
h.t.Logf("[hyperstart] "+format, args...)
}
func (h *Hyperstart) logData(data []byte) {
h.t.Log(hex.Dump(data))
}
//
// ctl channel
//
const ctlHeaderSize = 8
func (h *Hyperstart) writeCtl(data []byte) error {
h.wgConnected.Wait()
n, err := h.ctl.Write(data)
if err != nil {
return fmt.Errorf("Connection broken, cannot send data")
}
assert.Equal(h.t, n, len(data))
return nil
}
// SendMessage makes hyperstart send the hyper command cmd along with optional
// data on the control channel
func (h *Hyperstart) SendMessage(cmd int, data []byte) {
length := ctlHeaderSize + len(data)
header := make([]byte, ctlHeaderSize)
binary.BigEndian.PutUint32(header[:], uint32(cmd))
binary.BigEndian.PutUint32(header[4:], uint32(length))
err := h.writeCtl(header)
if err != nil {
return
}
if len(data) == 0 {
return
}
h.writeCtl(data)
}
func (h *Hyperstart) readCtl(data []byte) error {
h.wgConnected.Wait()
n, err := h.ctl.Read(data)
if err != nil {
return err
}
assert.Equal(h.t, n, len(data))
return nil
}
func (h *Hyperstart) ackData(nBytes int) {
data := make([]byte, 4)
binary.BigEndian.PutUint32(data[:], uint32(nBytes))
h.SendMessage(hyper.NextCode, data)
}
func (h *Hyperstart) readMessage() (int, []byte, error) {
buf := make([]byte, ctlHeaderSize)
if err := h.readCtl(buf); err != nil {
return -1, buf, err
}
h.ackData(len(buf))
cmd := int(binary.BigEndian.Uint32(buf[:4]))
length := int(binary.BigEndian.Uint32(buf[4:8]))
assert.True(h.t, length >= 8)
length -= 8
if length == 0 {
return cmd, nil, nil
}
data := make([]byte, length)
if err := h.readCtl(data); err != nil {
return -1, buf, err
}
h.ackData(len(data))
return cmd, data, nil
}
func cmdToString(cmd int) (string, error) {
_, ok := codeList[cmd]
if ok == false {
return "", fmt.Errorf("unknown command '%d'", cmd)
}
return codeList[cmd], nil
}
func (h *Hyperstart) handleCtl() {
for {
cmd, data, err := h.readMessage()
if err != nil {
break
}
cmdName, err := cmdToString(cmd)
assert.Nil(h.t, err)
h.logf("ctl: --> command %s, payload_len=%d\n", cmdName, len(data))
if len(data) != 0 {
h.logData(data)
}
h.lastMessages = append(h.lastMessages, hyper.DecodedMessage{
Code: uint32(cmd),
Message: data,
})
// answer back with the message exit status
// XXX: may be interesting to be able to configure the mock
// hyperstart to fail and test the reaction of proxy/clients
h.logf("ctl: <-- command %s executed successfully\n", cmdName)
h.SendMessage(hyper.AckCode, nil)
}
h.wg.Done()
}
//
// io channel
//
const ioHeaderSize = 12
func (h *Hyperstart) writeIo(data []byte) {
h.wgConnected.Wait()
n, err := h.io.Write(data)
assert.Nil(h.t, err)
assert.Equal(h.t, n, len(data))
}
// SendIo sends a packet of I/O data to a client connected the I/O channel.
// Multiple I/O streams are multiplexed on that channel. seq specifies which
// steam the data belongs to.
func (h *Hyperstart) SendIo(seq uint64, data []byte) {
length := ioHeaderSize + len(data)
header := make([]byte, ioHeaderSize)
h.logf("io: <-- writing %d bytes for seq %d\n", len(data), seq)
binary.BigEndian.PutUint64(header[:], uint64(seq))
binary.BigEndian.PutUint32(header[8:], uint32(length))
h.writeIo(header)
if len(data) == 0 {
return
}
h.writeIo(data)
}
// SendIoString sends a string a client connected the I/O channel.
// Multiple I/O streams are multiplexed on that channel. seq specifies which
// steam the data belongs to.
func (h *Hyperstart) SendIoString(seq uint64, data string) {
h.SendIo(seq, []byte(data))
}
// CloseIo closes the I/O stream specified by seq.
func (h *Hyperstart) CloseIo(seq uint64) {
h.SendIo(seq, nil)
}
// SendExitStatus sends the exit status on the I/O streams specified by seq.
// The exit status should only be sent after the stream has been closed with
// CloseIo.
func (h *Hyperstart) SendExitStatus(seq uint64, exitStatus uint8) {
status := []byte{exitStatus}
h.SendIo(seq, status)
}
// ReadIo reads data that has been sent on the I/O channel by a client. It
// returns the full packet (header & data) as well as the seq number decoded
// from the header.
func (h *Hyperstart) ReadIo(buf []byte) (n int, seq uint64) {
h.wgConnected.Wait()
n, err := h.io.Read(buf)
assert.Nil(h.t, err)
seq = binary.BigEndian.Uint64(buf[:8])
return
}
type acceptCb func(c net.Conn)
func (h *Hyperstart) startListening(path string, cb acceptCb) *net.UnixListener {
addr := &net.UnixAddr{Name: path, Net: "unix"}
l, err := net.ListenUnix("unix", addr)
assert.Nil(h.t, err)
go func() {
h.logf("%s: waiting for connection\n", path)
c, err := l.Accept()
if err != nil {
cb(nil)
return
}
cb(c)
h.logf("%s: accepted connection\n", path)
}()
return l
}
// Start will
// Once finished with the Hyperstart object, Close must be called.
func (h *Hyperstart) Start() {
h.log("start")
h.wgConnected.Add(1)
h.wgConnected.Add(1)
h.ctlListener = h.startListening(h.ctlSocketPath, func(s net.Conn) {
// a client is now connected to the ctl socket
h.ctl = s
// Close() was called before we had a chance to accept a
// connection.
if s == nil {
h.wgConnected.Done()
return
}
// start the goroutine that will handle the ctl socket
h.wg.Add(1)
go h.handleCtl()
// we need signal wgConnected late, so wg.Add(1) is done before
// the wg.Wait() in Close()
// See https://golang.org/pkg/sync/#WaitGroup.Wait:
// "Note that calls with a positive delta that occur when the
// counter is zero must happen before a Wait"
h.wgConnected.Done()
})
h.ioListener = h.startListening(h.ioSocketPath, func(s net.Conn) {
// a client is now connected to the ctl socket
h.io = s
h.wgConnected.Done()
})
}
// Stop closes all internal resources and waits for goroutines started by Start
// to finish. Stop shouldn't be called if Start hasn't been called.
func (h *Hyperstart) Stop() {
h.wgConnected.Wait()
h.ctl.Close()
h.io.Close()
h.ctlListener.Close()
h.ioListener.Close()
h.wg.Wait()
os.Remove(h.ctlSocketPath)
os.Remove(h.ioSocketPath)
h.log("stopped")
}

37
virtcontainers/pkg/hyperstart/mock/misc.go Normal file
View File

@@ -0,0 +1,37 @@
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mock
import (
"fmt"
"os"
"path/filepath"
)
// GetTmpPath will return a filename suitable for a tempory file according to
// the format string given in argument. The format string must contain a single
// %s which will be replaced by a random string. Eg.:
//
// GetTmpPath("test.foo.%s.sock")
//
// will return something like:
//
// "/tmp/test.foo.832222621.sock"
func GetTmpPath(format string) string {
filename := fmt.Sprintf(format, nextSuffix())
dir := os.TempDir()
return filepath.Join(dir, filename)
}

42
virtcontainers/pkg/hyperstart/mock/rand.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mock
import (
"os"
"strconv"
"sync"
"time"
)
// Taken from https://golang.org/src/io/ioutil/tempfile.go?s=#L19
var rand uint32
var randmu sync.Mutex
func reseed() uint32 {
return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}
func nextSuffix() string {
randmu.Lock()
r := rand
if r == 0 {
r = reseed()
}
r = r*1664525 + 1013904223 // constants from Numerical Recipes
rand = r
randmu.Unlock()
return strconv.Itoa(int(1e9 + r%1e9))[1:]
}

175
virtcontainers/pkg/hyperstart/multicast.go Normal file
View File

@@ -0,0 +1,175 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package hyperstart
import (
"encoding/json"
"fmt"
"net"
"sync"
)
type ctlDataType string
const (
eventType ctlDataType = "ctlEvent"
replyType ctlDataType = "ctlReply"
)
type multicast struct {
bufReplies []*DecodedMessage
reply []chan *DecodedMessage
event map[string]chan *DecodedMessage
ctl net.Conn
sync.Mutex
}
func newMulticast(ctlConn net.Conn) *multicast {
return &multicast{
bufReplies: []*DecodedMessage{},
reply: []chan *DecodedMessage{},
event: make(map[string]chan *DecodedMessage),
ctl: ctlConn,
}
}
func startCtlMonitor(ctlConn net.Conn, done chan<- interface{}) *multicast {
ctlMulticast := newMulticast(ctlConn)
go func() {
for {
msg, err := ReadCtlMessage(ctlMulticast.ctl)
if err != nil {
hyperLog.Infof("Read on CTL channel ended: %s", err)
break
}
err = ctlMulticast.write(msg)
if err != nil {
hyperLog.Errorf("Multicaster write error: %s", err)
break
}
}
close(done)
}()
return ctlMulticast
}
func (m *multicast) buildEventID(containerID, processID string) string {
return fmt.Sprintf("%s-%s", containerID, processID)
}
func (m *multicast) sendEvent(msg *DecodedMessage) error {
var paeData PAECommand
err := json.Unmarshal(msg.Message, paeData)
if err != nil {
return err
}
uniqueID := m.buildEventID(paeData.Container, paeData.Process)
channel, exist := m.event[uniqueID]
if !exist {
return nil
}
channel <- msg
delete(m.event, uniqueID)
return nil
}
func (m *multicast) sendReply(msg *DecodedMessage) error {
m.Lock()
if len(m.reply) == 0 {
m.bufReplies = append(m.bufReplies, msg)
m.Unlock()
return nil
}
replyChannel := m.reply[0]
m.reply = m.reply[1:]
m.Unlock()
// The current reply channel has been removed from the list, that's why
// we can be out of the mutex to send through that channel. Indeed, there
// is no risk that someone else tries to write on this channel.
replyChannel <- msg
return nil
}
func (m *multicast) processBufferedReply(channel chan *DecodedMessage) {
m.Lock()
if len(m.bufReplies) == 0 {
m.reply = append(m.reply, channel)
m.Unlock()
return
}
msg := m.bufReplies[0]
m.bufReplies = m.bufReplies[1:]
m.Unlock()
// The current buffered reply message has been removed from the list, and
// the channel have not been added to the reply list, that's why we can be
// out of the mutex to send the buffered message through that channel.
// There is no risk that someone else tries to write this message on another
// channel, or another message on this channel.
channel <- msg
}
func (m *multicast) write(msg *DecodedMessage) error {
switch msg.Code {
case NextCode:
return nil
case ProcessAsyncEventCode:
return m.sendEvent(msg)
default:
return m.sendReply(msg)
}
}
func (m *multicast) listen(containerID, processID string, dataType ctlDataType) (chan *DecodedMessage, error) {
switch dataType {
case replyType:
newChan := make(chan *DecodedMessage)
go m.processBufferedReply(newChan)
return newChan, nil
case eventType:
uniqueID := m.buildEventID(containerID, processID)
_, exist := m.event[uniqueID]
if exist {
return nil, fmt.Errorf("Channel already assigned for ID %s", uniqueID)
}
m.event[uniqueID] = make(chan *DecodedMessage)
return m.event[uniqueID], nil
default:
return nil, fmt.Errorf("Unknown data type: %s", dataType)
}
}

265
virtcontainers/pkg/hyperstart/types.go Normal file
View File

@@ -0,0 +1,265 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package hyperstart
import (
"syscall"
)
// Defines all available commands to communicate with hyperstart agent.
const (
VersionCode = iota
StartPodCode
GetPodDeprecatedCode
StopPodDeprecatedCode
DestroyPodCode
RestartContainerDeprecatedCode
ExecCmdCode
FinishCmdDeprecatedCode
ReadyCode
AckCode
ErrorCode
WinsizeCode
PingCode
FinishPodDeprecatedCode
NextCode
WriteFileCode
ReadFileCode
NewContainerCode
KillContainerCode
OnlineCPUMemCode
SetupInterfaceCode
SetupRouteCode
RemoveContainerCode
PsContainerCode
ProcessAsyncEventCode
)
// FileCommand is the structure corresponding to the format expected by
// hyperstart to interact with files.
type FileCommand struct {
Container string `json:"container"`
File string `json:"file"`
}
// KillCommand is the structure corresponding to the format expected by
// hyperstart to kill a container on the guest.
type KillCommand struct {
Container string `json:"container"`
Signal syscall.Signal `json:"signal"`
AllProcesses bool `json:"allProcesses"`
}
// ExecCommand is the structure corresponding to the format expected by
// hyperstart to execute a command on the guest.
type ExecCommand struct {
Container string `json:"container,omitempty"`
Process Process `json:"process"`
}
// RemoveCommand is the structure corresponding to the format expected by
// hyperstart to remove a container on the guest.
type RemoveCommand struct {
Container string `json:"container"`
}
// PsCommand is the structure corresponding to the format expected by
// hyperstart to list processes of a container on the guest.
type PsCommand struct {
Container string `json:"container"`
Format string `json:"format"`
PsArgs []string `json:"psargs"`
}
// PAECommand is the structure hyperstart can expects to
// receive after a process has been started/executed on a container.
type PAECommand struct {
Container string `json:"container"`
Process string `json:"process"`
Event string `json:"event"`
Info string `json:"info,omitempty"`
Status int `json:"status,omitempty"`
}
// DecodedMessage is the structure holding messages coming from CTL channel.
type DecodedMessage struct {
Code uint32
Message []byte
}
// TtyMessage is the structure holding messages coming from TTY channel.
type TtyMessage struct {
Session uint64
Message []byte
}
// WindowSizeMessage is the structure corresponding to the format expected by
// hyperstart to resize a container's window.
type WindowSizeMessage struct {
Container string `json:"container"`
Process string `json:"process"`
Row uint16 `json:"row"`
Column uint16 `json:"column"`
}
// VolumeDescriptor describes a volume related to a container.
type VolumeDescriptor struct {
Device string `json:"device"`
Addr string `json:"addr,omitempty"`
Mount string `json:"mount"`
Fstype string `json:"fstype,omitempty"`
ReadOnly bool `json:"readOnly"`
DockerVolume bool `json:"dockerVolume"`
}
// FsmapDescriptor describes a filesystem map related to a container.
type FsmapDescriptor struct {
Source string `json:"source"`
Path string `json:"path"`
ReadOnly bool `json:"readOnly"`
DockerVolume bool `json:"dockerVolume"`
AbsolutePath bool `json:"absolutePath"`
SCSIAddr string `json:"scsiAddr"`
}
// EnvironmentVar holds an environment variable and its value.
type EnvironmentVar struct {
Env string `json:"env"`
Value string `json:"value"`
}
// Rlimit describes a resource limit.
type Rlimit struct {
// Type of the rlimit to set
Type string `json:"type"`
// Hard is the hard limit for the specified type
Hard uint64 `json:"hard"`
// Soft is the soft limit for the specified type
Soft uint64 `json:"soft"`
}
// Capabilities specify the capabilities to keep when executing the process inside the container.
type Capabilities struct {
// Bounding is the set of capabilities checked by the kernel.
Bounding []string `json:"bounding"`
// Effective is the set of capabilities checked by the kernel.
Effective []string `json:"effective"`
// Inheritable is the capabilities preserved across execve.
Inheritable []string `json:"inheritable"`
// Permitted is the limiting superset for effective capabilities.
Permitted []string `json:"permitted"`
// Ambient is the ambient set of capabilities that are kept.
Ambient []string `json:"ambient"`
}
// Process describes a process running on a container inside a pod.
type Process struct {
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
AdditionalGroups []string `json:"additionalGroups,omitempty"`
// Terminal creates an interactive terminal for the process.
Terminal bool `json:"terminal"`
// Sequeue number for stdin and stdout
Stdio uint64 `json:"stdio,omitempty"`
// Sequeue number for stderr if it is not shared with stdout
Stderr uint64 `json:"stderr,omitempty"`
// Args specifies the binary and arguments for the application to execute.
Args []string `json:"args"`
// Envs populates the process environment for the process.
Envs []EnvironmentVar `json:"envs,omitempty"`
// Workdir is the current working directory for the process and must be
// relative to the container's root.
Workdir string `json:"workdir"`
// Rlimits specifies rlimit options to apply to the process.
Rlimits []Rlimit `json:"rlimits,omitempty"`
// NoNewPrivileges indicates that the process should not gain any additional privileges
NoNewPrivileges bool `json:"noNewPrivileges"`
// Capabilities specifies the sets of capabilities for the process(es) inside the container.
Capabilities Capabilities `json:"capabilities"`
}
// SystemMountsInfo describes additional information for system mounts that the agent
// needs to handle
type SystemMountsInfo struct {
// Indicates if /dev has been passed as a bind mount for the host /dev
BindMountDev bool `json:"bindMountDev"`
// Size of /dev/shm assigned on the host.
DevShmSize int `json:"devShmSize"`
}
// Constraints describes the constrains for a container
type Constraints struct {
// CPUQuota specifies the total amount of time in microseconds
// The number of microseconds per CPUPeriod that the container is guaranteed CPU access
CPUQuota int64
// CPUPeriod specifies the CPU CFS scheduler period of time in microseconds
CPUPeriod uint64
// CPUShares specifies container's weight vs. other containers
CPUShares uint64
}
// Container describes a container running on a pod.
type Container struct {
ID string `json:"id"`
Rootfs string `json:"rootfs"`
Fstype string `json:"fstype,omitempty"`
Image string `json:"image"`
SCSIAddr string `json:"scsiAddr,omitempty"`
Volumes []*VolumeDescriptor `json:"volumes,omitempty"`
Fsmap []*FsmapDescriptor `json:"fsmap,omitempty"`
Sysctl map[string]string `json:"sysctl,omitempty"`
Process *Process `json:"process"`
RestartPolicy string `json:"restartPolicy"`
Initialize bool `json:"initialize"`
SystemMountsInfo SystemMountsInfo `json:"systemMountsInfo"`
Constraints Constraints `json:"constraints"`
}
// IPAddress describes an IP address and its network mask.
type IPAddress struct {
IPAddress string `json:"ipAddress"`
NetMask string `json:"netMask"`
}
// NetworkIface describes a network interface to setup on the host.
type NetworkIface struct {
Device string `json:"device,omitempty"`
NewDevice string `json:"newDeviceName,omitempty"`
IPAddresses []IPAddress `json:"ipAddresses"`
MTU int `json:"mtu"`
MACAddr string `json:"macAddr"`
}
// Route describes a route to setup on the host.
type Route struct {
Dest string `json:"dest"`
Gateway string `json:"gateway,omitempty"`
Device string `json:"device,omitempty"`
}
// Pod describes the pod configuration to start inside the VM.
type Pod struct {
Hostname string `json:"hostname"`
Containers []Container `json:"containers,omitempty"`
Interfaces []NetworkIface `json:"interfaces,omitempty"`
DNS []string `json:"dns,omitempty"`
Routes []Route `json:"routes,omitempty"`
ShareDir string `json:"shareDir"`
}

482
virtcontainers/pkg/mock/cc_proxy_mock.go Normal file
View File

@@ -0,0 +1,482 @@
// Copyright (c) 2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mock
import (
"encoding/json"
"errors"
"fmt"
"net"
"os"
"sync"
"testing"
"github.com/clearcontainers/proxy/api"
"github.com/stretchr/testify/assert"
)
const testContainerid = "123456789"
const testToken = "pF56IaDpuax6hihJ5PneB8JypqmOvjkqY-wKGVYqgIM="
// CCProxyMock is an object mocking clearcontainers Proxy
type CCProxyMock struct {
t *testing.T
wg sync.WaitGroup
connectionPath string
// proxy socket
listener net.Listener
// single client to serve
cl net.Conn
//token to be used for the connection
token string
lastStdinStream []byte
ShimConnected chan bool
Signal chan ShimSignal
ShimDisconnected chan bool
StdinReceived chan bool
}
// NewCCProxyMock creates a hyperstart instance
func NewCCProxyMock(t *testing.T, path string) *CCProxyMock {
return &CCProxyMock{
t: t,
connectionPath: path,
lastStdinStream: nil,
Signal: make(chan ShimSignal, 5),
ShimConnected: make(chan bool),
ShimDisconnected: make(chan bool),
StdinReceived: make(chan bool),
token: testToken,
}
}
// GetProxyToken returns the token that mock proxy uses
// to verify its client connection
func (proxy *CCProxyMock) GetProxyToken() string {
return proxy.token
}
func newSignalList() []ShimSignal {
return make([]ShimSignal, 0, 5)
}
// GetLastStdinStream returns the last received stdin stream
func (proxy *CCProxyMock) GetLastStdinStream() []byte {
return proxy.lastStdinStream
}
func (proxy *CCProxyMock) log(s string) {
proxy.logF("%s\n", s)
}
func (proxy *CCProxyMock) logF(format string, args ...interface{}) {
proxy.t.Logf("[CCProxyMock] "+format, args...)
}
type client struct {
proxy *CCProxyMock
conn net.Conn
}
// ConnectShim payload defined here, as it has not been defined
// in proxy api package yet
type ConnectShim struct {
Token string `json:"token"`
}
// ShimSignal is the struct used to represent the signal received from the shim
type ShimSignal struct {
Signal int `json:"signalNumber"`
Row int `json:"rows,omitempty"`
Column int `json:"columns,omitempty"`
}
func connectShimHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
payload := ConnectShim{}
err := json.Unmarshal(data, &payload)
assert.Nil(proxy.t, err)
if payload.Token != proxy.token {
response.SetErrorMsg("Invalid Token")
}
proxy.logF("ConnectShim(token=%s)", payload.Token)
response.AddResult("version", api.Version)
proxy.ShimConnected <- true
}
func signalShimHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
signalPayload := ShimSignal{}
err := json.Unmarshal(data, &signalPayload)
assert.Nil(proxy.t, err)
proxy.logF("CCProxyMock received signal: %v", signalPayload)
proxy.Signal <- signalPayload
}
func disconnectShimHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
proxy.log("Client sent DisconnectShim Command")
proxy.ShimDisconnected <- true
}
func stdinShimHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
proxy.lastStdinStream = data
proxy.StdinReceived <- true
}
func registerVMHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
proxy.log("Register VM")
payload := api.RegisterVM{}
if err := json.Unmarshal(data, &payload); err != nil {
response.SetError(err)
return
}
// Generate fake tokens
var tokens []string
for i := 0; i < payload.NumIOStreams; i++ {
tokens = append(tokens, fmt.Sprintf("%d", i))
}
io := &api.IOResponse{
Tokens: tokens,
}
response.AddResult("io", io)
}
func unregisterVMHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
proxy.log("Unregister VM")
}
func attachVMHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
proxy.log("Attach VM")
payload := api.AttachVM{}
if err := json.Unmarshal(data, &payload); err != nil {
response.SetError(err)
return
}
// Generate fake tokens
var tokens []string
for i := 0; i < payload.NumIOStreams; i++ {
tokens = append(tokens, fmt.Sprintf("%d", i))
}
io := &api.IOResponse{
Tokens: tokens,
}
response.AddResult("io", io)
}
func hyperCmdHandler(data []byte, userData interface{}, response *handlerResponse) {
client := userData.(*client)
proxy := client.proxy
proxy.log("Hyper command")
response.SetData([]byte{})
}
// SendStdoutStream sends a Stdout Stream Frame to connected client
func (proxy *CCProxyMock) SendStdoutStream(payload []byte) {
err := api.WriteStream(proxy.cl, api.StreamStdout, payload)
assert.Nil(proxy.t, err)
}
// SendStderrStream sends a Stderr Stream Frame to connected client
func (proxy *CCProxyMock) SendStderrStream(payload []byte) {
err := api.WriteStream(proxy.cl, api.StreamStderr, payload)
assert.Nil(proxy.t, err)
}
// SendExitNotification sends an Exit Notification Frame to connected client
func (proxy *CCProxyMock) SendExitNotification(payload []byte) {
err := api.WriteNotification(proxy.cl, api.NotificationProcessExited, payload)
assert.Nil(proxy.t, err)
}
func (proxy *CCProxyMock) startListening() {
l, err := net.ListenUnix("unix", &net.UnixAddr{Name: proxy.connectionPath, Net: "unix"})
assert.Nil(proxy.t, err)
proxy.logF("listening on %s", proxy.connectionPath)
proxy.listener = l
}
func (proxy *CCProxyMock) serveClient(proto *ccProxyProtocol, newConn net.Conn) {
newClient := &client{
proxy: proxy,
conn: newConn,
}
err := proto.Serve(newConn, newClient)
proxy.logF("Error serving client : %v\n", err)
newConn.Close()
proxy.log("Client closed connection")
proxy.wg.Done()
}
func (proxy *CCProxyMock) serve() {
proto := newCCProxyProtocol()
// shim handlers
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdConnectShim)}, connectShimHandler)
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdDisconnectShim)}, disconnectShimHandler)
proto.Handle(FrameKey{api.TypeStream, int(api.StreamStdin)}, stdinShimHandler)
// runtime handlers
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdRegisterVM)}, registerVMHandler)
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdUnregisterVM)}, unregisterVMHandler)
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdAttachVM)}, attachVMHandler)
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdHyper)}, hyperCmdHandler)
// Shared handler between shim and runtime
proto.Handle(FrameKey{api.TypeCommand, int(api.CmdSignal)}, signalShimHandler)
//Wait for a single client connection
conn, err := proxy.listener.Accept()
if err != nil {
// Ending up into this case when the listener is closed, which
// is still a valid case. We don't want to throw an error in
// this case.
return
}
assert.NotNil(proxy.t, conn)
proxy.log("Client connected")
proxy.wg.Add(1)
proxy.cl = conn
proxy.serveClient(proto, conn)
}
// Start invokes mock proxy instance to start listening.
func (proxy *CCProxyMock) Start() {
proxy.startListening()
go func() {
for {
proxy.serve()
}
}()
}
// Stop causes mock proxy instance to stop listening,
// close connection to client and close all channels
func (proxy *CCProxyMock) Stop() {
proxy.listener.Close()
if proxy.cl != nil {
proxy.log("Closing client connection")
proxy.cl.Close()
proxy.cl = nil
} else {
proxy.log("Client connection already closed")
}
proxy.wg.Wait()
close(proxy.ShimConnected)
close(proxy.Signal)
close(proxy.ShimDisconnected)
close(proxy.StdinReceived)
os.Remove(proxy.connectionPath)
proxy.log("Stopped")
}
// XXX: could do with its own package to remove that ugly namespacing
type ccProxyProtocolHandler func([]byte, interface{}, *handlerResponse)
// Encapsulates the different parts of what a handler can return.
type handlerResponse struct {
err error
results map[string]interface{}
data []byte
}
// SetError indicates sets error for the response.
func (r *handlerResponse) SetError(err error) {
r.err = err
}
// SetErrorMsg sets an error with the passed string for the response.
func (r *handlerResponse) SetErrorMsg(msg string) {
r.err = errors.New(msg)
}
// SetErrorf sets an error with the formatted string for the response.
func (r *handlerResponse) SetErrorf(format string, a ...interface{}) {
r.SetError(fmt.Errorf(format, a...))
}
// AddResult adds the given key/val to the response.
func (r *handlerResponse) AddResult(key string, value interface{}) {
if r.results == nil {
r.results = make(map[string]interface{})
}
r.results[key] = value
}
func (r *handlerResponse) SetData(data []byte) {
r.data = data
}
// FrameKey is a struct composed of the the frame type and opcode,
// used as a key for retrieving the handler for handling the frame.
type FrameKey struct {
ftype api.FrameType
opcode int
}
func newFrameKey(frameType api.FrameType, opcode int) FrameKey {
return FrameKey{
ftype: frameType,
opcode: opcode,
}
}
type ccProxyProtocol struct {
cmdHandlers map[FrameKey]ccProxyProtocolHandler
}
func newCCProxyProtocol() *ccProxyProtocol {
return &ccProxyProtocol{
cmdHandlers: make(map[FrameKey]ccProxyProtocolHandler),
}
}
// Handle retreives the handler for handling the frame
func (proto *ccProxyProtocol) Handle(key FrameKey, handler ccProxyProtocolHandler) bool {
if _, ok := proto.cmdHandlers[key]; ok {
return false
}
proto.cmdHandlers[key] = handler
return true
}
type clientCtx struct {
conn net.Conn
userData interface{}
}
func newErrorResponse(opcode int, errMsg string) *api.Frame {
frame, err := api.NewFrameJSON(api.TypeResponse, opcode, &api.ErrorResponse{
Message: errMsg,
})
if err != nil {
frame, err = api.NewFrameJSON(api.TypeResponse, opcode, &api.ErrorResponse{
Message: fmt.Sprintf("couldn't marshal response: %v", err),
})
if err != nil {
frame = api.NewFrame(api.TypeResponse, opcode, nil)
}
}
frame.Header.InError = true
return frame
}
func (proto *ccProxyProtocol) handleCommand(ctx *clientCtx, cmd *api.Frame) *api.Frame {
hr := handlerResponse{}
// cmd.Header.Opcode is guaranteed to be within the right bounds by
// ReadFrame().
handler := proto.cmdHandlers[FrameKey{cmd.Header.Type, int(cmd.Header.Opcode)}]
handler(cmd.Payload, ctx.userData, &hr)
if hr.err != nil {
return newErrorResponse(cmd.Header.Opcode, hr.err.Error())
}
var payload interface{}
if len(hr.results) > 0 {
payload = hr.results
}
frame, err := api.NewFrameJSON(api.TypeResponse, cmd.Header.Opcode, payload)
if err != nil {
return newErrorResponse(cmd.Header.Opcode, err.Error())
}
return frame
}
// Serve serves the client connection in a continuous loop.
func (proto *ccProxyProtocol) Serve(conn net.Conn, userData interface{}) error {
ctx := &clientCtx{
conn: conn,
userData: userData,
}
for {
frame, err := api.ReadFrame(conn)
if err != nil {
// EOF or the client isn't even sending proper JSON,
// just kill the connection
return err
}
if frame.Header.Type != api.TypeCommand && frame.Header.Type != api.TypeStream {
// EOF or the client isn't even sending proper JSON,
// just kill the connection
return fmt.Errorf("serve: expected a command got a %v", frame.Header.Type)
}
// Execute the corresponding handler
resp := proto.handleCommand(ctx, frame)
// Send the response back to the client.
if err = api.WriteFrame(conn, resp); err != nil {
// Something made us unable to write the response back
// to the client (could be a disconnection, ...).
return err
}
}
}

228
virtcontainers/pkg/mock/mock.go Normal file
View File

@@ -0,0 +1,228 @@
//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package mock
import (
"flag"
"fmt"
"io/ioutil"
"net"
"net/url"
"os"
"path/filepath"
"google.golang.org/grpc"
)
// DefaultMockCCShimBinPath is populated at link time.
var DefaultMockCCShimBinPath string
// DefaultMockKataShimBinPath is populated at link time.
var DefaultMockKataShimBinPath string
// DefaultMockHookBinPath is populated at link time.
var DefaultMockHookBinPath string
// ShimStdoutOutput is the expected output sent by the mock shim on stdout.
const ShimStdoutOutput = "Some output on stdout"
// ShimStderrOutput is the expected output sent by the mock shim on stderr.
const ShimStderrOutput = "Some output on stderr"
// ShimMockConfig is the configuration structure for all virtcontainers shim mock implementations.
type ShimMockConfig struct {
Name string
URLParamName string
ContainerParamName string
TokenParamName string
}
// StartShim is a common routine for starting a shim mock.
func StartShim(config ShimMockConfig) error {
logDirPath, err := ioutil.TempDir("", config.Name+"-")
if err != nil {
return err
}
logFilePath := filepath.Join(logDirPath, "mock_"+config.Name+".log")
f, err := os.Create(logFilePath)
if err != nil {
return err
}
defer f.Close()
tokenFlag := flag.String(config.TokenParamName, "", "Container token")
urlFlag := flag.String(config.URLParamName, "", "Agent URL")
containerFlag := flag.String(config.ContainerParamName, "", "Container ID")
flag.Parse()
fmt.Fprintf(f, "INFO: Token = %s\n", *tokenFlag)
fmt.Fprintf(f, "INFO: URL = %s\n", *urlFlag)
fmt.Fprintf(f, "INFO: Container = %s\n", *containerFlag)
if *tokenFlag == "" {
err := fmt.Errorf("token should not be empty")
fmt.Fprintf(f, "%s\n", err)
return err
}
if *urlFlag == "" {
err := fmt.Errorf("url should not be empty")
fmt.Fprintf(f, "%s\n", err)
return err
}
if _, err := url.Parse(*urlFlag); err != nil {
err2 := fmt.Errorf("could not parse the URL %q: %s", *urlFlag, err)
fmt.Fprintf(f, "%s\n", err2)
return err2
}
if *containerFlag == "" {
err := fmt.Errorf("container should not be empty")
fmt.Fprintf(f, "%s\n", err)
return err
}
// Print some traces to stdout
fmt.Fprintf(os.Stdout, ShimStdoutOutput)
os.Stdout.Close()
// Print some traces to stderr
fmt.Fprintf(os.Stderr, ShimStderrOutput)
os.Stderr.Close()
fmt.Fprintf(f, "INFO: Shim exited properly\n")
return nil
}
// ProxyMock is the proxy mock interface.
// It allows for implementing different kind
// of containers proxies front end.
type ProxyMock interface {
Start(URL string) error
Stop() error
}
// ProxyUnixMock is the UNIX proxy mock
type ProxyUnixMock struct {
ClientHandler func(c net.Conn)
listener net.Listener
}
// ProxyGRPCMock is the gRPC proxy mock
type ProxyGRPCMock struct {
// GRPCImplementer is the structure implementing
// the GRPC interface we want the proxy to serve.
GRPCImplementer interface{}
// GRPCRegister is the registration routine for
// the GRPC service.
GRPCRegister func(s *grpc.Server, srv interface{})
listener net.Listener
}
// Start starts the UNIX proxy mock
func (p *ProxyUnixMock) Start(URL string) error {
if p.ClientHandler == nil {
return fmt.Errorf("Missing client handler")
}
url, err := url.Parse(URL)
if err != nil {
return err
}
l, err := net.Listen(url.Scheme, url.Path)
if err != nil {
return err
}
p.listener = l
go func() {
defer func() {
l.Close()
}()
for {
conn, err := l.Accept()
if err != nil {
return
}
go p.ClientHandler(conn)
}
}()
return nil
}
// Stop stops the UNIX proxy mock
func (p *ProxyUnixMock) Stop() error {
if p.listener == nil {
return fmt.Errorf("Missing proxy listener")
}
return p.listener.Close()
}
// Start starts the gRPC proxy mock
func (p *ProxyGRPCMock) Start(URL string) error {
if p.GRPCImplementer == nil {
return fmt.Errorf("Missing gRPC handler")
}
if p.GRPCRegister == nil {
return fmt.Errorf("Missing gRPC registration routine")
}
url, err := url.Parse(URL)
if err != nil {
return err
}
l, err := net.Listen(url.Scheme, url.Path)
if err != nil {
return err
}
p.listener = l
grpcServer := grpc.NewServer()
p.GRPCRegister(grpcServer, p.GRPCImplementer)
go func() {
grpcServer.Serve(l)
}()
return nil
}
// Stop stops the gRPC proxy mock
func (p *ProxyGRPCMock) Stop() error {
if p.listener == nil {
return fmt.Errorf("Missing proxy listener")
}
return p.listener.Close()
}

194
virtcontainers/pkg/nsenter/nsenter.go Normal file
View File

@@ -0,0 +1,194 @@
//
// Copyright (c) 2018 Intel Corporation
// Copyright 2015-2017 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package nsenter
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strconv"
"sync"
"syscall"
"golang.org/x/sys/unix"
)
// Filesystems constants.
const (
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
nsFSMagic = 0x6e736673
procFSMagic = 0x9fa0
procRootPath = "/proc"
nsDirPath = "ns"
taskDirPath = "task"
)
// NSType defines a namespace type.
type NSType string
// List of namespace types.
// Notice that neither "mnt" nor "user" are listed into this list.
// Because Golang is multithreaded, we get some errors when trying
// to switch to those namespaces, getting "invalid argument".
// The solution is to reexec the current code so that it will call
// into a C constructor, making sure the namespace can be entered
// without multithreading issues.
const (
NSTypeCGroup NSType = "cgroup"
NSTypeIPC = "ipc"
NSTypeNet = "net"
NSTypePID = "pid"
NSTypeUTS = "uts"
)
// CloneFlagsTable is exported so that consumers of this package don't need
// to define this same table again.
var CloneFlagsTable = map[NSType]int{
NSTypeCGroup: unix.CLONE_NEWCGROUP,
NSTypeIPC: unix.CLONE_NEWIPC,
NSTypeNet: unix.CLONE_NEWNET,
NSTypePID: unix.CLONE_NEWPID,
NSTypeUTS: unix.CLONE_NEWUTS,
}
// Namespace describes a namespace that will be entered.
type Namespace struct {
Path string
PID int
Type NSType
}
type nsPair struct {
targetNS *os.File
threadNS *os.File
}
func getNSPathFromPID(pid int, nsType NSType) string {
return filepath.Join(procRootPath, strconv.Itoa(pid), nsDirPath, string(nsType))
}
func getCurrentThreadNSPath(nsType NSType) string {
return filepath.Join(procRootPath, strconv.Itoa(os.Getpid()),
taskDirPath, strconv.Itoa(unix.Gettid()), nsDirPath, string(nsType))
}
func setNS(nsFile *os.File, nsType NSType) error {
if nsFile == nil {
return fmt.Errorf("File handler cannot be nil")
}
nsFlag, exist := CloneFlagsTable[nsType]
if !exist {
return fmt.Errorf("Unknown namespace type %q", nsType)
}
if err := unix.Setns(int(nsFile.Fd()), nsFlag); err != nil {
return fmt.Errorf("Error switching to ns %v: %v", nsFile.Name(), err)
}
return nil
}
// getFileFromNS checks the provided file path actually matches a real
// namespace filesystem, and then opens it to return a handler to this
// file. This is needed since the system call setns() expects a file
// descriptor to enter the given namespace.
func getFileFromNS(nsPath string) (*os.File, error) {
stat := syscall.Statfs_t{}
if err := syscall.Statfs(nsPath, &stat); err != nil {
return nil, fmt.Errorf("failed to Statfs %q: %v", nsPath, err)
}
switch stat.Type {
case nsFSMagic, procFSMagic:
break
default:
return nil, fmt.Errorf("unknown FS magic on %q: %x", nsPath, stat.Type)
}
file, err := os.Open(nsPath)
if err != nil {
return nil, err
}
return file, nil
}
// NsEnter executes the passed closure under the given namespace,
// restoring the original namespace afterwards.
func NsEnter(nsList []Namespace, toRun func() error) error {
targetNSList := make(map[NSType]*nsPair)
// Open all targeted namespaces.
for _, ns := range nsList {
targetNSPath := ns.Path
if targetNSPath == "" {
targetNSPath = getNSPathFromPID(ns.PID, ns.Type)
}
targetNS, err := getFileFromNS(targetNSPath)
if err != nil {
return fmt.Errorf("failed to open target ns: %v", err)
}
defer targetNS.Close()
targetNSList[ns.Type] = &nsPair{
targetNS: targetNS,
}
}
containedCall := func() error {
for nsType := range targetNSList {
threadNS, err := getFileFromNS(getCurrentThreadNSPath(nsType))
if err != nil {
return fmt.Errorf("failed to open current ns: %v", err)
}
defer threadNS.Close()
targetNSList[nsType].threadNS = threadNS
}
// Switch to namespaces all at once.
for nsType, pair := range targetNSList {
// Switch to targeted namespace.
if err := setNS(pair.targetNS, nsType); err != nil {
return fmt.Errorf("error switching to ns %v: %v", pair.targetNS.Name(), err)
}
// Switch back to initial namespace after closure return.
defer setNS(pair.threadNS, nsType)
}
return toRun()
}
var wg sync.WaitGroup
wg.Add(1)
var innerError error
go func() {
defer wg.Done()
runtime.LockOSThread()
defer runtime.UnlockOSThread()
innerError = containedCall()
}()
wg.Wait()
return innerError
}

256
virtcontainers/pkg/nsenter/nsenter_test.go Normal file
View File

@@ -0,0 +1,256 @@
//
// Copyright (c) 2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package nsenter
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
"golang.org/x/sys/unix"
)
const (
testPID = 12345
testNSPath = "/foo/bar/ns"
)
func TestGetNSPathFromPID(t *testing.T) {
for nsType := range CloneFlagsTable {
expectedPath := fmt.Sprintf("/proc/%d/ns/%s", testPID, nsType)
path := getNSPathFromPID(testPID, nsType)
assert.Equal(t, path, expectedPath)
}
}
func TestGetCurrentThreadNSPath(t *testing.T) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
currentPID := os.Getpid()
currentTID := unix.Gettid()
for nsType := range CloneFlagsTable {
expectedPath := fmt.Sprintf("/proc/%d/task/%d/ns/%s", currentPID, currentTID, nsType)
path := getCurrentThreadNSPath(nsType)
assert.Equal(t, path, expectedPath)
}
}
func TestGetFileFromNSEmptyNSPathFailure(t *testing.T) {
nsFile, err := getFileFromNS("")
assert.NotNil(t, err, "Empty path should result as a failure")
assert.Nil(t, nsFile, "The file handler returned should be nil")
}
func TestGetFileFromNSNotExistingNSPathFailure(t *testing.T) {
nsFile, err := ioutil.TempFile("", "not-existing-ns-path")
if err != nil {
t.Fatal(err)
}
nsFilePath := nsFile.Name()
nsFile.Close()
if err := os.Remove(nsFilePath); err != nil {
t.Fatal(err)
}
nsFile, err = getFileFromNS(nsFilePath)
assert.NotNil(t, err, "Not existing path should result as a failure")
assert.Nil(t, nsFile, "The file handler returned should be nil")
}
func TestGetFileFromNSWrongNSPathFailure(t *testing.T) {
nsFile, err := ioutil.TempFile("", "wrong-ns-path")
if err != nil {
t.Fatal(err)
}
nsFilePath := nsFile.Name()
nsFile.Close()
defer os.Remove(nsFilePath)
nsFile, err = getFileFromNS(nsFilePath)
assert.NotNil(t, err, "Should fail because wrong filesystem")
assert.Nil(t, nsFile, "The file handler returned should be nil")
}
func TestGetFileFromNSSuccessful(t *testing.T) {
for nsType := range CloneFlagsTable {
nsFilePath := fmt.Sprintf("/proc/self/ns/%s", string(nsType))
nsFile, err := getFileFromNS(nsFilePath)
assert.Nil(t, err, "Should have succeeded: %v", err)
assert.NotNil(t, nsFile, "The file handler should not be nil")
if nsFile != nil {
nsFile.Close()
}
}
}
func startSleepBinary(duration int, cloneFlags int) (int, error) {
sleepBinName := "sleep"
sleepPath, err := exec.LookPath(sleepBinName)
if err != nil {
return -1, fmt.Errorf("Could not find %q: %v", sleepBinName, err)
}
cmd := exec.Command(sleepPath, strconv.Itoa(duration))
cmd.SysProcAttr = &syscall.SysProcAttr{
Cloneflags: uintptr(cloneFlags),
}
if err := cmd.Start(); err != nil {
return -1, err
}
return cmd.Process.Pid, nil
}
func TestSetNSNilFileHandlerFailure(t *testing.T) {
err := setNS(nil, "")
assert.NotNil(t, err, "Should fail because file handler is nil")
}
func TestSetNSUnknownNSTypeFailure(t *testing.T) {
file := &os.File{}
err := setNS(file, "")
assert.NotNil(t, err, "Should fail because unknown ns type")
}
func TestSetNSWrongFileFailure(t *testing.T) {
nsFile, err := ioutil.TempFile("", "wrong-ns-path")
if err != nil {
t.Fatal(err)
}
defer func() {
nsFilePath := nsFile.Name()
nsFile.Close()
os.Remove(nsFilePath)
}()
err = setNS(nsFile, NSTypeIPC)
assert.NotNil(t, err, "Should fail because file is not a namespace")
}
var testNamespaceList = []Namespace{
{
Type: NSTypeCGroup,
},
{
Type: NSTypeIPC,
},
{
Type: NSTypeNet,
},
{
Type: NSTypePID,
},
{
Type: NSTypeUTS,
},
}
func testToRunNil() error {
return nil
}
func TestNsEnterEmptyPathAndPIDFromNSListFailure(t *testing.T) {
err := NsEnter(testNamespaceList, testToRunNil)
assert.NotNil(t, err, "Should fail because neither a path nor a PID"+
" has been provided by every namespace of the list")
}
func TestNsEnterEmptyNamespaceListSuccess(t *testing.T) {
err := NsEnter([]Namespace{}, testToRunNil)
assert.Nil(t, err, "Should not fail since closure should return nil: %v", err)
}
func TestNsEnterSuccessful(t *testing.T) {
nsList := testNamespaceList
sleepDuration := 60
cloneFlags := 0
for _, ns := range nsList {
cloneFlags |= CloneFlagsTable[ns.Type]
}
sleepPID, err := startSleepBinary(sleepDuration, cloneFlags)
if err != nil {
t.Fatal(err)
}
defer func() {
if sleepPID > 1 {
unix.Kill(sleepPID, syscall.SIGKILL)
}
}()
for idx := range nsList {
nsList[idx].Path = getNSPathFromPID(sleepPID, nsList[idx].Type)
nsList[idx].PID = sleepPID
}
var sleepPIDFromNsEnter int
testToRun := func() error {
sleepPIDFromNsEnter, err = startSleepBinary(sleepDuration, 0)
if err != nil {
return err
}
return nil
}
err = NsEnter(nsList, testToRun)
assert.Nil(t, err, "%v", err)
defer func() {
if sleepPIDFromNsEnter > 1 {
unix.Kill(sleepPIDFromNsEnter, syscall.SIGKILL)
}
}()
for _, ns := range nsList {
nsPathEntered := getNSPathFromPID(sleepPIDFromNsEnter, ns.Type)
// Here we are trying to resolve the path but it fails because
// namespaces links don't really exist. For this reason, the
// call to EvalSymlinks will fail when it will try to stat the
// resolved path found. As we only care about the path, we can
// retrieve it from the PathError structure.
evalExpectedNSPath, err := filepath.EvalSymlinks(ns.Path)
if err != nil {
evalExpectedNSPath = err.(*os.PathError).Path
}
// Same thing here, resolving the namespace path.
evalNSEnteredPath, err := filepath.EvalSymlinks(nsPathEntered)
if err != nil {
evalNSEnteredPath = err.(*os.PathError).Path
}
_, evalExpectedNS := filepath.Split(evalExpectedNSPath)
_, evalNSEntered := filepath.Split(evalNSEnteredPath)
assert.Equal(t, evalExpectedNS, evalNSEntered)
}
}

201
virtcontainers/pkg/oci/LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

680
virtcontainers/pkg/oci/utils.go Normal file
View File

@@ -0,0 +1,680 @@
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package oci
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"path/filepath"
"strconv"
"strings"
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
vc "github.com/containers/virtcontainers"
vcAnnotations "github.com/containers/virtcontainers/pkg/annotations"
dockershimAnnotations "github.com/containers/virtcontainers/pkg/annotations/dockershim"
crioAnnotations "github.com/kubernetes-incubator/cri-o/pkg/annotations"
spec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
type annotationContainerType struct {
annotation string
containerType vc.ContainerType
}
var (
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
ErrNoLinux = errors.New("missing Linux section")
// CRIContainerTypeKeyList lists all the CRI keys that could define
// the container type from annotations in the config.json.
CRIContainerTypeKeyList = []string{criContainerdAnnotations.ContainerType, crioAnnotations.ContainerType, dockershimAnnotations.ContainerTypeLabelKey}
// CRISandboxNameKeyList lists all the CRI keys that could define
// the sandbox ID (pod ID) from annotations in the config.json.
CRISandboxNameKeyList = []string{criContainerdAnnotations.SandboxID, crioAnnotations.SandboxID, dockershimAnnotations.SandboxIDLabelKey}
// CRIContainerTypeList lists all the maps from CRI ContainerTypes annotations
// to a virtcontainers ContainerType.
CRIContainerTypeList = []annotationContainerType{
{crioAnnotations.ContainerTypeSandbox, vc.PodSandbox},
{crioAnnotations.ContainerTypeContainer, vc.PodContainer},
{criContainerdAnnotations.ContainerTypeSandbox, vc.PodSandbox},
{criContainerdAnnotations.ContainerTypeContainer, vc.PodContainer},
{dockershimAnnotations.ContainerTypeLabelSandbox, vc.PodSandbox},
{dockershimAnnotations.ContainerTypeLabelContainer, vc.PodContainer},
}
)
const (
// StateCreated represents a container that has been created and is
// ready to be run.
StateCreated = "created"
// StateRunning represents a container that's currently running.
StateRunning = "running"
// StateStopped represents a container that has been stopped.
StateStopped = "stopped"
)
// CompatOCIProcess is a structure inheriting from spec.Process defined
// in runtime-spec/specs-go package. The goal is to be compatible with
// both v1.0.0-rc4 and v1.0.0-rc5 since the latter introduced a change
// about the type of the Capabilities field.
// Refer to: https://github.com/opencontainers/runtime-spec/commit/37391fb
type CompatOCIProcess struct {
spec.Process
Capabilities interface{} `json:"capabilities,omitempty" platform:"linux"`
}
// CompatOCISpec is a structure inheriting from spec.Spec defined
// in runtime-spec/specs-go package. It relies on the CompatOCIProcess
// structure declared above, in order to be compatible with both
// v1.0.0-rc4 and v1.0.0-rc5.
// Refer to: https://github.com/opencontainers/runtime-spec/commit/37391fb
type CompatOCISpec struct {
spec.Spec
Process *CompatOCIProcess `json:"process,omitempty"`
}
// RuntimeConfig aggregates all runtime specific settings
type RuntimeConfig struct {
VMConfig vc.Resources
HypervisorType vc.HypervisorType
HypervisorConfig vc.HypervisorConfig
AgentType vc.AgentType
AgentConfig interface{}
ProxyType vc.ProxyType
ProxyConfig vc.ProxyConfig
ShimType vc.ShimType
ShimConfig interface{}
Console string
//Determines how the VM should be connected to the
//the container network interface
InterNetworkModel vc.NetInterworkingModel
}
// AddKernelParam allows the addition of new kernel parameters to an existing
// hypervisor configuration stored inside the current runtime configuration.
func (config *RuntimeConfig) AddKernelParam(p vc.Param) error {
return config.HypervisorConfig.AddKernelParam(p)
}
var ociLog = logrus.FieldLogger(logrus.New())
// SetLogger sets the logger for oci package.
func SetLogger(logger logrus.FieldLogger) {
ociLog = logger.WithField("source", "virtcontainers/oci")
}
func cmdEnvs(spec CompatOCISpec, envs []vc.EnvVar) []vc.EnvVar {
for _, env := range spec.Process.Env {
kv := strings.Split(env, "=")
if len(kv) < 2 {
continue
}
envs = append(envs,
vc.EnvVar{
Var: kv[0],
Value: kv[1],
})
}
return envs
}
func newHook(h spec.Hook) vc.Hook {
timeout := 0
if h.Timeout != nil {
timeout = *h.Timeout
}
return vc.Hook{
Path: h.Path,
Args: h.Args,
Env: h.Env,
Timeout: timeout,
}
}
func containerHooks(spec CompatOCISpec) vc.Hooks {
ociHooks := spec.Hooks
if ociHooks == nil {
return vc.Hooks{}
}
var hooks vc.Hooks
for _, h := range ociHooks.Prestart {
hooks.PreStartHooks = append(hooks.PreStartHooks, newHook(h))
}
for _, h := range ociHooks.Poststart {
hooks.PostStartHooks = append(hooks.PostStartHooks, newHook(h))
}
for _, h := range ociHooks.Poststop {
hooks.PostStopHooks = append(hooks.PostStopHooks, newHook(h))
}
return hooks
}
func newMount(m spec.Mount) vc.Mount {
return vc.Mount{
Source: m.Source,
Destination: m.Destination,
Type: m.Type,
Options: m.Options,
}
}
func containerMounts(spec CompatOCISpec) []vc.Mount {
ociMounts := spec.Spec.Mounts
if ociMounts == nil {
return []vc.Mount{}
}
var mnts []vc.Mount
for _, m := range ociMounts {
mnts = append(mnts, newMount(m))
}
return mnts
}
func contains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func newLinuxDeviceInfo(d spec.LinuxDevice) (*vc.DeviceInfo, error) {
allowedDeviceTypes := []string{"c", "b", "u", "p"}
if !contains(allowedDeviceTypes, d.Type) {
return nil, fmt.Errorf("Unexpected Device Type %s for device %s", d.Type, d.Path)
}
if d.Path == "" {
return nil, fmt.Errorf("Path cannot be empty for device")
}
deviceInfo := vc.DeviceInfo{
ContainerPath: d.Path,
DevType: d.Type,
Major: d.Major,
Minor: d.Minor,
}
if d.UID != nil {
deviceInfo.UID = *d.UID
}
if d.GID != nil {
deviceInfo.GID = *d.GID
}
if d.FileMode != nil {
deviceInfo.FileMode = *d.FileMode
}
return &deviceInfo, nil
}
func containerDeviceInfos(spec CompatOCISpec) ([]vc.DeviceInfo, error) {
ociLinuxDevices := spec.Spec.Linux.Devices
if ociLinuxDevices == nil {
return []vc.DeviceInfo{}, nil
}
var devices []vc.DeviceInfo
for _, d := range ociLinuxDevices {
linuxDeviceInfo, err := newLinuxDeviceInfo(d)
if err != nil {
return []vc.DeviceInfo{}, err
}
devices = append(devices, *linuxDeviceInfo)
}
return devices, nil
}
func containerCapabilities(s CompatOCISpec) (vc.LinuxCapabilities, error) {
capabilities := s.Process.Capabilities
var c vc.LinuxCapabilities
// In spec v1.0.0-rc4, capabilities was a list of strings. This was changed
// to an object with v1.0.0-rc5.
// Check for the interface type to support both the versions.
switch caps := capabilities.(type) {
case map[string]interface{}:
for key, value := range caps {
switch val := value.(type) {
case []interface{}:
var list []string
for _, str := range val {
list = append(list, str.(string))
}
switch key {
case "bounding":
c.Bounding = list
case "effective":
c.Effective = list
case "inheritable":
c.Inheritable = list
case "ambient":
c.Ambient = list
case "permitted":
c.Permitted = list
}
default:
return c, fmt.Errorf("Unexpected format for capabilities: %v", caps)
}
}
case []interface{}:
var list []string
for _, str := range caps {
list = append(list, str.(string))
}
c = vc.LinuxCapabilities{
Bounding: list,
Effective: list,
Inheritable: list,
Ambient: list,
Permitted: list,
}
case nil:
ociLog.Debug("Empty capabilities have been passed")
return c, nil
default:
return c, fmt.Errorf("Unexpected format for capabilities: %v", caps)
}
return c, nil
}
func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.NetworkConfig, error) {
linux := ocispec.Linux
if linux == nil {
return vc.NetworkConfig{}, ErrNoLinux
}
var netConf vc.NetworkConfig
for _, n := range linux.Namespaces {
if n.Type != spec.NetworkNamespace {
continue
}
// Bug: This is not the interface count
// It is just an indication that you need networking
netConf.NumInterfaces = 1
if n.Path != "" {
netConf.NetNSPath = n.Path
}
}
netConf.InterworkingModel = config.InterNetworkModel
return netConf, nil
}
// getConfigPath returns the full config path from the bundle
// path provided.
func getConfigPath(bundlePath string) string {
return filepath.Join(bundlePath, "config.json")
}
// ParseConfigJSON unmarshals the config.json file.
func ParseConfigJSON(bundlePath string) (CompatOCISpec, error) {
configPath := getConfigPath(bundlePath)
ociLog.Debugf("converting %s", configPath)
configByte, err := ioutil.ReadFile(configPath)
if err != nil {
return CompatOCISpec{}, err
}
var ocispec CompatOCISpec
if err := json.Unmarshal(configByte, &ocispec); err != nil {
return CompatOCISpec{}, err
}
return ocispec, nil
}
// GetContainerType determines which type of container matches the annotations
// table provided.
func GetContainerType(annotations map[string]string) (vc.ContainerType, error) {
if containerType, ok := annotations[vcAnnotations.ContainerTypeKey]; ok {
return vc.ContainerType(containerType), nil
}
ociLog.Errorf("Annotations[%s] not found, cannot determine the container type",
vcAnnotations.ContainerTypeKey)
return vc.UnknownContainerType, fmt.Errorf("Could not find container type")
}
// ContainerType returns the type of container and if the container type was
// found from CRI servers annotations.
func (spec *CompatOCISpec) ContainerType() (vc.ContainerType, error) {
for _, key := range CRIContainerTypeKeyList {
containerTypeVal, ok := spec.Annotations[key]
if !ok {
continue
}
for _, t := range CRIContainerTypeList {
if t.annotation == containerTypeVal {
return t.containerType, nil
}
}
return vc.UnknownContainerType, fmt.Errorf("Unknown container type %s", containerTypeVal)
}
return vc.PodSandbox, nil
}
// PodID determines the pod ID related to an OCI configuration. This function
// is expected to be called only when the container type is "PodContainer".
func (spec *CompatOCISpec) PodID() (string, error) {
for _, key := range CRISandboxNameKeyList {
podID, ok := spec.Annotations[key]
if ok {
return podID, nil
}
}
return "", fmt.Errorf("Could not find pod ID")
}
func vmConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.Resources, error) {
resources := config.VMConfig
if ocispec.Linux == nil || ocispec.Linux.Resources == nil {
return resources, nil
}
if ocispec.Linux.Resources.Memory != nil &&
ocispec.Linux.Resources.Memory.Limit != nil {
memBytes := *ocispec.Linux.Resources.Memory.Limit
if memBytes <= 0 {
return vc.Resources{}, fmt.Errorf("Invalid OCI memory limit %d", memBytes)
}
// Use some math magic to round up to the nearest Mb.
// This has the side effect that we can never have <1Mb assigned.
resources.Memory = uint((memBytes + (1024*1024 - 1)) / (1024 * 1024))
}
return resources, nil
}
func addAssetAnnotations(ocispec CompatOCISpec, config *vc.PodConfig) {
assetAnnotations := []string{
vcAnnotations.KernelPath,
vcAnnotations.ImagePath,
vcAnnotations.KernelHash,
vcAnnotations.ImageHash,
vcAnnotations.AssetHashType,
}
for _, a := range assetAnnotations {
value, ok := ocispec.Annotations[a]
if !ok {
continue
}
config.Annotations[a] = value
}
}
// PodConfig converts an OCI compatible runtime configuration file
// to a virtcontainers pod configuration structure.
func PodConfig(ocispec CompatOCISpec, runtime RuntimeConfig, bundlePath, cid, console string, detach bool) (vc.PodConfig, error) {
containerConfig, err := ContainerConfig(ocispec, bundlePath, cid, console, detach)
if err != nil {
return vc.PodConfig{}, err
}
networkConfig, err := networkConfig(ocispec, runtime)
if err != nil {
return vc.PodConfig{}, err
}
resources, err := vmConfig(ocispec, runtime)
if err != nil {
return vc.PodConfig{}, err
}
ociSpecJSON, err := json.Marshal(ocispec)
if err != nil {
return vc.PodConfig{}, err
}
podConfig := vc.PodConfig{
ID: cid,
Hostname: ocispec.Hostname,
Hooks: containerHooks(ocispec),
VMConfig: resources,
HypervisorType: runtime.HypervisorType,
HypervisorConfig: runtime.HypervisorConfig,
AgentType: runtime.AgentType,
AgentConfig: runtime.AgentConfig,
ProxyType: runtime.ProxyType,
ProxyConfig: runtime.ProxyConfig,
ShimType: runtime.ShimType,
ShimConfig: runtime.ShimConfig,
NetworkModel: vc.CNMNetworkModel,
NetworkConfig: networkConfig,
Containers: []vc.ContainerConfig{containerConfig},
Annotations: map[string]string{
vcAnnotations.ConfigJSONKey: string(ociSpecJSON),
vcAnnotations.BundlePathKey: bundlePath,
},
}
addAssetAnnotations(ocispec, &podConfig)
return podConfig, nil
}
// ContainerConfig converts an OCI compatible runtime configuration
// file to a virtcontainers container configuration structure.
func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
ociSpecJSON, err := json.Marshal(ocispec)
if err != nil {
return vc.ContainerConfig{}, err
}
rootfs := ocispec.Root.Path
if !filepath.IsAbs(rootfs) {
rootfs = filepath.Join(bundlePath, ocispec.Root.Path)
}
ociLog.Debugf("container rootfs: %s", rootfs)
cmd := vc.Cmd{
Args: ocispec.Process.Args,
Envs: cmdEnvs(ocispec, []vc.EnvVar{}),
WorkDir: ocispec.Process.Cwd,
User: strconv.FormatUint(uint64(ocispec.Process.User.UID), 10),
PrimaryGroup: strconv.FormatUint(uint64(ocispec.Process.User.GID), 10),
Interactive: ocispec.Process.Terminal,
Console: console,
Detach: detach,
NoNewPrivileges: ocispec.Process.NoNewPrivileges,
}
cmd.SupplementaryGroups = []string{}
for _, gid := range ocispec.Process.User.AdditionalGids {
cmd.SupplementaryGroups = append(cmd.SupplementaryGroups, strconv.FormatUint(uint64(gid), 10))
}
deviceInfos, err := containerDeviceInfos(ocispec)
if err != nil {
return vc.ContainerConfig{}, err
}
cmd.Capabilities, err = containerCapabilities(ocispec)
if err != nil {
return vc.ContainerConfig{}, err
}
var resources vc.ContainerResources
if ocispec.Linux.Resources.CPU != nil {
if ocispec.Linux.Resources.CPU.Quota != nil &&
ocispec.Linux.Resources.CPU.Period != nil {
resources.CPUQuota = *ocispec.Linux.Resources.CPU.Quota
resources.CPUPeriod = *ocispec.Linux.Resources.CPU.Period
}
if ocispec.Linux.Resources.CPU.Shares != nil {
resources.CPUShares = *ocispec.Linux.Resources.CPU.Shares
}
}
containerConfig := vc.ContainerConfig{
ID: cid,
RootFs: rootfs,
ReadonlyRootfs: ocispec.Spec.Root.Readonly,
Cmd: cmd,
Annotations: map[string]string{
vcAnnotations.ConfigJSONKey: string(ociSpecJSON),
vcAnnotations.BundlePathKey: bundlePath,
},
Mounts: containerMounts(ocispec),
DeviceInfos: deviceInfos,
Resources: resources,
}
cType, err := ocispec.ContainerType()
if err != nil {
return vc.ContainerConfig{}, err
}
containerConfig.Annotations[vcAnnotations.ContainerTypeKey] = string(cType)
return containerConfig, nil
}
// StatusToOCIState translates a virtcontainers container status into an OCI state.
func StatusToOCIState(status vc.ContainerStatus) spec.State {
return spec.State{
Version: spec.Version,
ID: status.ID,
Status: StateToOCIState(status.State),
Pid: status.PID,
Bundle: status.Annotations[vcAnnotations.BundlePathKey],
Annotations: status.Annotations,
}
}
// StateToOCIState translates a virtcontainers container state into an OCI one.
func StateToOCIState(state vc.State) string {
switch state.State {
case vc.StateReady:
return StateCreated
case vc.StateRunning:
return StateRunning
case vc.StateStopped:
return StateStopped
default:
return ""
}
}
// EnvVars converts an OCI process environment variables slice
// into a virtcontainers EnvVar slice.
func EnvVars(envs []string) ([]vc.EnvVar, error) {
var envVars []vc.EnvVar
envDelimiter := "="
expectedEnvLen := 2
for _, env := range envs {
envSlice := strings.SplitN(env, envDelimiter, expectedEnvLen)
if len(envSlice) < expectedEnvLen {
return []vc.EnvVar{}, fmt.Errorf("Wrong string format: %s, expecting only %v parameters separated with %q",
env, expectedEnvLen, envDelimiter)
}
if envSlice[0] == "" {
return []vc.EnvVar{}, fmt.Errorf("Environment variable cannot be empty")
}
envSlice[1] = strings.Trim(envSlice[1], "' ")
if envSlice[1] == "" {
return []vc.EnvVar{}, fmt.Errorf("Environment value cannot be empty")
}
envVar := vc.EnvVar{
Var: envSlice[0],
Value: envSlice[1],
}
envVars = append(envVars, envVar)
}
return envVars, nil
}
// GetOCIConfig returns an OCI spec configuration from the annotation
// stored into the container status.
func GetOCIConfig(status vc.ContainerStatus) (CompatOCISpec, error) {
ociConfigStr, ok := status.Annotations[vcAnnotations.ConfigJSONKey]
if !ok {
return CompatOCISpec{}, fmt.Errorf("Annotation[%s] not found", vcAnnotations.ConfigJSONKey)
}
var ociSpec CompatOCISpec
if err := json.Unmarshal([]byte(ociConfigStr), &ociSpec); err != nil {
return CompatOCISpec{}, err
}
return ociSpec, nil
}

Some files were not shown because too many files have changed in this diff Show More