From 643ddf990762b3681c38a2a14c9812bb1e7f6239 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 30 Sep 2019 11:49:35 -0700 Subject: [PATCH 01/10] release: Add option to generate versions based on tag Signed-off-by: Eric Ernst --- obs-packaging/gen_versions_txt.sh | 60 +++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/obs-packaging/gen_versions_txt.sh b/obs-packaging/gen_versions_txt.sh index 31d3550ab..2f22dbb55 100755 --- a/obs-packaging/gen_versions_txt.sh +++ b/obs-packaging/gen_versions_txt.sh @@ -26,12 +26,13 @@ get_kata_version() { gen_version_file() { local branch="$1" - local use_head="$2" + local kata_version="$2" local ref="refs/heads/${branch}" - local kata_version="${branch}" - if [ -n "$branch" ] && [ -z "${use_head}" ]; then - kata_version=$(get_kata_version "${branch}") + if [ "${kata_version}" == "HEAD" ]; then + kata_version="${branch}" + ref="refs/heads/${branch}" + else ref="refs/tags/${kata_version}^{}" fi @@ -135,6 +136,7 @@ EOT main() { local compareOnly= local use_head= + local use_tag= case "${1:-}" in "-h"|"--help") @@ -148,13 +150,57 @@ main() { use_head=1 shift ;; + --tag) + use_tag=1 + shift + ;; -*) die "Invalid option: ${1:-}" "1" shift ;; esac - local branch="${1:-}" - [ -n "${branch}" ] || die "No branch specified" "1" + + local kata_version= + if [ -n "$use_tag" ]; then + if [ -n "${use_head}" ]; then + die "tag and head options are mutually exclusive" + fi + + # We are generating versions based on the provided tag + local tag="${1:-}" + [ -n "${tag}" ] || die "No tag specified" "1" + + # use the runtime's repository to determine branch information + local repo="github.com/kata-containers/runtime" + local repo_dir="runtime" + git clone --quiet "https://${repo}.git" "${repo_dir}" + pushd "${repo_dir}" >> /dev/null + local branch=$(git branch -r -q --contains "${tag}" | grep -E "master|stable" | grep -v HEAD) + + popd >> /dev/null + rm -rf ${repo_dir} + + [ -n "${branch}" ] || die "branch for tag ${tag} not found" + + # in the event this is on master as well as stable, or multiple stables, just pick the first branch + # (ie, 1.8.0-alpha0 may live on stable-1.8 as well as master: we'd just use master in this case) + branch=$(echo ${branch} | awk -F" " '{print $1}') + + # format will be origin/ - let's drop origin: + branch=$(echo ${branch} | awk -F"/" '{print $2}') + + echo "generating versions for tag ${tag} which is on branch ${branch}" + kata_version=${tag} + else + local branch="${1:-}" + [ -n "${branch}" ] || die "No branch specified" "1" + + if [ -n "${use_head}" ]; then + kata_version="HEAD" + else + kata_version=$(get_kata_version "${branch}") + fi + fi if [ -n "$compareOnly" ]; then source "./${versions_txt}" || exit 1 @@ -167,7 +213,7 @@ main() { return fi - gen_version_file "${branch}" "${use_head}" + gen_version_file "${branch}" "${kata_version}" } main $@ From f2ef8411fa756b3d633e169b5355fa71e9462d3e Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 15 Oct 2019 11:28:25 -0700 Subject: [PATCH 02/10] release: don't checkout packaging from packaging We ran into issues in the past since we didn't create stable branches for the packaging repository. We will maintain this appropriately going forward, so let's go ahead and remove the notion of local versus remote. Signed-off-by: Eric Ernst --- release/kata-deploy-binaries.sh | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index d956c12cd..53d992cf9 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -109,15 +109,7 @@ install_image() { #Install kernel asset install_kernel() { - if [[ "$test_local" == "true" ]]; then - pushd "${script_dir}/../" - else - go get "github.com/${project}/packaging" || true - pushd ${GOPATH}/src/github.com/${project}/packaging >>/dev/null - git checkout "${kata_version}-kernel-config" || - git checkout "${kata_version}" - fi - + pushd "${script_dir}/../" info "build kernel" ./kernel/build-kernel.sh setup ./kernel/build-kernel.sh build @@ -128,15 +120,7 @@ install_kernel() { #Install experimental kernel asset install_experimental_kernel() { - if [[ "$test_local" == "true" ]]; then - pushd "${script_dir}/../" - else - go get "github.com/${project}/packaging" || true - pushd ${GOPATH}/src/github.com/${project}/packaging >>/dev/null - git checkout "${kata_version}-kernel-config" || - git checkout "${kata_version}" - fi - + pushd "${script_dir}/../" info "build experimental kernel" ./kernel/build-kernel.sh -e setup ./kernel/build-kernel.sh -e build From 420eb6e984b0ae6e871ee3d77ade86e913ec8171 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 15 Oct 2019 16:39:47 -0700 Subject: [PATCH 03/10] qemu-virtiofs: Fix tar naming for qemu with virtiofs support Fixes the naming of tarball for qemu-virtiofs so that it is different from the qemu one. Signed-off-by: Archana Shinde --- release/kata-deploy-binaries.sh | 2 +- static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index 53d992cf9..0c39c3d9e 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -142,7 +142,7 @@ install_qemu_virtiofsd() { info "build static qemu-virtiofs" "${script_dir}/../static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh" info "Install static qemu-virtiofs" - tar xf kata-qemu-static.tar.gz -C "${destdir}" + tar xf kata-qemu-virtiofs-static.tar.gz -C "${destdir}" } # Install static firecracker asset diff --git a/static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh b/static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh index 141255935..e1a21b090 100755 --- a/static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh +++ b/static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh @@ -18,7 +18,8 @@ qemu_virtiofs_repo=$(get_from_kata_deps "assets.hypervisor.qemu-experimental.url # This tag will be supported on the runtime versions.yaml qemu_virtiofs_tag=$(get_from_kata_deps "assets.hypervisor.qemu-experimental.tag") qemu_tar="kata-qemu-static.tar.gz" -qemu_tmp_tar="kata-qemu-static-tmp.tar.gz" +qemu_virtiofs_tar="kata-qemu-virtiofs-static.tar.gz" +qemu_tmp_tar="kata-qemu-virtiofs-static-tmp.tar.gz" info "Build ${qemu_virtiofs_repo} tag: ${qemu_virtiofs_tag}" @@ -46,4 +47,4 @@ sudo chown ${USER}:${USER} "${PWD}/${qemu_tar}" # Remove blacklisted binaries gzip -d < "${qemu_tar}" | tar --delete --wildcards -f - ${qemu_black_list[*]} | gzip > "${qemu_tmp_tar}" -mv -f "${qemu_tmp_tar}" "${qemu_tar}" +mv -f "${qemu_tmp_tar}" "${qemu_virtiofs_tar}" From 7a932cf3a96795bc73a975250ef963a7d00ace5e Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 15 Oct 2019 16:48:11 -0700 Subject: [PATCH 04/10] release: Create tarballs after every stage Make every build function output a tarball. Signed-off-by: Archana Shinde --- release/kata-deploy-binaries.sh | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index 0c39c3d9e..6ad6b466e 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -105,6 +105,9 @@ install_image() { ln -sf "${image}" kata-containers.img ln -sf "${initrd}" kata-containers-initrd.img popd >>/dev/null + pushd ${destdir} + tar -czvf ../kata-image.tar.gz * + popd } #Install kernel asset @@ -116,6 +119,9 @@ install_kernel() { info "install kernel" DESTDIR="${destdir}" PREFIX="${prefix}" ./kernel/build-kernel.sh install popd + pushd ${destdir} + tar -czvf ../kata-kernel.tar.gz * + popd } #Install experimental kernel asset @@ -127,22 +133,21 @@ install_experimental_kernel() { info "install experimental kernel" DESTDIR="${destdir}" PREFIX="${prefix}" ./kernel/build-kernel.sh -e install popd + pushd ${destdir} + tar -czvf ../kata-kernel-experimental.tar.gz * + popd } # Install static qemu asset install_qemu() { info "build static qemu" "${script_dir}/../static-build/qemu/build-static-qemu.sh" - info "Install static qemu" - tar xf kata-qemu-static.tar.gz -C "${destdir}" } # Install static qemu-virtiofsd asset install_qemu_virtiofsd() { info "build static qemu-virtiofs" "${script_dir}/../static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh" - info "Install static qemu-virtiofs" - tar xf kata-qemu-virtiofs-static.tar.gz -C "${destdir}" } # Install static firecracker asset @@ -153,7 +158,9 @@ install_firecracker() { mkdir -p "${destdir}/opt/kata/bin/" sudo install -D --owner root --group root --mode 0744 firecracker/firecracker-static "${destdir}/opt/kata/bin/firecracker" sudo install -D --owner root --group root --mode 0744 firecracker/jailer-static "${destdir}/opt/kata/bin/jailer" - + pushd ${destdir} + tar -czvf ../kata-firecracker-static.tar.gz * + popd } install_docker_config_script() { @@ -218,6 +225,16 @@ EOT sudo chmod +x kata-qemu-virtiofs popd + pushd ${destdir} + tar -czvf ../kata-components.tar.gz * + popd +} + +untar_qemu_binaries() { + info "Install static qemu" + tar xf kata-qemu-static.tar.gz -C "${destdir}" + info "Install static qemu-virtiofs" + tar xf kata-qemu-virtiofs-static.tar.gz -C "${destdir}" } main() { @@ -251,6 +268,8 @@ main() { install_firecracker install_docker_config_script + untar_qemu_binaries + tarball_name="${destdir}.tar.xz" pushd "${destdir}" >>/dev/null tar cfJ "${tarball_name}" "./opt" From 5307b03adbd8c8ee3c1d87872a7c84a7bcacab6c Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 15 Oct 2019 17:43:42 -0700 Subject: [PATCH 05/10] release: Define a default value for destdir We want to isupport calling individual functions from the script, independendent of the actual script being called. Define a default value for $destdir. Signed-off-by: Archana Shinde --- release/kata-deploy-binaries.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index 6ad6b466e..21a5d696d 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -24,6 +24,9 @@ workdir="${WORKDIR:-$PWD}" # that are local test_local="false" +destdir="${workdir}/kata-static" +mkdir -p "${destdir}" + exit_handler() { [ -d "${tmp_dir}" ] || sudo rm -rf "${tmp_dir}" } From 6c8df7fc3d203644cfeb57a33834b57449d28558 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Wed, 16 Oct 2019 11:00:21 -0700 Subject: [PATCH 06/10] release: Call kata-deploy-binaries.sh main only if it not sourced Allow script to be sourced. Signed-off-by: Archana Shinde --- release/kata-deploy-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index 21a5d696d..a0772d796 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -284,4 +284,4 @@ main() { fi } -main $@ +[[ "${BASH_SOURCE[0]}" == "${0}" ]] && main "$@" From dc8fe056548ba143b5ee216edf67a0852b017d0a Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 17 Oct 2019 15:45:36 -0700 Subject: [PATCH 07/10] release: Allow functions to take release versions Allow functions to take release tags Signed-off-by: Archana Shinde --- release/kata-deploy-binaries.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index a0772d796..1ba0259c6 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -92,6 +92,7 @@ verify_hub() { #Install guest image/initrd asset install_image() { + kata_version=${1:-kata_version} image_destdir="${destdir}/${prefix}/share/kata-containers/" info "Create image" image_tarball=$(find . -name 'kata-containers-'"${kata_version}"'-*.tar.gz') @@ -180,6 +181,7 @@ install_docker_config_script() { #Install all components that are not assets install_kata_components() { + kata_version=${1:-kata_version} for p in "${projects[@]}"; do echo "Download ${p}" go get "github.com/${project}/$p" || true From 4f89e97e5e67538242e416ea6dcd166c25c82d8c Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 16 Oct 2019 12:24:01 -0700 Subject: [PATCH 08/10] kata-deploy: look for kata artifacts locally When building the kata-deploy images before, we would look to pull the latest artifacts from the release URL. It would be better to allow the user to pull from this URL, or to create the artifacts locally, and pass the location of this tar.xz to the build process. Instead of providing KATA_VER, builders should provide KATA_ARTIFACTS, which is the filename that is assumed to be located within the docker build path. Signed-off-by: Eric Ernst --- kata-deploy/Dockerfile | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile index 500923294..5db3a0bca 100644 --- a/kata-deploy/Dockerfile +++ b/kata-deploy/Dockerfile @@ -1,24 +1,22 @@ FROM centos/systemd -ARG KATA_VER -ARG ARCH=x86_64 ARG KUBE_ARCH=amd64 -ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA_VER} -ARG KATA_FILE=kata-static-${KATA_VER}-${ARCH}.tar.xz +ARG KATA_ARTIFACTS=./kata-static.tar.xz +ARG DESTINATION=/opt/kata-artifacts + +COPY ${KATA_ARTIFACTS} . RUN \ yum install -y epel-release && \ yum install -y bzip2 jq && \ -curl -sOL ${KATA_URL}/${KATA_FILE} && \ -mkdir -p /opt/kata-artifacts && \ -tar xvf ${KATA_FILE} -C /opt/kata-artifacts/ && \ -chown -R root:root /opt/kata-artifacts/ && \ -rm ${KATA_FILE} +mkdir -p ${DESTINATION} && \ +tar xvf ${KATA_ARTIFACTS} -C ${DESTINATION}/ && \ +chown -R root:root ${DESTINATION}/ RUN \ curl -Lso /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${KUBE_ARCH}/kubectl && \ chmod +x /bin/kubectl -COPY scripts /opt/kata-artifacts/scripts +COPY scripts ${DESTINATION}/scripts RUN \ -ln -s /opt/kata-artifacts/scripts/kata-deploy-docker.sh /usr/bin/kata-deploy-docker && \ -ln -s /opt/kata-artifacts/scripts/kata-deploy.sh /usr/bin/kata-deploy +ln -s ${DESTINATION}/scripts/kata-deploy-docker.sh /usr/bin/kata-deploy-docker && \ +ln -s ${DESTINATION}/scripts/kata-deploy.sh /usr/bin/kata-deploy From 4eb376b40e50ae8fcf71f90a7c488f85f3e757c9 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 20 Sep 2019 16:05:30 -0700 Subject: [PATCH 09/10] artifact-list: provide script to get items to build artifact-list.sh is created so a builder can quickly determine which artifacts may be built within this repository. I user may get this list, which indicates exactly which functions are available within ./release/kata-deploy-binaries.sh for building. Signed-off-by: Eric Ernst --- artifact-list.sh | 23 +++++++++++++++++++++++ release/kata-deploy-binaries.sh | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100755 artifact-list.sh diff --git a/artifact-list.sh b/artifact-list.sh new file mode 100755 index 000000000..87669c8e8 --- /dev/null +++ b/artifact-list.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +supported_artifacts=( + "install_docker_config_script" + "install_experimental_kernel" + "install_firecracker" + "install_image" + "install_kata_components" + "install_kernel" + "install_qemu" + "install_qemu_virtiofsd" +) + +for c in ${supported_artifacts[@]}; do echo $c; done diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index 1ba0259c6..44ea30f0b 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -286,4 +286,6 @@ main() { fi } -[[ "${BASH_SOURCE[0]}" == "${0}" ]] && main "$@" +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main $@ +fi From 9a7d6922b42a579f712c1306e7060b03b612f423 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 16 Oct 2019 13:38:21 -0700 Subject: [PATCH 10/10] kata-deploy: Add a simple GitHub Action Create a container based action which will test a Kata artifact tarball in the kata-deploy daemonset on AKS. This AZ credentials are available from the callers environment. Signed-off-by: Eric Ernst --- kata-deploy/action/Dockerfile | 31 ++++ kata-deploy/action/action.yaml | 13 ++ kata-deploy/action/entrypoint.sh | 24 +++ kata-deploy/action/kubernetes-containerd.json | 41 +++++ kata-deploy/action/setup-aks.sh | 53 ++++++ kata-deploy/action/test-kata.sh | 163 ++++++++++++++++++ kata-deploy/examples/nginx-deployment-fc.yaml | 20 +++ .../examples/nginx-deployment-nemu.yaml | 20 +++ .../nginx-deployment-qemu-virtiofs.yaml | 20 +++ .../examples/nginx-deployment-qemu.yaml | 20 +++ 10 files changed, 405 insertions(+) create mode 100644 kata-deploy/action/Dockerfile create mode 100644 kata-deploy/action/action.yaml create mode 100755 kata-deploy/action/entrypoint.sh create mode 100644 kata-deploy/action/kubernetes-containerd.json create mode 100755 kata-deploy/action/setup-aks.sh create mode 100755 kata-deploy/action/test-kata.sh create mode 100644 kata-deploy/examples/nginx-deployment-fc.yaml create mode 100644 kata-deploy/examples/nginx-deployment-nemu.yaml create mode 100644 kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml create mode 100644 kata-deploy/examples/nginx-deployment-qemu.yaml diff --git a/kata-deploy/action/Dockerfile b/kata-deploy/action/Dockerfile new file mode 100644 index 000000000..7047680f6 --- /dev/null +++ b/kata-deploy/action/Dockerfile @@ -0,0 +1,31 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +FROM microsoft/azure-cli:latest + +LABEL com.github.actions.name="Test kata-deploy in an AKS cluster" +LABEL com.github.actions.description="Test kata-deploy in an AKS cluster" + +# Default to latest validated AKS-engine version +ARG AKS_ENGINE_VER="v0.42.0" +ARG ARCH=amd64 + +ENV GITHUB_ACTION_NAME="Test kata-deploy in an AKS cluster" + +# When run, we expect the caller (GitHub Action workflow) to provide the +# PKG_SHA environment variable +ENV PKG_SHA=HEAD + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${ARCH}/kubectl \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + +RUN curl -LO https://github.com/Azure/aks-engine/releases/download/${AKS_ENGINE_VER}/aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}.tar.gz \ + && tar xvf aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}.tar.gz \ + && mv aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}/aks-engine /usr/local/bin/aks-engine \ + && rm aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}.tar.gz + +COPY kubernetes-containerd.json / +COPY setup-aks.sh test-kata.sh entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/kata-deploy/action/action.yaml b/kata-deploy/action/action.yaml new file mode 100644 index 000000000..b894fecaa --- /dev/null +++ b/kata-deploy/action/action.yaml @@ -0,0 +1,13 @@ +# action.yml +name: 'kata-deploy' +description: 'test Kata container image in AKS' +inputs: + packaging-sha: + description: 'SHA we are using for pulling packaing manifests' + required: true + default: '' +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.packaging-sha }} diff --git a/kata-deploy/action/entrypoint.sh b/kata-deploy/action/entrypoint.sh new file mode 100755 index 000000000..899bada6f --- /dev/null +++ b/kata-deploy/action/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -o errexit +set -o pipefail +set -o nounset + +# This entrypoint expects an environment variable, PKG_SHA, to be +# within the container runtime. A default is provided in the Dockerfile, +# but we expect the caller to pass this into the container run (ie docker run -e PKG_SHA=foo ...) +echo "provided package reference: ${PKG_SHA}" + +# Since this is the entrypoint for the container image, we know that the AKS and Kata setup/testing +# scripts are located at root. +source /setup-aks.sh +source /test-kata.sh + +trap destroy_aks EXIT + +setup_aks +test_kata \ No newline at end of file diff --git a/kata-deploy/action/kubernetes-containerd.json b/kata-deploy/action/kubernetes-containerd.json new file mode 100644 index 000000000..1b92e04f9 --- /dev/null +++ b/kata-deploy/action/kubernetes-containerd.json @@ -0,0 +1,41 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorVersion": "1.15.4", + "kubernetesConfig": { + "networkPlugin": "flannel", + "containerRuntime": "containerd", + "containerdVersion": "1.2.4" + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 1, + "vmSize": "Standard_D4s_v3", + "availabilityProfile": "AvailabilitySet" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/kata-deploy/action/setup-aks.sh b/kata-deploy/action/setup-aks.sh new file mode 100755 index 000000000..967f500b7 --- /dev/null +++ b/kata-deploy/action/setup-aks.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -o errexit +set -o pipefail +set -o nounset + +export AZURE_HTTP_USER_AGENT="GITHUBACTIONS_${GITHUB_ACTION_NAME}_${GITHUB_REPOSITORY}" + +LOCATION=${LOCATION:-westus2} +DNS_PREFIX=${DNS_PREFIX:-kata-deploy-${GITHUB_SHA:0:10}} +CLUSTER_CONFIG=${CLUSTER_CONFIG:-/kubernetes-containerd.json} + +function die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function destroy_aks() { + set +x + + export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" + kubectl describe ds -n kube-system kata-deploy || true + kubectl describe ds -n kube-system kata-cleanup || true + + az login --service-principal -u "$AZ_APPID" -p "$AZ_PASSWORD" --tenant "$AZ_TENANT_ID" + az group delete --name "$DNS_PREFIX" --yes --no-wait + az logout +} + +function setup_aks() { + [[ -z "$AZ_APPID" ]] && die "no Azure service principal ID provided" + [[ -z "$AZ_PASSWORD" ]] && die "no Azure service principal secret provided" + [[ -z "$AZ_SUBSCRIPTION_ID" ]] && die "no Azure subscription ID provided" + [[ -z "$AZ_TENANT_ID" ]] && die "no Azure tenant ID provided" + + aks-engine deploy --subscription-id "$AZ_SUBSCRIPTION_ID" \ + --client-id "$AZ_APPID" --client-secret "$AZ_PASSWORD" \ + --location "$LOCATION" --dns-prefix "$DNS_PREFIX" \ + --api-model "$CLUSTER_CONFIG" --force-overwrite + + export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" + + # wait for the cluster to be settled: + kubectl wait --timeout=10m --for=condition=Ready --all nodes + + # make sure coredns is up before moving forward: + kubectl wait --timeout=5m -n kube-system --for=condition=Available deployment.extensions/coredns +} diff --git a/kata-deploy/action/test-kata.sh b/kata-deploy/action/test-kata.sh new file mode 100755 index 000000000..7f0dcde1b --- /dev/null +++ b/kata-deploy/action/test-kata.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +function die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function waitForProcess() { + wait_time="$1" + sleep_time="$2" + cmd="$3" + while [ "$wait_time" -gt 0 ]; do + if eval "$cmd"; then + return 0 + else + echo "waiting" + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + return 1 +} + +# waitForLabelRemoval will wait for the kata-runtime labels to removed until a given +# timeout expires +function waitForLabelRemoval() { + wait_time="$1" + sleep_time="$2" + + while [[ "$wait_time" -gt 0 ]]; do + # if a node is found which matches node-select, the output will include a column for node name, + # NAME. Let's look for that + if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime | grep NAME) ]] + then + return 0 + else + echo "waiting for kata-runtime label to be removed" + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + + echo "failed to cleanup" + return 1 +} + + +function run_test() { + PKG_SHA=$1 + YAMLPATH="https://raw.githubusercontent.com/amshinde/kata-packaging/$PKG_SHA/kata-deploy" + echo "verify connectivity with a pod using Kata" + + deployment="" + busybox_pod="test-nginx" + busybox_image="busybox" + cmd="kubectl get pods | grep $busybox_pod | grep Completed" + wait_time=120 + sleep_time=3 + + configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs") + for deployment in "${configurations[@]}"; do + # start the kata pod: + kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml" + + # in case the control plane is slow, give it a few seconds to accept the yaml, otherwise + # our 'wait' for deployment status will fail to find the deployment at all + sleep 3 + + kubectl wait --timeout=5m --for=condition=Available deployment/${deployment} + kubectl expose deployment/${deployment} + + # test pod connectivity: + kubectl run $busybox_pod --restart=Never --image="$busybox_image" -- wget --timeout=5 "$deployment" + waitForProcess "$wait_time" "$sleep_time" "$cmd" + kubectl logs "$busybox_pod" | grep "index.html" + kubectl describe pod "$busybox_pod" + + # cleanup: + kubectl delete deployment "$deployment" + kubectl delete service "$deployment" + kubectl delete pod "$busybox_pod" + done +} + + +function test_kata() { + set -x + + [[ -z "$PKG_SHA" ]] && die "no PKG_SHA provided" + echo "$PKG_SHA" + + #kubectl all the things + kubectl get pods,nodes --all-namespaces + + YAMLPATH="https://raw.githubusercontent.com/amshinde/kata-packaging/$PKG_SHA/kata-deploy" + + kubectl apply -f "$YAMLPATH/kata-rbac.yaml" + + # apply runtime classes: + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-runtimeClass.yaml" + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-virtiofs-runtimeClass.yaml" + + kubectl get runtimeclasses + + curl -LO "$YAMLPATH/kata-deploy.yaml" + curl -LO "$YAMLPATH/kata-cleanup.yaml" + + # update deployment daemonset to utilize the container under test: + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-deploy.yaml + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-cleanup.yaml + + cat kata-deploy.yaml + + # deploy kata: + kubectl apply -f kata-deploy.yaml + + # in case the control plane is slow, give it a few seconds to accept the yaml, otherwise + # our 'wait' for deployment status will fail to find the deployment at all. If it can't persist + # the daemonset to etcd in 30 seconds... then we'll fail. + sleep 30 + + # wait for kata-deploy to be up + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + + # show running pods, and labels of nodes + kubectl get pods,nodes --all-namespaces --show-labels + + run_test $PKG_SHA + + kubectl get pods,nodes --show-labels + + # Remove Kata + kubectl delete -f kata-deploy.yaml + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + kubectl get pods,nodes --show-labels + + kubectl apply -f kata-cleanup.yaml + + # The cleanup daemonset will run a single time, since it will clear the node-label. Thus, its difficult to + # check the daemonset's status for completion. instead, let's wait until the kata-runtime labels are removed + # from all of the worker nodes. If this doesn't happen in 45 seconds, let's fail + timeout=45 + sleeptime=1 + waitForLabelRemoval $timeout $sleeptime + + kubectl delete -f kata-cleanup.yaml + + rm kata-cleanup.yaml + rm kata-deploy.yaml + + set +x +} diff --git a/kata-deploy/examples/nginx-deployment-fc.yaml b/kata-deploy/examples/nginx-deployment-fc.yaml new file mode 100644 index 000000000..2053e0dee --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-fc.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-fc +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-fc + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80 diff --git a/kata-deploy/examples/nginx-deployment-nemu.yaml b/kata-deploy/examples/nginx-deployment-nemu.yaml new file mode 100644 index 000000000..907e3c69d --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-nemu.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-nemu +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-nemu + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80 diff --git a/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml b/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml new file mode 100644 index 000000000..06bf7d449 --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-qemu-virtiofs +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-qemu-virtiofs + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80 diff --git a/kata-deploy/examples/nginx-deployment-qemu.yaml b/kata-deploy/examples/nginx-deployment-qemu.yaml new file mode 100644 index 000000000..467c99e91 --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-qemu.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-qemu +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-qemu + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80