From 26be8836da7e73817458bad165f05896af5ecfad Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 20 Sep 2019 13:22:30 -0700 Subject: [PATCH] github action v2 play Signed-off-by: Eric Ernst --- .github/kata-artifacts-action/Dockerfile | 14 +++ .github/kata-artifacts-action/entrypoint.sh | 22 ++++ .github/kata-deploy-action/Dockerfile | 24 ++++ .github/kata-deploy-action/entrypoint.sh | 22 ++++ .../kubernetes-containerd.json | 41 +++++++ .github/kata-deploy-action/setup-aks.sh | 44 +++++++ .github/kata-deploy-action/test-kata.sh | 112 ++++++++++++++++++ .github/kata-deploy-action/trigger | 4 + .github/workflows/kata-release.yml | 55 +++++++++ 9 files changed, 338 insertions(+) create mode 100644 .github/kata-artifacts-action/Dockerfile create mode 100755 .github/kata-artifacts-action/entrypoint.sh create mode 100644 .github/kata-deploy-action/Dockerfile create mode 100755 .github/kata-deploy-action/entrypoint.sh create mode 100644 .github/kata-deploy-action/kubernetes-containerd.json create mode 100755 .github/kata-deploy-action/setup-aks.sh create mode 100755 .github/kata-deploy-action/test-kata.sh create mode 100755 .github/kata-deploy-action/trigger create mode 100644 .github/workflows/kata-release.yml diff --git a/.github/kata-artifacts-action/Dockerfile b/.github/kata-artifacts-action/Dockerfile new file mode 100644 index 000000000..477abbf0f --- /dev/null +++ b/.github/kata-artifacts-action/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:latest + +LABEL version="0.0.0" +LABEL maintainer="Kata folks" +LABEL com.github.actions.name="Prepare artifacts for Kata release page" +LABEL com.github.actions.description="Create and upload static binaries and Kata images to release page for a given release" + +ENV GITHUB_ACTION_NAME="Prepare artifacts for Kata release" +ENV NEW_VERSION="1.8.2" +ENV BRANCH="master" + +RUN git clone https://github.com/kata-containers/packaging.git && cd packaging + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/kata-artifacts-action/entrypoint.sh b/.github/kata-artifacts-action/entrypoint.sh new file mode 100755 index 000000000..e50f99901 --- /dev/null +++ b/.github/kata-artifacts-action/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +# Entrypoint for the container image, we know that the AKS and Kata setup/testing +# scripts are located at root. + +cd obs-packaging +bash -x ./gen_versions_txt.sh ${BRANCH} +cd ../release +bash -x ./publish-kata-image.sh -p ${NEW_VERSION} +bash -x ./kata-deploy-binaries.sh -p ${NEW_VERSION} + +echo "maybe it worked" diff --git a/.github/kata-deploy-action/Dockerfile b/.github/kata-deploy-action/Dockerfile new file mode 100644 index 000000000..1084b3235 --- /dev/null +++ b/.github/kata-deploy-action/Dockerfile @@ -0,0 +1,24 @@ +FROM microsoft/azure-cli:2.0.47 + +LABEL version="0.0.0" +LABEL maintainer="eric and sai" +LABEL com.github.actions.name="Test kata-deploy in an AKS cluster" +LABEL com.github.actions.description="Wow. Where do i start. Create an AKS cluster with containerd+runtimeclass, then deploys kata onto it and even might start a workload. nbd" + +ARG AKS_ENGINE_VER="v0.36.4" + +ENV GITHUB_ACTION_NAME="Test kata-deploy in an AKS cluster" + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + +RUN curl -LO https://github.com/Azure/aks-engine/releases/download/${AKS_ENGINE_VER}/aks-engine-${AKS_ENGINE_VER}-linux-amd64.tar.gz \ + && tar xvf aks-engine-${AKS_ENGINE_VER}-linux-amd64.tar.gz \ + && mv aks-engine-${AKS_ENGINE_VER}-linux-amd64/aks-engine /usr/local/bin/aks-engine \ + && rm aks-engine-${AKS_ENGINE_VER}-linux-amd64.tar.gz + +COPY kubernetes-containerd.json / +COPY setup-aks.sh test-kata.sh entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/kata-deploy-action/entrypoint.sh b/.github/kata-deploy-action/entrypoint.sh new file mode 100755 index 000000000..25cb40470 --- /dev/null +++ b/.github/kata-deploy-action/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +# Since this is the entrypoint for the container image, we know that the AKS and Kata setup/testing +# scripts are located at root. +source /setup-aks.sh +source /test-kata.sh + +trap destroy_aks EXIT + +setup_aks + +test_kata diff --git a/.github/kata-deploy-action/kubernetes-containerd.json b/.github/kata-deploy-action/kubernetes-containerd.json new file mode 100644 index 000000000..ffcaeaa42 --- /dev/null +++ b/.github/kata-deploy-action/kubernetes-containerd.json @@ -0,0 +1,41 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorVersion": "1.14.1", + "kubernetesConfig": { + "networkPlugin": "flannel", + "containerRuntime": "containerd", + "containerdVersion": "1.2.4" + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 1, + "vmSize": "Standard_D4s_v3", + "availabilityProfile": "AvailabilitySet" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/.github/kata-deploy-action/setup-aks.sh b/.github/kata-deploy-action/setup-aks.sh new file mode 100755 index 000000000..e8109edea --- /dev/null +++ b/.github/kata-deploy-action/setup-aks.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +export AZURE_HTTP_USER_AGENT="GITHUBACTIONS_${GITHUB_ACTION_NAME}_${GITHUB_REPOSITORY}" + +LOCATION=${LOCATION:-westus2} +DNS_PREFIX=${DNS_PREFIX:-kata-deploy-${GITHUB_SHA:0:10}} +CLUSTER_CONFIG=${CLUSTER_CONFIG:-/kubernetes-containerd.json} + +function die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function destroy_aks() { + set +x + az login --service-principal -u "$AZ_APPID" -p "$AZ_PASSWORD" --tenant "$AZ_TENANT_ID" + az group delete --name "$DNS_PREFIX" --yes --no-wait + az logout +} + +function setup_aks() { + + [[ -z "$AZ_APPID" ]] && die "no Azure service principal ID provided" + [[ -z "$AZ_PASSWORD" ]] && die "no Azure service principal secret provided" + [[ -z "$AZ_SUBSCRIPTION_ID" ]] && die "no Azure subscription ID provided" + [[ -z "$AZ_TENANT_ID" ]] && die "no Azure tenant ID provided" + + # check cluster config existence + # TODO + + # Give it a try + + aks-engine deploy --subscription-id "$AZ_SUBSCRIPTION_ID" \ + --client-id "$AZ_APPID" --client-secret "$AZ_PASSWORD" \ + --location "$LOCATION" --dns-prefix "$DNS_PREFIX" \ + --api-model "$CLUSTER_CONFIG" --force-overwrite + + export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" +} diff --git a/.github/kata-deploy-action/test-kata.sh b/.github/kata-deploy-action/test-kata.sh new file mode 100755 index 000000000..993267761 --- /dev/null +++ b/.github/kata-deploy-action/test-kata.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + + +function waitForProcess() { + wait_time="$1" + sleep_time="$2" + cmd="$3" + while [ "$wait_time" -gt 0 ]; do + if eval "$cmd"; then + return 0 + else + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + return 1 +} + + + +function run_test() { + YAMLPATH="https://raw.githubusercontent.com/egernst/kata-deploy/$GITHUB_SHA/kata-deploy" + echo "verify connectivity with a pod using Kata" + + deployment="" + busybox_pod="test-nginx" + busybox_image="busybox" + cmd="kubectl get pods | grep $busybox_pod | grep Completed" + wait_time=120 + sleep_time=3 + + for deployment in "nginx-deployment-qemu" "nginx-deployment-nemu"; do + # start the kata pod: + kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml" + kubectl wait --timeout=5m --for=condition=Available deployment/${deployment} + kubectl wait --timeout=5m --for=condition=Available deployment/${deployment} + kubectl expose deployment/${deployment} + + # test pod connectivity: + kubectl run $busybox_pod --restart=Never --image="$busybox_image" -- wget --timeout=5 "$deployment" + waitForProcess "$wait_time" "$sleep_time" "$cmd" + kubectl logs "$busybox_pod" | grep "index.html" + kubectl describe pod "$busybox_pod" + + kubectl delete deployment "$deployment" + kubectl delete service "$deployment" + kubectl delete pod "$busybox_pod" + done +} + + +function test_kata() { + set -x + #kubectl all the things + kubectl get pods --all-namespaces + + YAMLPATH="https://raw.githubusercontent.com/egernst/kata-deploy/$GITHUB_SHA/kata-deploy" + + kubectl apply -f "$YAMLPATH/kata-rbac.yaml" + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-nemu-runtimeClass.yaml" + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-runtimeClass.yaml" + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-fc-runtimeClass.yaml" + + sleep 5 + + kubectl get runtimeclasses + + wget "$YAMLPATH/kata-deploy.yaml" + wget "$YAMLPATH/kata-cleanup.yaml" + + # update deployment daemonset to utilize the container under test: + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${GITHUB_SHA}#g" kata-deploy.yaml + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${GITHUB_SHA}#g" kata-cleanup.yaml + + cat kata-deploy.yaml + + sleep 100 + + # deploy kata: + kubectl apply -f kata-deploy.yaml + + sleep 1 + + #wait for kata-deploy to be up + kubectl -n kube-system wait --timeout=5m --for=condition=Ready -l name=kata-deploy pod + + #Do I see this? + kubectl get pods --all-namespaces --show-labels + kubectl get node --show-labels + + run_test + + # remove kata (yeah, we are about to destroy, but good to test this flow as well): + kubectl delete -f kata-deploy.yaml + kubectl -n kube-system wait --timeout=5m --for=delete -l name=kata-deploy pod + kubectl apply -f kata-cleanup.yaml + kubectl -n kube-system wait --timeout=5m --for=condition=Ready -l name=kubelet-kata-cleanup pod + + kubectl get pods --all-namespaces --show-labels + kubectl get node --show-labels + + kubectl delete -f kata-cleanup.yaml + + rm kata-cleanup.yaml + rm kata-deploy.yaml + + set +x +} diff --git a/.github/kata-deploy-action/trigger b/.github/kata-deploy-action/trigger new file mode 100755 index 000000000..12348c3aa --- /dev/null +++ b/.github/kata-deploy-action/trigger @@ -0,0 +1,4 @@ +VERSION=1.8.0-alpha1 +git tag --delete $VERSION +git push origin :$VERSION +git tag -a $VERSION -m "test tag - $VERSION" && git push origin $VERSION diff --git a/.github/workflows/kata-release.yml b/.github/workflows/kata-release.yml new file mode 100644 index 000000000..6a0e16f6c --- /dev/null +++ b/.github/workflows/kata-release.yml @@ -0,0 +1,55 @@ +# When a release page is published, start the release artifact process +on: release +name: Build, Test, and Publish kata-deploy + +jobs: + # create image and upload to release page (can we get branch information from release tag? + publish-artifacts: + runs-on: ubuntu-latest + steps: + - name: publish-images + - uses: TBD + with: + args: tag? sha? + - name: create-static-binaries + - uses: TBD + with: tag? + + # test the artifacts + kata-deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: tag-filter + uses: actions/bin/filter@master + with: + args: tag + - name: docker-build + uses: actions/docker/cli@master + with: + args: build --build-arg KATA_VER=${GITHUB_REF##*/} -t katadocker/kata-deploy-ci:${{ + github.sha }} ./kata-deploy + - name: docker-login + uses: actions/docker/login@master + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + - name: docker-push-sha + uses: actions/docker/cli@master + with: + args: push katadocker/kata-deploy-ci:${{ github.sha }} + - name: aks-test + uses: ./kata-deploy/action + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} + - name: docker-tag-ref + uses: actions/docker/cli@master + with: + args: tag katadocker/kata-deploy-ci:${{ github.sha }} katadocker/kata-deploy:${GITHUB_REF##*/} + - name: docker-push-ref + uses: actions/docker/cli@master + with: + args: push katadocker/kata-deploy:${GITHUB_REF##*/}