From 09cc0ed43853de13f0d0c21356f68dbb4d1d1d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 15:35:11 +0200 Subject: [PATCH 1/8] ci: Move deploy_k8s() to gha-run-k8s-common.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will allow us to re-use the function in the kata-deploy tests, which will come soon. Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 39 +++++++++++++++++++++++++ tests/integration/kubernetes/gha-run.sh | 39 ------------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index fd9b98109..1dfb13cc8 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -96,3 +96,42 @@ function get_nodes_and_pods_info() { kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest || true kubectl get pods -o name | grep node-debugger | xargs kubectl delete || true } + +function deploy_k3s() { + curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 + + # This is an arbitrary value that came up from local tests + sleep 120s + + # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl + # + # We need to do this to avoid hitting issues like: + # ```sh + # error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied + # ``` + # Which happens basically because k3s links `/usr/local/bin/kubectl` + # to `/usr/local/bin/k3s`, and that does extra stuff that vanilla + # `kubectl` doesn't do. + ARCH=$(uname -m) + if [ "${ARCH}" = "x86_64" ]; then + ARCH=amd64 + fi + kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + sudo rm -rf /usr/local/bin/kubectl + + mkdir -p ~/.kube + cp /etc/rancher/k3s/k3s.yaml ~/.kube/config +} + +function deploy_k8s() { + echo "::group::Deploying ${KUBERNETES}" + + case ${KUBERNETES} in + k3s) deploy_k3s ;; + *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; + esac + + echo "::endgroup::" +} diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d510bc295..a8650de31 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -143,45 +143,6 @@ function deploy_kata() { echo "::endgroup::" } -function deploy_k3s() { - curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 - - # This is an arbitrary value that came up from local tests - sleep 120s - - # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl - # - # We need to do this to avoid hitting issues like: - # ```sh - # error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied - # ``` - # Which happens basically because k3s links `/usr/local/bin/kubectl` - # to `/usr/local/bin/k3s`, and that does extra stuff that vanilla - # `kubectl` doesn't do. - ARCH=$(uname -m) - if [ "${ARCH}" = "x86_64" ]; then - ARCH=amd64 - fi - kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//') - sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl - sudo chmod +x /usr/bin/kubectl - sudo rm -rf /usr/local/bin/kubectl - - mkdir -p ~/.kube - cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -} - -function deploy_k8s() { - echo "::group::Deploying ${KUBERNETES}" - - case ${KUBERNETES} in - k3s) deploy_k3s ;; - *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; - esac - - echo "::endgroup::" -} - function run_tests() { # Delete any spurious tests namespace that was left behind kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true From 9e1fb8a96660d8c87a8c2e3effd52ff50603bbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 16:14:42 +0200 Subject: [PATCH 2/8] ci: kata-deploy: Export KUBERNETES env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we have a better control on which flavour of kubernetes kata-deploy is expected to be targetting. This was also done as part of fa62a4c01b9ffac59bfe8740ee9c82e4c4f2dc81, for the k8s tests. Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-kata-deploy-tests-on-aks.yaml | 1 + .github/workflows/run-kata-deploy-tests-on-tdx.yaml | 1 + tests/functional/kata-deploy/kata-deploy.bats | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-kata-deploy-tests-on-aks.yaml b/.github/workflows/run-kata-deploy-tests-on-aks.yaml index 8962fcf32..74fcb0ea5 100644 --- a/.github/workflows/run-kata-deploy-tests-on-aks.yaml +++ b/.github/workflows/run-kata-deploy-tests-on-aks.yaml @@ -44,6 +44,7 @@ jobs: GH_PR_NUMBER: ${{ inputs.pr-number }} KATA_HOST_OS: ${{ matrix.host_os }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-kata-deploy-tests-on-tdx.yaml b/.github/workflows/run-kata-deploy-tests-on-tdx.yaml index b9e7e0588..6b439cea8 100644 --- a/.github/workflows/run-kata-deploy-tests-on-tdx.yaml +++ b/.github/workflows/run-kata-deploy-tests-on-tdx.yaml @@ -36,6 +36,7 @@ jobs: DOCKER_TAG: ${{ inputs.tag }} PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "k3s" USING_NFD: "true" steps: - uses: actions/checkout@v3 diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index 84faf8fe1..95d32e2b9 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -48,7 +48,7 @@ setup() { echo "::endgroup::" kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k3s" ]; then kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s" else kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" @@ -74,7 +74,7 @@ setup() { teardown() { kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation" - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k3s" ]; then deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"" cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s"" else From b12b9e1886fae09b12c758bde6ed6860c4c28bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 16:27:53 +0200 Subject: [PATCH 3/8] ci: kata-deploy: Add placeholder for tests on GARM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We'll be testing kata-deploy with different kubernetes flavours as part of our GARM tests, and this is a place-holder for this. Once enabled, we'll do nothing, just `return 0`, so we can then properly add the tests after this commit gets merged. Signed-off-by: Fabiano Fidêncio --- .../run-kata-deploy-tests-on-garm.yaml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/run-kata-deploy-tests-on-garm.yaml diff --git a/.github/workflows/run-kata-deploy-tests-on-garm.yaml b/.github/workflows/run-kata-deploy-tests-on-garm.yaml new file mode 100644 index 000000000..016708896 --- /dev/null +++ b/.github/workflows/run-kata-deploy-tests-on-garm.yaml @@ -0,0 +1,65 @@ +name: CI | Run kata-deploy tests on GARM +on: + workflow_call: + inputs: + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + pr-number: + required: true + type: string + commit-hash: + required: false + type: string + target-branch: + required: false + type: string + default: "" + +jobs: + run-kata-deploy-tests: + strategy: + fail-fast: false + matrix: + vmm: + - clh + - qemu + k8s: + - k0s + - k3s + - rke2 + runs-on: garm-ubuntu-2004-small + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + PR_NUMBER: ${{ inputs.pr-number }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: ${{ matrix.k8s }} + USING_NFD: "false" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: Deploy ${{ matrix.k8s }} + run: bash tests/functional/kata-deploy/gha-run.sh deploy-k8s + + - name: Install `bats` + run: bash tests/functional/kata-deploy/gha-run.sh install-bats + + - name: Run tests + run: bash tests/functional/kata-deploy/gha-run.sh run-tests From bf2cb02283062de062ac4e4ecab4c6fc7fe05507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 16:34:38 +0200 Subject: [PATCH 4/8] ci: kata-deploy: Expland tests to run on k0s / rke2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We just need to make sure the correct overlay is applied, following what we already have been doing for k3s. Signed-off-by: Fabiano Fidêncio --- tests/functional/kata-deploy/kata-deploy.bats | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index 95d32e2b9..d4f957d05 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -48,8 +48,12 @@ setup() { echo "::endgroup::" kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - if [ "${KUBERNETES}" = "k3s" ]; then + if [ "${KUBERNETES}" = "k0s" ]; then + kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s" + elif [ "${KUBERNETES}" = "k3s" ]; then kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s" + elif [ "${KUBERNETES}" = "rke2" ]; then + kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2" else kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi @@ -74,12 +78,18 @@ setup() { teardown() { kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation" - if [ "${KUBERNETES}" = "k3s" ]; then - deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"" - cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + if [ "${KUBERNETES}" = "k0s" ]; then + deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\"" + cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\"" + elif [ "${KUBERNETES}" = "k3s" ]; then + deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\"" + cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\"" + elif [ "${KUBERNETES}" = "rke2" ]; then + deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\"" + cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\"" else - deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" - cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\"" + cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\"" fi kubectl delete ${deploy_spec} From 001525763606d19d7a891ff7715b212c65a1db02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 16:36:18 +0200 Subject: [PATCH 5/8] ci: kata-deploy: Add deploy-k8s argument to gha-run.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We'll be using exactly the same code used for the k8s tests, which are already deploying k3s on GARM. Signed-off-by: Fabiano Fidêncio --- tests/functional/kata-deploy/gha-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/kata-deploy/gha-run.sh b/tests/functional/kata-deploy/gha-run.sh index 8e1c9a1d0..744e73f1c 100755 --- a/tests/functional/kata-deploy/gha-run.sh +++ b/tests/functional/kata-deploy/gha-run.sh @@ -56,6 +56,7 @@ function main() { install-azure-cli) install_azure_cli ;; login-azure) login_azure ;; create-cluster) create_cluster "kata-deploy" ;; + deploy-k8s) deploy_k8s ;; install-bats) install_bats ;; install-kubectl) install_kubectl ;; get-cluster-credentials) get_cluster_credentials "kata-deploy" ;; From eaf61649163a3dc7d60df48a42152fd163bac6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 15:44:10 +0200 Subject: [PATCH 6/8] ci: kata-deploy: Add the ability to deploy k0s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will be very useful in the near future, when we start testing kata-deploy with k0s as well. Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 1dfb13cc8..ba25a7539 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -97,6 +97,31 @@ function get_nodes_and_pods_info() { kubectl get pods -o name | grep node-debugger | xargs kubectl delete || true } +function deploy_k0s() { + curl -sSLf https://get.k0s.sh | sudo sh + + sudo k0s install controller --single + + sudo k0s start + + # This is an arbitrary value that came up from local tests + sleep 120s + + # Download the kubectl binary into /usr/bin so we can avoid depending + # on `k0s kubectl` command + ARCH=$(uname -m) + if [ "${ARCH}" = "x86_64" ]; then + ARCH=amd64 + fi + kubectl_version=$(sudo k0s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + + mkdir -p ~/.kube + sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config + sudo chown ${USER}:${USER} ~/.kube/config +} + function deploy_k3s() { curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 @@ -129,6 +154,7 @@ function deploy_k8s() { echo "::group::Deploying ${KUBERNETES}" case ${KUBERNETES} in + k0s) deploy_k0s ;; k3s) deploy_k3s ;; *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac From 2c908b598cbb636732c82960ef240e3a3d3302fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 16:05:14 +0200 Subject: [PATCH 7/8] ci: kata-deploy: Add the ability to deploy rke2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will be very useful in the near future, when we start testing kata-deploy with rke2 as well. Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index ba25a7539..2def893f2 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -150,12 +150,29 @@ function deploy_k3s() { cp /etc/rancher/k3s/k3s.yaml ~/.kube/config } +function deploy_rke2() { + curl -sfL https://get.rke2.io | sudo sh - + + systemctl enable --now rke2-server.service + + # This is an arbitrary value that came up from local tests + sleep 120s + + # Link the kubectl binary into /usr/bin + sudo ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + + mkdir -p ~/.kube + sudo cp /etc/rancher/rke2/rke2.yaml ~/.kube/config + sudo chown ${USER}:${USER} ~/.kube/config +} + function deploy_k8s() { echo "::group::Deploying ${KUBERNETES}" case ${KUBERNETES} in k0s) deploy_k0s ;; k3s) deploy_k3s ;; + rke2) deploy_rke2 ;; *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac From f7fa7f602aa434d945adeeae9364b6d3709c8fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 11 Sep 2023 18:41:35 +0200 Subject: [PATCH 8/8] ci: Enable kata-deploy tests for all the supported k8s flavours MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's ensure we test kata-deploy on RKE2 and k0s as well. Fixes: #7890 Signed-off-by: Fabiano Fidêncio --- .github/workflows/ci.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b8ad8ab4..791db56c0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -101,6 +101,18 @@ jobs: target-branch: ${{ inputs.target-branch }} secrets: inherit + run-kata-deploy-tests-on-garm: + needs: publish-kata-deploy-payload-amd64 + uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml + with: + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ inputs.tag }}-amd64 + commit-hash: ${{ inputs.commit-hash }} + pr-number: ${{ inputs.pr-number }} + target-branch: ${{ inputs.target-branch }} + secrets: inherit + run-kata-deploy-tests-on-tdx: needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image] uses: ./.github/workflows/run-kata-deploy-tests-on-tdx.yaml