diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b8ad8ab4..791db56c0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -101,6 +101,18 @@ jobs: target-branch: ${{ inputs.target-branch }} secrets: inherit + run-kata-deploy-tests-on-garm: + needs: publish-kata-deploy-payload-amd64 + uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml + with: + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ inputs.tag }}-amd64 + commit-hash: ${{ inputs.commit-hash }} + pr-number: ${{ inputs.pr-number }} + target-branch: ${{ inputs.target-branch }} + secrets: inherit + run-kata-deploy-tests-on-tdx: needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image] uses: ./.github/workflows/run-kata-deploy-tests-on-tdx.yaml diff --git a/.github/workflows/run-kata-deploy-tests-on-aks.yaml b/.github/workflows/run-kata-deploy-tests-on-aks.yaml index 8962fcf32..74fcb0ea5 100644 --- a/.github/workflows/run-kata-deploy-tests-on-aks.yaml +++ b/.github/workflows/run-kata-deploy-tests-on-aks.yaml @@ -44,6 +44,7 @@ jobs: GH_PR_NUMBER: ${{ inputs.pr-number }} KATA_HOST_OS: ${{ matrix.host_os }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-kata-deploy-tests-on-garm.yaml b/.github/workflows/run-kata-deploy-tests-on-garm.yaml new file mode 100644 index 000000000..016708896 --- /dev/null +++ b/.github/workflows/run-kata-deploy-tests-on-garm.yaml @@ -0,0 +1,65 @@ +name: CI | Run kata-deploy tests on GARM +on: + workflow_call: + inputs: + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + pr-number: + required: true + type: string + commit-hash: + required: false + type: string + target-branch: + required: false + type: string + default: "" + +jobs: + run-kata-deploy-tests: + strategy: + fail-fast: false + matrix: + vmm: + - clh + - qemu + k8s: + - k0s + - k3s + - rke2 + runs-on: garm-ubuntu-2004-small + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + PR_NUMBER: ${{ inputs.pr-number }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: ${{ matrix.k8s }} + USING_NFD: "false" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: Deploy ${{ matrix.k8s }} + run: bash tests/functional/kata-deploy/gha-run.sh deploy-k8s + + - name: Install `bats` + run: bash tests/functional/kata-deploy/gha-run.sh install-bats + + - name: Run tests + run: bash tests/functional/kata-deploy/gha-run.sh run-tests diff --git a/.github/workflows/run-kata-deploy-tests-on-tdx.yaml b/.github/workflows/run-kata-deploy-tests-on-tdx.yaml index b9e7e0588..6b439cea8 100644 --- a/.github/workflows/run-kata-deploy-tests-on-tdx.yaml +++ b/.github/workflows/run-kata-deploy-tests-on-tdx.yaml @@ -36,6 +36,7 @@ jobs: DOCKER_TAG: ${{ inputs.tag }} PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "k3s" USING_NFD: "true" steps: - uses: actions/checkout@v3 diff --git a/tests/functional/kata-deploy/gha-run.sh b/tests/functional/kata-deploy/gha-run.sh index 8e1c9a1d0..744e73f1c 100755 --- a/tests/functional/kata-deploy/gha-run.sh +++ b/tests/functional/kata-deploy/gha-run.sh @@ -56,6 +56,7 @@ function main() { install-azure-cli) install_azure_cli ;; login-azure) login_azure ;; create-cluster) create_cluster "kata-deploy" ;; + deploy-k8s) deploy_k8s ;; install-bats) install_bats ;; install-kubectl) install_kubectl ;; get-cluster-credentials) get_cluster_credentials "kata-deploy" ;; diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index 84faf8fe1..d4f957d05 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -48,8 +48,12 @@ setup() { echo "::endgroup::" kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k0s" ]; then + kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s" + elif [ "${KUBERNETES}" = "k3s" ]; then kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s" + elif [ "${KUBERNETES}" = "rke2" ]; then + kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2" else kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi @@ -74,12 +78,18 @@ setup() { teardown() { kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation" - if [ "${platform}" = "tdx" ]; then - deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"" - cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + if [ "${KUBERNETES}" = "k0s" ]; then + deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\"" + cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\"" + elif [ "${KUBERNETES}" = "k3s" ]; then + deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\"" + cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\"" + elif [ "${KUBERNETES}" = "rke2" ]; then + deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\"" + cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\"" else - deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" - cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\"" + cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\"" fi kubectl delete ${deploy_spec} diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 5b8193c88..b4631614f 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -111,3 +111,85 @@ function get_nodes_and_pods_info() { kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest || true kubectl get pods -o name | grep node-debugger | xargs kubectl delete || true } + +function deploy_k0s() { + curl -sSLf https://get.k0s.sh | sudo sh + + sudo k0s install controller --single + + sudo k0s start + + # This is an arbitrary value that came up from local tests + sleep 120s + + # Download the kubectl binary into /usr/bin so we can avoid depending + # on `k0s kubectl` command + ARCH=$(uname -m) + if [ "${ARCH}" = "x86_64" ]; then + ARCH=amd64 + fi + kubectl_version=$(sudo k0s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + + mkdir -p ~/.kube + sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config + sudo chown ${USER}:${USER} ~/.kube/config +} + +function deploy_k3s() { + curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 + + # This is an arbitrary value that came up from local tests + sleep 120s + + # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl + # + # We need to do this to avoid hitting issues like: + # ```sh + # error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied + # ``` + # Which happens basically because k3s links `/usr/local/bin/kubectl` + # to `/usr/local/bin/k3s`, and that does extra stuff that vanilla + # `kubectl` doesn't do. + ARCH=$(uname -m) + if [ "${ARCH}" = "x86_64" ]; then + ARCH=amd64 + fi + kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + sudo rm -rf /usr/local/bin/kubectl + + mkdir -p ~/.kube + cp /etc/rancher/k3s/k3s.yaml ~/.kube/config +} + +function deploy_rke2() { + curl -sfL https://get.rke2.io | sudo sh - + + systemctl enable --now rke2-server.service + + # This is an arbitrary value that came up from local tests + sleep 120s + + # Link the kubectl binary into /usr/bin + sudo ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + + mkdir -p ~/.kube + sudo cp /etc/rancher/rke2/rke2.yaml ~/.kube/config + sudo chown ${USER}:${USER} ~/.kube/config +} + +function deploy_k8s() { + echo "::group::Deploying ${KUBERNETES}" + + case ${KUBERNETES} in + k0s) deploy_k0s ;; + k3s) deploy_k3s ;; + rke2) deploy_rke2 ;; + *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; + esac + + echo "::endgroup::" +} diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d510bc295..a8650de31 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -143,45 +143,6 @@ function deploy_kata() { echo "::endgroup::" } -function deploy_k3s() { - curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 - - # This is an arbitrary value that came up from local tests - sleep 120s - - # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl - # - # We need to do this to avoid hitting issues like: - # ```sh - # error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied - # ``` - # Which happens basically because k3s links `/usr/local/bin/kubectl` - # to `/usr/local/bin/k3s`, and that does extra stuff that vanilla - # `kubectl` doesn't do. - ARCH=$(uname -m) - if [ "${ARCH}" = "x86_64" ]; then - ARCH=amd64 - fi - kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//') - sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl - sudo chmod +x /usr/bin/kubectl - sudo rm -rf /usr/local/bin/kubectl - - mkdir -p ~/.kube - cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -} - -function deploy_k8s() { - echo "::group::Deploying ${KUBERNETES}" - - case ${KUBERNETES} in - k3s) deploy_k3s ;; - *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; - esac - - echo "::endgroup::" -} - function run_tests() { # Delete any spurious tests namespace that was left behind kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true