CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #7083
Signed-off-by: stevenhorsman <steven@uk.ibm.com>

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -2,6 +2,10 @@ name: CI | Build kata-static tarball for amd64
 on:
   workflow_call:
     inputs:
+      stage:
+        required: false
+        type: string
+        default: test
       tarball-suffix:
         required: false
         type: string
@@ -15,8 +19,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        stage:
+          - ${{ inputs.stage }}
         asset:
           - cloud-hypervisor
+          - cloud-hypervisor-glibc
           - firecracker
           - kernel
           - kernel-sev
@@ -32,11 +39,16 @@ jobs:
           - qemu-snp-experimental
           - qemu-tdx-experimental
           - rootfs-image
+          - rootfs-image-tdx
           - rootfs-initrd
+          - rootfs-initrd-mariner
           - rootfs-initrd-sev
           - shim-v2
           - tdvf
           - virtiofsd
+        exclude:
+          - stage: release
+            asset: cloud-hypervisor-glibc
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}

.github/workflows/ci-on-push.yaml

@@ -52,3 +52,7 @@ jobs:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64
+
+  run-metrics-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-launchtimes-metrics.yaml

.github/workflows/release-amd64.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   build-kata-static-tarball-amd64:
     uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      stage: release
 
   kata-deploy:
     needs: build-kata-static-tarball-amd64

.github/workflows/release-arm64.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   build-kata-static-tarball-arm64:
     uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      stage: release
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64

.github/workflows/release-s390x.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   build-kata-static-tarball-s390x:
     uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      stage: release
 
   kata-deploy:
     needs: build-kata-static-tarball-s390x

.github/workflows/release.yaml

@@ -83,7 +83,7 @@ jobs:
       - name: push amd64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
+          tarball="kata-static-$tag-amd64.tar.xz"
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
@@ -97,7 +97,7 @@ jobs:
       - name: push arm64 static tarball to github
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-aarch64.tar.xz"
+          tarball="kata-static-$tag-arm64.tar.xz"
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -17,79 +17,54 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        host_os:
+          - ubuntu
         vmm:
           - clh
           - dragonball
           - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
     runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ github.event.pull_request.number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Download Azure CLI
-        run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+        run: bash tests/integration/gha-run.sh install-azure-cli
 
       - name: Log into the Azure account
-        run: |
-          az login \
-            --service-principal \
-            -u "${{ secrets.AZ_APPID }}" \
-            -p "${{ secrets.AZ_PASSWORD }}" \
-            --tenant "${{ secrets.AZ_TENANT_ID }}"
+        run: bash tests/integration/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
 
       - name: Create AKS cluster
-        run: |
-          az aks create \
-            -g "kataCI" \
-            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
-            -s "Standard_D4s_v5" \
-            --node-count 1 \
-            --generate-ssh-keys
+        run: bash tests/integration/gha-run.sh create-cluster
 
       - name: Install `bats`
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install bats
+        run: bash tests/integration/gha-run.sh install-bats
 
       - name: Install `kubectl`
-        run: |
-          sudo az aks install-cli
+        run: bash tests/integration/gha-run.sh install-kubectl
 
       - name: Download credentials for the Kubernetes CLI to use them
-        run: |
-          az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64
+        run: bash tests/integration/gha-run.sh get-cluster-credentials
 
       - name: Run tests
-        timeout-minutes: 35
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 150s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        timeout-minutes: 60
+        run: bash tests/integration/gha-run.sh run-tests-aks
 
       - name: Delete AKS cluster
         if: always()
-        run: |
-          az aks delete \
-            -g "kataCI" \
-            -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \
-            --yes \
-            --no-wait
+        run: bash tests/integration/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -21,6 +21,10 @@ jobs:
           - qemu-sev
     runs-on: sev
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /home/kata/.kube/config
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
 
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-sev
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -21,6 +21,10 @@ jobs:
           - qemu-snp
     runs-on: sev-snp
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /home/kata/.kube/config
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
 
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-snp
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -21,6 +21,10 @@ jobs:
           - qemu-tdx
     runs-on: tdx
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /etc/rancher/k3s/k3s.yaml
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
 
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-tdx
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          sleep 180s
-
-          kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/gha-run.sh cleanup-tdx

.github/workflows/run-launchtimes-metrics.yaml

@@ -0,0 +1,19 @@
+name: CI | Run launch-times metrics
+on:
+  workflow_call:
+
+jobs:
+  launch-times-tests:
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: run launch times on qemu
+        run: bash tests/metrics/gha-run.sh run-test-launchtimes-qemu
+
+      - name: run launch times on clh
+        run:  bash tests/metrics/gha-run.sh run-test-launchtimes-clh

.github/workflows/snap-release.yaml

@@ -1,53 +0,0 @@
-name: Release Kata in snapcraft store
-on:
-  push:
-    tags:
-      - '[0-9]+.[0-9]+.[0-9]+*'
-
-env:
-  SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }}
-
-jobs:
-  release-snap:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        run: |
-          # Required to avoid snapcraft install failure
-          sudo chown root:root /
-
-          # "--classic" is needed for the GitHub action runner
-          # environment.
-          sudo snap install snapcraft --classic
-
-          # Allow other parts to access snap binaries
-          echo /snap/bin >> "$GITHUB_PATH"
-
-      - name: Build snap
-        run: |
-          # Removing man-db, workflow kept failing, fixes: #4480
-          sudo apt -y remove --purge man-db
-          sudo apt-get update
-          sudo apt-get install -y git git-extras
-          kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
-          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
-          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
-            # Current version is the latest version, build it
-            snapcraft snap --debug --destructive-mode
-          fi
-
-      - name: Upload snap
-        run: |
-          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          snap_file="kata-containers_${snap_version}_amd64.snap"
-          # Upload the snap if it exists
-          if [ -f ${snap_file} ]; then
-            snapcraft upload --release=stable ${snap_file}
-          fi

.github/workflows/snap.yaml

@@ -1,37 +0,0 @@
-name: snap CI
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - edited
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          # Required to avoid snapcraft install failure
-          sudo chown root:root /
-
-          # "--classic" is needed for the GitHub action runner
-          # environment.
-          sudo snap install snapcraft --classic
-
-          # Allow other parts to access snap binaries
-          echo /snap/bin >> "$GITHUB_PATH"
-
-      - name: Build snap
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          snapcraft snap --debug --destructive-mode