From 54c0a471b1faedd427535ff9727a3cc02be9327e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 19 Sep 2023 19:45:20 +0200 Subject: [PATCH 1/4] ci: k8s: k0s: Allow passing parameters to the k0s installer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We'll need this in order to setup k0s with a different container engine. Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index b4631614f..6c6622b2b 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -115,7 +115,7 @@ function get_nodes_and_pods_info() { function deploy_k0s() { curl -sSLf https://get.k0s.sh | sudo sh - sudo k0s install controller --single + sudo k0s install controller --single ${KUBERNETES_EXTRA_PARAMS:-} sudo k0s start From d7105cf7a498bdac1b057007e4977f8cf0e14f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 19 Sep 2023 19:46:10 +0200 Subject: [PATCH 2/4] ci: k8s: Add a method to install CRI-O MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is based on official CRI-O documentations[0] and right now we're making this specific to Ubuntu as that's what we have as runners. We may want to expand this in the future, but we're good for now. [0]: https://github.com/cri-o/cri-o/blob/main/install.md#apt-based-operating-systems Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 47 +++++++++++++++++++++++++ tests/integration/kubernetes/gha-run.sh | 1 + 2 files changed, 48 insertions(+) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 6c6622b2b..f706db5eb 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -181,6 +181,53 @@ function deploy_rke2() { sudo chown ${USER}:${USER} ~/.kube/config } +function _get_k0s_kubernetes_version_for_crio() { + # k0s version will look like: + # v1.27.5+k0s.0 + # + # The CRI-O repo for such version of Kubernetes expects something like: + # 1.27 + k0s_version=$(curl -sSLf "https://docs.k0sproject.io/stable.txt") + + # Remove everything after the second '.' + crio_version=${k0s_version%\.*+*} + # Remove the 'v' + crio_version=${crio_version#v} + + echo ${crio_version} +} + +function _get_os_for_crio() { + source /etc/os-release + + if [ "${NAME}" != "Ubuntu" ]; then + echo "Only Ubuntu is supported for now" + exit 2 + fi + + echo "x${NAME}_${VERSION_ID}" +} + +function setup_crio() { + # Get the CRI-O version to be installed depending on the version of the + # "k8s distro" that we are using + case ${KUBERNETES} in + k0s) crio_version=$(_get_k0s_kubernetes_version_for_crio) ;; + *) >&2 echo "${KUBERNETES} flavour is not supported with CRI-O"; exit 2 ;; + + esac + + os=$(_get_os_for_crio) + + echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/${os}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list + echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/${crio_version}/${os}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:${crio_version}.list + curl -L https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:${crio_version}/${os}/Release.key | sudo apt-key add - + curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/${os}/Release.key | sudo apt-key add - + sudo apt update + sudo apt install cri-o cri-o-runc + sudo systemctl enable --now crio +} + function deploy_k8s() { echo "::group::Deploying ${KUBERNETES}" diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index a8650de31..bfde44342 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -215,6 +215,7 @@ function main() { login-azure) login_azure ;; create-cluster) create_cluster ;; configure-snapshotter) configure_snapshotter ;; + setup-crio) setup_crio ;; deploy-k8s) deploy_k8s ;; install-bats) install_bats ;; install-kubectl) install_kubectl ;; From 03b82e84840d2101e334484338b522c487b25d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 19 Sep 2023 19:44:14 +0200 Subject: [PATCH 3/4] ci: k8s: Add a CRI-O test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's make sure we'll also be testing k8s using CRI-O. For now, we'll only be running the CRI-O test with QEMU. Once it becomes stable we can expand this to other Hypervisors as well. Fixes: #8005 Signed-off-by: Fabiano Fidêncio --- .../run-k8s-tests-with-crio-on-garm.yaml | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/run-k8s-tests-with-crio-on-garm.yaml diff --git a/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml b/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml new file mode 100644 index 000000000..14000dc63 --- /dev/null +++ b/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml @@ -0,0 +1,86 @@ +name: CI | Run kubernetes tests, using CRI-O, on GARM +on: + workflow_call: + inputs: + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + pr-number: + required: true + type: string + commit-hash: + required: false + type: string + target-branch: + required: false + type: string + default: "" + +jobs: + run-k8s-tests: + strategy: + fail-fast: false + matrix: + vmm: + - qemu + k8s: + - k0s + instance: + - garm-ubuntu-2004 + - garm-ubuntu-2004-smaller + include: + - instance: garm-ubuntu-2004 + instance-type: normal + - instance: garm-ubuntu-2004-smaller + instance-type: small + - k8s: k0s + k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"' + runs-on: ${{ matrix.instance }} + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + PR_NUMBER: ${{ inputs.pr-number }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: ${{ matrix.k8s }} + KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }} + USING_NFD: "false" + K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit-hash }} + fetch-depth: 0 + + - name: Rebase atop of the latest target branch + run: | + ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" + env: + TARGET_BRANCH: ${{ inputs.target-branch }} + + - name: Configure CRI-O + run: bash tests/integration/kubernetes/gha-run.sh setup-crio + + - name: Deploy ${{ matrix.k8s }} + run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s + + - name: Deploy Kata + timeout-minutes: 10 + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats + + - name: Run tests + timeout-minutes: 30 + run: bash tests/integration/kubernetes/gha-run.sh run-tests + + - name: Delete kata-deploy + if: always() + run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm From 07a6e63a6bd96ba72ac99b35cd182a0cf56c397d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 20 Sep 2023 08:48:29 +0200 Subject: [PATCH 4/4] ci: k8s: rke2: Use sudo to call systemd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise we'll face the following error: ``` Failed to enable unit: Interactive authentication required. ``` Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index f706db5eb..01d825c28 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -168,7 +168,7 @@ function deploy_k3s() { function deploy_rke2() { curl -sfL https://get.rke2.io | sudo sh - - systemctl enable --now rke2-server.service + sudo systemctl enable --now rke2-server.service # This is an arbitrary value that came up from local tests sleep 120s