From 9e1710674ae9c35c02d64bf70fa841376f313198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 21 Jul 2023 15:30:53 +0200 Subject: [PATCH 01/11] kata-runtimeClasses: Alphabetically sort the enrties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will become handy in the near future, as we want to have separate enrties for each file, while still keeping this one. Having the entries sorted will make our lives easier to test those are always in sync. Signed-off-by: Fabiano Fidêncio --- .../runtimeclasses/kata-runtimeClasses.yaml | 130 +++++++++--------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml index b55096226..e67cafe10 100644 --- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml @@ -1,58 +1,6 @@ --- kind: RuntimeClass apiVersion: node.k8s.io/v1 -metadata: - name: kata-qemu -handler: kata-qemu -overhead: - podFixed: - memory: "160Mi" - cpu: "250m" -scheduling: - nodeSelector: - katacontainers.io/kata-runtime: "true" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1 -metadata: - name: kata-qemu-sev -handler: kata-qemu-sev -overhead: - podFixed: - memory: "2048Mi" - cpu: "1.0" -scheduling: - nodeSelector: - katacontainers.io/kata-runtime: "true" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1 -metadata: - name: kata-qemu-tdx -handler: kata-qemu-tdx -overhead: - podFixed: - memory: "2048Mi" - cpu: "1.0" -scheduling: - nodeSelector: - katacontainers.io/kata-runtime: "true" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1 -metadata: - name: kata-qemu-snp -handler: kata-qemu-snp -overhead: - podFixed: - memory: "2048Mi" - cpu: "1.0" -scheduling: - nodeSelector: - katacontainers.io/kata-runtime: "true" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1 metadata: name: kata-clh handler: kata-clh @@ -66,19 +14,6 @@ scheduling: --- kind: RuntimeClass apiVersion: node.k8s.io/v1 -metadata: - name: kata-fc -handler: kata-fc -overhead: - podFixed: - memory: "130Mi" - cpu: "250m" -scheduling: - nodeSelector: - katacontainers.io/kata-runtime: "true" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1 metadata: name: kata-dragonball handler: kata-dragonball @@ -92,6 +27,19 @@ scheduling: --- kind: RuntimeClass apiVersion: node.k8s.io/v1 +metadata: + name: kata-fc +handler: kata-fc +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 metadata: name: kata-qemu-nvidia-gpu handler: kata-qemu-nvidia-gpu @@ -102,3 +50,55 @@ overhead: scheduling: nodeSelector: katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-sev +handler: kata-qemu-sev +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-snp +handler: kata-qemu-snp +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-tdx +handler: kata-qemu-tdx +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu +handler: kata-qemu +overhead: + podFixed: + memory: "160Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" From 69535b8089da84a3bdfe189b7796a109de110885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 21 Jul 2023 15:32:17 +0200 Subject: [PATCH 02/11] kata-deploy: runtimeclass: Split out entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will make things simpler to only create the handlers defined by the kata-deploy user. Signed-off-by: Fabiano Fidêncio --- .../kata-deploy/runtimeclasses/kata-clh.yaml | 13 +++++++++++++ .../kata-deploy/runtimeclasses/kata-dragonball.yaml | 13 +++++++++++++ .../kata-deploy/runtimeclasses/kata-fc.yaml | 13 +++++++++++++ .../runtimeclasses/kata-qemu-nvidia-gpu.yaml | 13 +++++++++++++ .../kata-deploy/runtimeclasses/kata-qemu-sev.yaml | 13 +++++++++++++ .../kata-deploy/runtimeclasses/kata-qemu-snp.yaml | 13 +++++++++++++ .../kata-deploy/runtimeclasses/kata-qemu-tdx.yaml | 13 +++++++++++++ .../kata-deploy/runtimeclasses/kata-qemu.yaml | 13 +++++++++++++ 8 files changed, 104 insertions(+) create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-clh.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-dragonball.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-fc.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-qemu-nvidia-gpu.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-qemu-sev.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-qemu-snp.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-qemu-tdx.yaml create mode 100644 tools/packaging/kata-deploy/runtimeclasses/kata-qemu.yaml diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-clh.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-clh.yaml new file mode 100644 index 000000000..3c2260252 --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-clh.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-clh +handler: kata-clh +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-dragonball.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-dragonball.yaml new file mode 100644 index 000000000..664822c94 --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-dragonball.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-dragonball +handler: kata-dragonball +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-fc.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-fc.yaml new file mode 100644 index 000000000..2a087cc8d --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-fc.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-fc +handler: kata-fc +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-nvidia-gpu.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-nvidia-gpu.yaml new file mode 100644 index 000000000..f99d3a280 --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-nvidia-gpu.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-nvidia-gpu +handler: kata-qemu-nvidia-gpu +overhead: + podFixed: + memory: "160Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-sev.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-sev.yaml new file mode 100644 index 000000000..a9eb75a8e --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-sev.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-sev +handler: kata-qemu-sev +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-snp.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-snp.yaml new file mode 100644 index 000000000..a285d616d --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-snp.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-snp +handler: kata-qemu-snp +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-tdx.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-tdx.yaml new file mode 100644 index 000000000..e684291d7 --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu-tdx.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-tdx +handler: kata-qemu-tdx +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-qemu.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu.yaml new file mode 100644 index 000000000..c362e3ff6 --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-qemu.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu +handler: kata-qemu +overhead: + podFixed: + memory: "160Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" From a274333248b3c66c5bda36aba818c20f67695caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 26 Jul 2023 16:07:59 +0200 Subject: [PATCH 03/11] kata-deploy: Change default values of DEBUG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This can be easily done as there was no official release with the previous values. The reason we're doing so is because when using `yq` to replace the value, even when forcing `--tag '!!str' "yes"`, the content is placed without quotes, causing errors in our CI. While here, we're also removing the fallback value for DEBUG, as it is **always** set in the kata-deploy.yaml file. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 2 +- .../kata-deploy/kata-cleanup/base/kata-cleanup.yaml | 2 +- .../kata-deploy/kata-deploy/base/kata-deploy.yaml | 2 +- tools/packaging/kata-deploy/scripts/kata-deploy.sh | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d6d58ac37..eba290081 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -71,7 +71,7 @@ function deploy_kata() { sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable debug for Kata Containers - yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' "\"yes\"" + yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true" if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index e8b23e60f..9ef9e04fa 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -27,7 +27,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: DEBUG - value: "no" + value: "false" - name: SHIMS value: "clh dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu" - name: DEFAULT_SHIM diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index 0c688b8c8..ccab22c34 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -29,7 +29,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: DEBUG - value: "no" + value: "false" - name: SHIMS value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx" - name: DEFAULT_SHIM diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index ae39cd6a9..05f1af213 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -58,7 +58,7 @@ function install_artifacts() { chmod +x /opt/kata/runtime-rs/bin/* # Allow enabling debug for Kata Containers - if [[ "${DEBUG:-"no"}" == "yes" ]]; then + if [[ "${DEBUG}" == "true" ]]; then config_path="/opt/kata/share/defaults/kata-containers/" for shim in "${shims[@]}"; do sed -i -e 's/^#\(enable_debug\).*=.*$/\1 = true/g' "${config_path}/configuration-${shim}.toml" @@ -216,7 +216,7 @@ function configure_crio() { done - if [ "${DEBUG:-"no"}" == "yes" ]; then + if [ "${DEBUG}" == "true" ]; then cat < Date: Fri, 21 Jul 2023 13:41:55 +0200 Subject: [PATCH 04/11] kata-deploy: Allow runtimeclasses to be created by the daemonset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's allow the daemonset to create the runtimeclasses, which will decrease one manual step a user of kata-deploy should take, and also help us in the Confidential Containers land as the Operator can just delegate it to this script. Fixes: #7409 Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/Dockerfile | 1 + .../kata-cleanup/base/kata-cleanup.yaml | 6 ++- .../kata-deploy/base/kata-deploy.yaml | 6 ++- .../kata-deploy/kata-rbac/base/kata-rbac.yaml | 14 +++--- .../kata-deploy/scripts/kata-deploy.sh | 43 +++++++++++++++++++ 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index f8b9edf4c..8e7f6e2ac 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -28,3 +28,4 @@ tar xvf ${WORKDIR}/${KATA_ARTIFACTS} -C ${DESTINATION} && \ rm -f ${WORKDIR}/${KATA_ARTIFACTS} COPY scripts ${DESTINATION}/scripts +COPY runtimeclasses ${DESTINATION}/runtimeclasses diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index 9ef9e04fa..36d1b9b5b 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -13,7 +13,7 @@ spec: labels: name: kubelet-kata-cleanup spec: - serviceAccountName: kata-label-node + serviceAccountName: kata-deploy-sa nodeSelector: katacontainers.io/kata-runtime: cleanup containers: @@ -32,6 +32,10 @@ spec: value: "clh dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu" - name: DEFAULT_SHIM value: "qemu" + - name: CREATE_RUNTIMECLASSES + value: "false" + - name: CREATE_DEFAULT_RUNTIMECLASS + value: "false" securityContext: privileged: true volumeMounts: diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index ccab22c34..5431a47fa 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -13,7 +13,7 @@ spec: labels: name: kata-deploy spec: - serviceAccountName: kata-label-node + serviceAccountName: kata-deploy-sa containers: - name: kube-kata image: quay.io/kata-containers/kata-deploy:latest @@ -34,6 +34,10 @@ spec: value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx" - name: DEFAULT_SHIM value: "qemu" + - name: CREATE_RUNTIMECLASSES + value: "false" + - name: CREATE_DEFAULT_RUNTIMECLASS + value: "false" securityContext: privileged: true volumeMounts: diff --git a/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml b/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml index 408b5be90..3bde9f0a8 100644 --- a/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml +++ b/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml @@ -2,28 +2,30 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: kata-label-node + name: kata-deploy-sa namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: node-labeler + name: kata-deploy-role rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "patch"] +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: kata-label-node-rb + name: kata-deploy-rb roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: node-labeler + name: kata-deploy-role subjects: - kind: ServiceAccount - name: kata-label-node + name: kata-deploy-sa namespace: kube-system - diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 05f1af213..a1eda4b64 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -29,6 +29,41 @@ function print_usage() { echo "Usage: $0 [install/cleanup/reset]" } +function create_runtimeclasses() { + echo "Creating the runtime classes" + + for shim in "${shims[@]}"; do + echo "Creating the kata-${shim} runtime class" + kubectl apply -f /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml + done + + if [[ "${CREATE_DEFAULT_RUNTIMECLASS}" == "true" ]]; then + echo "Creating the kata runtime class for the default shim (an alias for kata-${default_shim})" + cp /opt/kata-artifacts/runtimeclasses/kata-${default_shim}.yaml /tmp/kata.yaml + sed -i -e 's/kata-'${default_shim}'/kata/g' /tmp/kata.yaml + kubectl apply -f /tmp/kata.yaml + rm -f /tmp/kata.yaml + fi +} + +function delete_runtimeclasses() { + echo "Deleting the runtime classes" + + for shim in "${shims[@]}"; do + echo "Deleting the kata-${shim} runtime class" + kubectl delete -f /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml + done + + + if [[ "${CREATE_DEFAULT_RUNTIMECLASS}" == "true" ]]; then + echo "Deleting the kata runtime class for the default shim (an alias for kata-${default_shim})" + cp /opt/kata-artifacts/runtimeclasses/kata-${default_shim}.yaml /tmp/kata.yaml + sed -i -e 's/kata-'${default_shim}'/kata/g' /tmp/kata.yaml + kubectl delete -f /tmp/kata.yaml + rm -f /tmp/kata.yaml + fi +} + function get_container_runtime() { local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') @@ -75,6 +110,10 @@ function install_artifacts() { sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}" sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}" fi + + if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then + create_runtimeclasses + fi } function wait_till_node_is_ready() { @@ -174,6 +213,10 @@ function cleanup_different_shims_base() { rm "${default_shim_file}" || true restore_shim "${default_shim_file}" + + if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then + delete_runtimeclasses + fi } function configure_crio_runtime() { From 39192c60841222c643d98841d1fc4468b5f08202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 26 Jul 2023 10:11:03 +0200 Subject: [PATCH 05/11] kata-deploy: Print variables passed to the script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will help folks to debug / understand what's been passed to the kata-deploy.sh script. Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/scripts/kata-deploy.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index a1eda4b64..5ed4df067 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -381,6 +381,14 @@ function reset_runtime() { } function main() { + echo "Environment variables passed to this script" + echo "* NODE_NAME: ${NODE_NAME}" + echo "* DEBUG: ${DEBUG}" + echo "* SHIMS: ${SHIMS}" + echo "* DEFAULT_SHIM: ${DEFAULT_SHIM}" + echo "* CREATE_RUNTIMECLASSES: ${CREATE_RUNTIMECLASSES}" + echo "* CREATE_DEFAULT_RUNTIMECLASS: ${CREATE_DEFAULT_RUNTIMECLASS}" + # script requires that user is root euid=$(id -u) if [[ $euid -ne 0 ]]; then From 87568ed98523183a0cda56d55ac1281672f7f643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 21 Jul 2023 15:52:58 +0200 Subject: [PATCH 06/11] gha: Test split out runtimeclasses are in sync with all-in-one file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed in order to not lose track of what's been created and what's been added here and there. Signed-off-by: Fabiano Fidêncio --- .../workflows/kata-runtime-classes-sync.yaml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/kata-runtime-classes-sync.yaml diff --git a/.github/workflows/kata-runtime-classes-sync.yaml b/.github/workflows/kata-runtime-classes-sync.yaml new file mode 100644 index 000000000..9cb995df1 --- /dev/null +++ b/.github/workflows/kata-runtime-classes-sync.yaml @@ -0,0 +1,36 @@ +on: + pull_request: + types: + - opened + - edited + - reopened + - synchronize + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + kata-deploy-runtime-classes-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Ensure the split out runtime classes match the all-in-one file + run: | + pushd tools/packaging/kata-deploy/runtimeclasses/ + echo "::group::Combine runtime classes" + for runtimeClass in `find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort`; do + echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml" + cat ${runtimeClass} >> resultingRuntimeClasses.yaml; + done + echo "::endgroup::" + echo "::group::Displaying the content of resultingRuntimeClasses.yaml" + cat resultingRuntimeClasses.yaml + echo "::endgroup::" + echo "" + echo "::group::Displaying the content of kata-runtimeClasses.yaml" + cat kata-runtimeClasses.yaml + echo "::endgroup::" + echo "" + diff resultingRuntimeClasses.yaml kata-runtimeClasses.yaml From 9d285c62264810b58918e94feaa3c7b7bf9c58ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 21 Jul 2023 14:25:21 +0200 Subject: [PATCH 07/11] ci: k8s: Let kata-deploy take care of the runtimeclasses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By doing this we can test the change done for the daemonset. :-) Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index eba290081..fd1dc8f3f 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -72,6 +72,8 @@ function deploy_kata() { # Enable debug for Kata Containers yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true" + # Let the `kata-deploy` script take care of the runtime class creation / removal + yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" @@ -87,7 +89,6 @@ function deploy_kata() { kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" echo "Gather information about the nodes and pods after having kata-deploy ready" get_nodes_and_pods_info @@ -100,6 +101,8 @@ function deploy_kata() { else sleep 60s fi + + kubectl get runtimeclass } function run_tests() { @@ -121,6 +124,7 @@ function run_tests() { function cleanup() { platform="${1}" + ensure_yq echo "Gather information about the nodes and pods before cleaning up the node" get_nodes_and_pods_info "yes" @@ -145,6 +149,9 @@ function cleanup() { kubectl delete ${deploy_spec} kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + # Let the `kata-deploy` script take care of the runtime class creation / removal + yq write -i "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true" + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" @@ -153,7 +160,6 @@ function cleanup() { kubectl delete ${cleanup_spec} kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" } function delete_cluster() { From 9cc6b5f461d76d490693a56a01bddade015f9e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 25 Jul 2023 14:40:03 +0200 Subject: [PATCH 08/11] ci: k8s: Get logs from kata-deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's make sure we can debug kata-deploy in case something goes wrong during its execution. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index fd1dc8f3f..69dcdf258 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -102,6 +102,7 @@ function deploy_kata() { sleep 60s fi + kubectl -n kube-system logs -l name=kata-deploy kubectl get runtimeclass } From 5261e3a60c481aed11f2351d116c1c3cc6efaf6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 26 Jul 2023 21:44:45 +0200 Subject: [PATCH 09/11] ci: k8s: Group messages to improve readability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right now is getting way too easy to get lost in the logs. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 69dcdf258..c702e62df 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -79,8 +79,11 @@ function deploy_kata() { yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}" fi + + echo "::group::Final kata-deploy.yaml that is used in the test" cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" + echo "::endgroup::" kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" if [ "${platform}" = "tdx" ]; then @@ -102,8 +105,13 @@ function deploy_kata() { sleep 60s fi + echo "::group::kata-deploy logs" kubectl -n kube-system logs -l name=kata-deploy + echo "::endgroup::" + + echo "::group::Runtime classes" kubectl get runtimeclass + echo "::endgroup::" } function run_tests() { From 6ad5d7112e02baafdb3b02c48b78cdab6f686157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 26 Jul 2023 21:45:39 +0200 Subject: [PATCH 10/11] ci: k8s: Do not gather node info before running the tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's been proven to not be useful, and ends up making things more confusing due to the amount of logs printed. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c702e62df..353dc3ba2 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -93,9 +93,6 @@ function deploy_kata() { fi kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - echo "Gather information about the nodes and pods after having kata-deploy ready" - get_nodes_and_pods_info - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, # which may cause issues like not having the node properly labeled or the artefacts # properly deployed when the tests actually start running. @@ -122,9 +119,6 @@ function run_tests() { kubectl apply -f ${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml kubectl config set-context --current --namespace=kata-containers-k8s-tests - echo "Gather information about the nodes and pods just before starting the tests" - get_nodes_and_pods_info - pushd "${kubernetes_dir}" bash setup.sh bash run_kubernetes_tests.sh From 8353aae41a279865725195c64e13d063e9e96a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 26 Jul 2023 21:48:18 +0200 Subject: [PATCH 11/11] ci: k8s: Rework get_nodes_and_pods_info() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The amount of info we've added seemed unnecessary, and ends up making our lives even harder when trying to find errors. Let's just rely on the kata-debug container to collect the needed info for us. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 353dc3ba2..0d02e7082 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -130,7 +130,7 @@ function cleanup() { ensure_yq echo "Gather information about the nodes and pods before cleaning up the node" - get_nodes_and_pods_info "yes" + get_nodes_and_pods_info if [ "${platform}" = "aks" ]; then delete_cluster @@ -173,21 +173,6 @@ function delete_cluster() { } function get_nodes_and_pods_info() { - describe_pods="${1:-"no"}" - - echo "::group::Get node information" - kubectl get nodes -o wide --show-labels=true - echo "::endgroup::" - echo "" - echo "::group::Get all the pods running" - kubectl get pods -A - echo "::endgroup::" - echo "" - if [[ "${describe_pods}" == "yes" ]]; then - echo "::group::Describe all the pods" - kubectl describe pods -A - echo "::endgroup::" - fi kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest kubectl get pods -o name | grep node-debugger | xargs kubectl delete }