Merge pull request #5045 from fidencio/topic/kata-deploy-misc-fixes

CC | kata-deploy: Misc fixes related to building the payload

docs/how-to/README.md

@@ -46,3 +46,4 @@
 
 ## Confidential Containers
 - [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
+- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -0,0 +1,44 @@
+# Generating a Kata Containers payload for the Confidential Containers Operator
+
+[Confidential Containers
+Operator](https://github.com/confidential-containers/operator) consumes a Kata
+Containers payload, generated from the `CCv0` branch, and here one can find all
+the necessary info on how to build such a payload.
+
+## Requirements
+
+* `make` installed in the machine
+* Docker installed in the machine
+* `sudo` access to the machine
+
+## Process
+
+* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
+  ```sh
+  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
+  ```
+  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
+    ```sh
+    git checkout CCv0
+    ```
+  * Ensure your tree is clean and in sync with upstream `CCv0`
+    ```sh
+    git clean -xfd
+    git reset --hard <upstream>/CCv0
+    ```
+* Make sure you're authenticated to `quay.io`
+    ```sh
+    sudo docker login quay.io
+    ```
+* From the top repo directory, run:
+  ```sh
+  sudo make cc-payload
+  ```
+* Make sure the image was upload to the [Confidential Containers
+  runtime-payload
+registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
+
+## Notes
+
+Make sure to run it on a machine that's not the one you're hacking on, prepare a
+cup of tea, and get back to it an hour later (at least).

tools/packaging/kata-deploy-cc/scripts/kata-deploy.sh

@@ -70,6 +70,20 @@ function configure_cri_runtime() {
 	systemctl restart "$1"
 }
 
+function backup_shim() {
+	local shim_file="$1"
+	local shim_backup="${shim_file}.bak"
+
+	if [ -f "${shim_file}" ]; then
+		echo "warning: ${shim_file} already exists" >&2
+		if [ ! -f "${shim_backup}" ]; then
+			mv "${shim_file}" "${shim_backup}"
+		else
+			rm "${shim_file}"
+		fi
+	fi
+}
+
 function configure_different_shims_base() {
 	# Currently containerd has an assumption on the location of the shimv2 implementation
 	# This forces kata-deploy to create files in a well-defined location that's part of
@@ -78,49 +92,49 @@ function configure_different_shims_base() {
 	#   https://github.com/containerd/containerd/issues/3073
 	#   https://github.com/containerd/containerd/issues/5006
 
+	local default_shim_file="/usr/local/bin/containerd-shim-kata-v2"
+
 	mkdir -p /usr/local/bin
 
 	for shim in "${shims[@]}"; do
 		local shim_binary="containerd-shim-kata-${shim}-v2"
 		local shim_file="/usr/local/bin/${shim_binary}"
-		local shim_backup="/usr/local/bin/${shim_binary}.bak"
 
-		if [ -f "${shim_file}" ]; then
-			echo "warning: ${shim_binary} already exists" >&2
-			if [ ! -f "${shim_backup}" ]; then
-				mv "${shim_file}" "${shim_backup}"
-			else
-				rm "${shim_file}"
-			fi
-		fi
-
-		cat << EOF | tee "$shim_file"
-#!/usr/bin/env bash
-KATA_CONF_FILE=/opt/confidential-containers/share/defaults/kata-containers/configuration-${shim}.toml /opt/confidential-containers/bin/containerd-shim-kata-v2 "\$@"
-EOF
+		backup_shim "${shim_file}"
+		ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}"
 		chmod +x "$shim_file"
 
 		if [ "${shim}" == "${default_shim}" ]; then
+			backup_shim "${default_shim_file}"
+
 			echo "Creating the default shim-v2 binary"
-			ln -sf "${shim_file}" /usr/local/bin/containerd-shim-kata-v2
+			ln -sf "${shim_file}" "${default_shim_file}"
 		fi
 	done
 }
 
+function restore_shim() {
+	local shim_file="$1"
+	local shim_backup="${shim_file}.bak"
+
+	if [ -f "${shim_backup}" ]; then
+		mv "$shim_backup" "$shim_file"
+	fi
+}
+
 function cleanup_different_shims_base() {
+	local default_shim_file="/usr/local/bin/containerd-shim-kata-v2"
+
 	for shim in "${shims[@]}"; do
 		local shim_binary="containerd-shim-kata-${shim}-v2"
 		local shim_file="/usr/local/bin/${shim_binary}"
-		local shim_backup="/usr/local/bin/${shim_binary}.bak"
 
 		rm "${shim_file}" || true
-
-		if [ -f "${shim_backup}" ]; then
-			mv "$shim_backup" "$shim_file"
-		fi
+		restore_shim "${shim_file}"
 	done
 
-	rm /usr/local/bin/containerd-shim-kata-v2
+	rm "${default_shim_file}" || true
+	restore_shim "${default_shim_file}"
 }
 
 function configure_containerd_runtime() {
@@ -199,16 +213,17 @@ function remove_artifacts() {
 		/opt/confidential-containers/bin/qemu-system-x86_64 \
 		/opt/confidential-containers/bin/qemu-system-x86_64-tdx \
 		/opt/confidential-containers/bin/cloud-hypervisor
+
+	# Try to remove the /opt/confidential-containers directory.
+	# If it's not empty, don't bother force removing it, as the
+	# pre-install script also drops files here.
+	rmdir /opt/confidential-containers 2>/dev/null
 }
 
 function cleanup_cri_runtime() {
 	cleanup_different_shims_base
 
-	case $1 in
-	containerd | k3s | k3s-agent | rke2-agent | rke2-server)
-		cleanup_containerd
-		;;
-	esac
+	cleanup_containerd
 
 }
 
@@ -223,9 +238,7 @@ function reset_runtime() {
 	kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime-
 	systemctl daemon-reload
 	systemctl restart "$1"
-	if [ "$1" == "containerd" ]; then
-		systemctl restart kubelet
-	fi
+	systemctl restart kubelet
 }
 
 function main() {
@@ -236,21 +249,13 @@ function main() {
 	fi
 
 	runtime=$(get_container_runtime)
+	if [ "$runtime" != "containerd" ]; then
+		die "$runtime is not supported for now"
+	fi
 
-	if [ "$runtime" == "k3s" ] || [ "$runtime" == "k3s-agent" ] || [ "$runtime" == "rke2-agent" ] || [ "$runtime" == "rke2-server" ]; then
-		containerd_conf_tmpl_file="${containerd_conf_file}.tmpl"
-		if [ ! -f "$containerd_conf_tmpl_file" ]; then
-			cp "$containerd_conf_file" "$containerd_conf_tmpl_file"
-		fi
-
-		containerd_conf_file="${containerd_conf_tmpl_file}"
-		containerd_conf_file_backup="${containerd_conf_file}.bak"
-	elif [ "$runtime" == "containerd" ]; then
-		# runtime == containerd
-		if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \
-			[ -x $(command -v containerd) ]; then
-			containerd config default > "$containerd_conf_file"
-		fi
+	if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \
+		[ -x $(command -v containerd) ]; then
+		containerd config default > "$containerd_conf_file"
 	fi
 
 	action=${1:-}
@@ -259,29 +264,25 @@ function main() {
 		die "invalid arguments"
 	fi
 
-	# only install / remove / update if we are dealing with containerd
-	if [[ "$runtime" =~ ^(containerd|k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then
-
-		case "$action" in
-		install)
-			install_artifacts
-			configure_cri_runtime "$runtime"
-			kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true
-			;;
-		cleanup)
-			cleanup_cri_runtime "$runtime"
-			kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup
-			remove_artifacts
-			;;
-		reset)
-			reset_runtime $runtime
-			;;
-		*)
-			echo invalid arguments
-			print_usage
-			;;
-		esac
-	fi
+	case "$action" in
+	install)
+		install_artifacts
+		configure_cri_runtime "$runtime"
+		kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true
+		;;
+	cleanup)
+		cleanup_cri_runtime "$runtime"
+		kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup
+		remove_artifacts
+		;;
+	reset)
+		reset_runtime $runtime
+		;;
+	*)
+		echo invalid arguments
+		print_usage
+		;;
+	esac
 
 	#It is assumed this script will be called as a daemonset. As a result, do
         # not return, otherwise the daemon will restart and rexecute the script

tools/packaging/kata-deploy/local-build/Makefile

@@ -71,10 +71,10 @@ merge-builds:
 install-tarball:
 	tar -xvf ./kata-static.tar.xz -C /
 
-image: kata-tarball
-	$(MK_DIR)kata-deploy-build-and-upload-image.sh $(CURDIR)/kata-static.tar.xz
+cc-payload: cc-tarball
+	$(MK_DIR)kata-deploy-build-and-upload-payload.sh $(CURDIR)/kata-static.tar.xz
 
-cc-tarball: | cc-parallel merge-builds
+cc-tarball: | cc merge-builds
 
 cc-parallel: $(MK_DIR)/dockerbuild/install_yq.sh
 	${MAKE} -f $(MK_PATH) cc -j$$(( $$(nproc) - 1  )) V=
@@ -87,6 +87,7 @@ cc: cc-cloud-hypervisor-tarball \
 	cc-virtiofsd-tarball \
 	cc-tdx-kernel-tarball \
 	cc-tdx-qemu-tarball \
+	cc-tdx-td-shim-tarball \
 	cc-tdx-tdvf-tarball
 
 cc-cloud-hypervisor-tarball:

tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh

@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 #
-# Copyright 2021 Fabiano Fidêncio
+# Copyright 2022 Intel
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 
-KATA_DEPLOY_DIR="`dirname $0`/../"
-KATA_DEPLOY_ARTIFACT="$1"
+KATA_DEPLOY_DIR="`dirname $0`/../../kata-deploy-cc"
+KATA_DEPLOY_ARTIFACT="${1:-"kata-static.tar.xz"}"
 
 echo "Copying $KATA_DEPLOY_ARTIFACT to $KATA_DEPLOY_DIR"
 cp $KATA_DEPLOY_ARTIFACT $KATA_DEPLOY_DIR
 
 pushd $KATA_DEPLOY_DIR
 
-IMAGE_TAG="quay.io/kata-containers/kata-deploy-cc:v0"
+IMAGE_TAG="quay.io/confidential-containers/runtime-payload:kata-containers-$(git rev-parse HEAD)"
 
 echo "Building the image"
 docker build --tag $IMAGE_TAG .

tools/packaging/static-build/qemu/Dockerfile

@@ -56,7 +56,11 @@ ARG QEMU_REPO
 # commit/tag/branch
 ARG QEMU_VERSION
 ARG PREFIX
+# BUILD_SUFFIX is used by the qemu-build-post.sh script to
+# properly rename non vanilla versions of the QEMU
 ARG BUILD_SUFFIX
+ARG HYPERVISOR_NAME
+ARG PKGVERSION
 ARG QEMU_DESTDIR
 ARG QEMU_TARBALL
 
@@ -78,8 +82,6 @@ RUN git clone --depth=1 "${QEMU_REPO}" qemu && \
     git fetch --depth=1 origin "${QEMU_VERSION}" && git checkout FETCH_HEAD && \
     scripts/git-submodule.sh update meson capstone && \
     /root/patch_qemu.sh "${QEMU_VERSION}" "/root/kata_qemu/patches" && \
-    [ -n "${BUILD_SUFFIX}" ] && HYPERVISOR_NAME="kata-qemu-${BUILD_SUFFIX}" || HYPERVISOR_NAME="kata-qemu" && \
-    [ -n "${BUILD_SUFFIX}" ] && PKGVERSION="kata-static-${BUILD_SUFFIX}" || PKGVERSION="kata-static" && \
     (PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s "${HYPERVISOR_NAME}" | xargs ./configure \
 	--with-pkgversion="${PKGVERSION}") && \
     make -j"$(nproc ${CI:+--ignore 1})" && \

tools/packaging/static-build/qemu/build-base-qemu.sh

@@ -34,9 +34,16 @@ prefix="${prefix:-"/opt/kata"}"
 
 CACHE_TIMEOUT=$(date +"%Y-%m-%d")
 
+[ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu"
+[ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static"
+
+container_image="qemu-static-${qemu_version,,}"
+
 sudo "${container_engine}" build \
 	--build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \
-	--build-arg BUILD_SUFFIX="${build_suffix}" \
+	--build-arg BUILD_SUFFIX=${build_suffix} \
+	--build-arg HYPERVISOR_NAME="${HYPERVISOR_NAME}" \
+	--build-arg PKGVERSION="${PKGVERSION}" \
 	--build-arg http_proxy="${http_proxy}" \
 	--build-arg https_proxy="${https_proxy}" \
 	--build-arg QEMU_DESTDIR="${qemu_destdir}" \
@@ -46,12 +53,14 @@ sudo "${container_engine}" build \
 	--build-arg PREFIX="${prefix}" \
 	"${packaging_dir}" \
 	-f "${script_dir}/Dockerfile" \
-	-t qemu-static
+	-t "${container_image}"
 
 sudo "${container_engine}" run \
 	--rm \
 	-i \
-	-v "${PWD}":/share qemu-static \
+	-v "${PWD}":/share "${container_image}" \
 	mv "${qemu_destdir}/${qemu_tar}" /share/
 
+sudo docker image rm "${container_image}"
+
 sudo chown ${USER}:$(id -gn ${USER}) "${PWD}/${qemu_tar}"