From b0e6a094be16b58fe3c04639955ede973a826e38 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Mon, 17 Apr 2023 23:06:17 +0000 Subject: [PATCH 01/10] packaging: Add sev kernel build capability Adding code that builds sev kernel. Fixes: #6572 Signed-off-by: Unmesh Deodhar --- .../kata-deploy/local-build/Makefile | 3 + .../local-build/kata-deploy-binaries.sh | 70 ++++++++++++++++--- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 82356f1fa..88de4399a 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -72,6 +72,9 @@ kernel-experimental-tarball: kernel-tdx-experimental-tarball: ${MAKE} $@-build +kernel-sev-tarball: + ${MAKE} $@-build + nydus-tarball: ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 902b067c9..7b0227442 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -82,6 +82,7 @@ options: kernel-dragonball-experimental kernel-experimental kernel-tdx-experimental + kernel-sev-tarball kernel-gpu kernel-gpu-snp kernel-gpu-tdx-experimental @@ -175,7 +176,45 @@ install_initrd() { && return 0 info "Create initrd" - "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" + "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}" +} + +#Install guest initrd for sev +install_initrd_sev() { + install_initrd "initrd-sev" "sev" +} + +#Install kernel component helper +install_cached_kernel_tarball_component() { + local kernel_name=${1} + + install_cached_tarball_component \ + "${kernel_name}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + || return 1 + + if [[ "${kernel_name}" != "kernel-sev" ]]; then + return 0 + fi + + # SEV specific code path + install_cached_tarball_component \ + "${kernel_name}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "kata-static-kernel-sev-modules.tar.xz" \ + "${workdir}/kata-static-kernel-sev-modules.tar.xz" \ + || return 1 + + mkdir -p "${module_dir}" + tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C "${module_dir}" && return 0 + + return 1 } #Install kernel asset @@ -185,16 +224,14 @@ install_kernel_helper() { local extra_cmd=${3} export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})" - local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + local module_dir="" - install_cached_tarball_component \ - "${kernel_name}" \ - "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ - "${kernel_version}-${kernel_kata_config_version}" \ - "$(get_kernel_image_name)" \ - "${final_tarball_name}" \ - "${final_tarball_path}" \ - && return 0 + if [[ "${kernel_name}" == "kernel-sev" ]]; then + job_name="kata-containers-main-${kernel_name}-sev-$(uname -m)" + fi + + install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0 info "build ${kernel_name}" info "Kernel version ${kernel_version}" @@ -264,6 +301,17 @@ install_kernel_tdx_experimental() { "-x tdx -u ${kernel_url}" } +#Install sev kernel asset +install_kernel_sev() { + info "build sev kernel" + local kernel_url="$(get_from_kata_deps assets.kernel.sev.url)" + + install_kernel_helper \ + "assets.kernel.sev.version" \ + "kernel-sev" \ + "-x sev -u ${kernel_url}" +} + install_qemu_helper() { local qemu_repo_yaml_path="${1}" local qemu_version_yaml_path="${2}" @@ -481,6 +529,8 @@ handle_build() { kernel-tdx-experimental) install_kernel_tdx_experimental ;; + kernel-sev) install_kernel_sev ;; + kernel-gpu) install_kernel_gpu ;; kernel-gpu-snp) install_kernel_gpu_snp;; From b87820ee8c38ba0ca442e981b025142b3d73f86d Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Mon, 17 Apr 2023 23:19:25 +0000 Subject: [PATCH 02/10] packaging: add support to build initrd for sev We need special initrd for SEV. The work on SEV initrd is based on Ubuntu. Thus, adding another entry in versions.yaml This binary will have '-sev' suffix to distinguish it from the generic binary. Fixes: #6572 Signed-Off-By: Unmesh Deodhar --- tools/packaging/guest-image/build_image.sh | 15 ++++++- .../kata-deploy/local-build/Makefile | 4 ++ .../local-build/kata-deploy-binaries.sh | 45 ++++--------------- versions.yaml | 3 ++ 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index 914b72a79..d602b85c0 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -22,6 +22,8 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" export GOPATH=${GOPATH:-${HOME}/go} arch_target="$(uname -m)" +final_initrd_name="kata-containers-initrd" +image_initrd_extension=".img" build_initrd() { info "Build initrd" @@ -37,7 +39,7 @@ build_initrd() { mv "kata-containers-initrd.img" "${install_dir}/${initrd_name}" ( cd "${install_dir}" - ln -sf "${initrd_name}" kata-containers-initrd.img + ln -sf "${initrd_name}" "${final_initrd_name}${image_initrd_extension}" ) } @@ -71,6 +73,7 @@ Options: --imagetype=${image_type} --prefix=${prefix} --destdir=${destdir} + --image_initrd_suffix=${image_initrd_suffix} EOF exit "${return_code}" @@ -80,6 +83,7 @@ main() { image_type=image destdir="$PWD" prefix="/opt/kata" + image_initrd_suffix="" builddir="${PWD}" while getopts "h-:" opt; do case "$opt" in @@ -99,6 +103,15 @@ main() { initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version") initrd_name="kata-${initrd_distro}-${initrd_os_version}.${image_type}" ;; + image_initrd_suffix=*) + image_initrd_suffix=${OPTARG#*=} + if [ "${image_initrd_suffix}" == "sev" ]; then + initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name") + initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version") + initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}" + final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" + fi + ;; prefix=*) prefix=${OPTARG#*=} ;; diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 88de4399a..5e20f9108 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -39,6 +39,7 @@ all: serial-targets \ serial-targets: ${MAKE} -f $(MK_PATH) -j 1 V= \ rootfs-image-tarball \ + rootfs-initrd-sev-tarball \ rootfs-initrd-tarball \ cloud-hypervisor-tarball @@ -87,6 +88,9 @@ qemu-tdx-experimental-tarball: rootfs-image-tarball: ${MAKE} $@-build +rootfs-initrd-sev-tarball: kernel-sev-tarball + ${MAKE} $@-build + rootfs-initrd-tarball: ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 7b0227442..f551fc8d9 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -91,6 +91,7 @@ options: qemu-tdx-experimental rootfs-image rootfs-initrd + rootfs-initrd-sev shim-v2 tdvf virtiofsd @@ -155,8 +156,10 @@ install_image() { #Install guest initrd install_initrd() { - local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-initrd-$(uname -m)/${cached_artifacts_path}" - local component="rootfs-initrd" + local initrd_type="${1:-""}" + local initrd_suffix="${2:-""}" + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-${initrd_type}" local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" @@ -169,7 +172,7 @@ install_initrd() { install_cached_tarball_component \ "${component}" \ "${jenkins}" \ - "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-initrd" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${initrd_type}" \ "" \ "${final_tarball_name}" \ "${final_tarball_path}" \ @@ -184,39 +187,6 @@ install_initrd_sev() { install_initrd "initrd-sev" "sev" } -#Install kernel component helper -install_cached_kernel_tarball_component() { - local kernel_name=${1} - - install_cached_tarball_component \ - "${kernel_name}" \ - "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ - "${kernel_version}-${kernel_kata_config_version}" \ - "$(get_kernel_image_name)" \ - "${final_tarball_name}" \ - "${final_tarball_path}" \ - || return 1 - - if [[ "${kernel_name}" != "kernel-sev" ]]; then - return 0 - fi - - # SEV specific code path - install_cached_tarball_component \ - "${kernel_name}" \ - "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ - "${kernel_version}-${kernel_kata_config_version}" \ - "$(get_kernel_image_name)" \ - "kata-static-kernel-sev-modules.tar.xz" \ - "${workdir}/kata-static-kernel-sev-modules.tar.xz" \ - || return 1 - - mkdir -p "${module_dir}" - tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C "${module_dir}" && return 0 - - return 1 -} - #Install kernel asset install_kernel_helper() { local kernel_version_yaml_path="${1}" @@ -504,6 +474,7 @@ handle_build() { install_firecracker install_image install_initrd + install_initrd_sev install_kernel install_kernel_dragonball_experimental install_kernel_tdx_experimental @@ -545,6 +516,8 @@ handle_build() { rootfs-initrd) install_initrd ;; + rootfs-initrd-sev) install_initrd_sev ;; + shim-v2) install_shimv2 ;; tdvf) install_tdvf ;; diff --git a/versions.yaml b/versions.yaml index c222499d9..d95946749 100644 --- a/versions.yaml +++ b/versions.yaml @@ -156,6 +156,9 @@ assets: x86_64: name: *default-initrd-name version: *default-initrd-version + sev: + name: *glibc-initrd-name + version: *glibc-initrd-version kernel: description: "Linux kernel optimised for virtual machines" From 12c5ef9020cdcf6dbb265b93e3fe1d714536e6ea Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Fri, 21 Apr 2023 02:34:23 +0000 Subject: [PATCH 03/10] packaging: add support to build OVMF for SEV SEV requires special OVMF to work with kernel hashes. Thus, adding changes that builds this custom OVMF for SEV. Fixes: #6572 Signed-Off-By: Unmesh Deodhar --- src/runtime/Makefile | 2 ++ tools/packaging/kata-deploy/local-build/Makefile | 4 ++++ .../kata-deploy/local-build/kata-deploy-binaries.sh | 9 +++++++++ tools/packaging/static-build/ovmf/build-ovmf.sh | 6 +++++- versions.yaml | 2 +- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index aa03ce852..f2389c73c 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -126,6 +126,8 @@ FIRMWAREVOLUMEPATH := FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd FIRMWARETDVFVOLUMEPATH := +FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd + # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 5e20f9108..91d76dab0 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -30,6 +30,7 @@ all: serial-targets \ kernel-gpu-snp-tarball \ kernel-gpu-tdx-experimental-tarball \ nydus-tarball \ + ovmf-sev-tarball \ qemu-tarball \ qemu-tdx-experimental-tarball \ shim-v2-tarball \ @@ -79,6 +80,9 @@ kernel-sev-tarball: nydus-tarball: ${MAKE} $@-build +ovmf-sev-tarball: + ${MAKE} $@-build + qemu-tarball: ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index f551fc8d9..25e05da55 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -87,6 +87,7 @@ options: kernel-gpu-snp kernel-gpu-tdx-experimental nydus + ovmf-sev qemu qemu-tdx-experimental rootfs-image @@ -453,6 +454,11 @@ install_tdvf() { install_ovmf "tdx" "edk2-tdx.tar.gz" } +# Install OVMF SEV +install_ovmf_sev() { + install_ovmf "sev" "edk2-sev.tar.gz" +} + get_kata_version() { local v v=$(cat "${version_file}") @@ -479,6 +485,7 @@ handle_build() { install_kernel_dragonball_experimental install_kernel_tdx_experimental install_nydus + install_ovmf_sev install_qemu install_qemu_tdx_experimental install_shimv2 @@ -502,6 +509,8 @@ handle_build() { kernel-sev) install_kernel_sev ;; + ovmf-sev) install_ovmf_sev ;; + kernel-gpu) install_kernel_gpu ;; kernel-gpu-snp) install_kernel_gpu_snp;; diff --git a/tools/packaging/static-build/ovmf/build-ovmf.sh b/tools/packaging/static-build/ovmf/build-ovmf.sh index ebdf0669f..c0e7d26e2 100755 --- a/tools/packaging/static-build/ovmf/build-ovmf.sh +++ b/tools/packaging/static-build/ovmf/build-ovmf.sh @@ -82,7 +82,11 @@ if [ "${ovmf_build}" == "tdx" ]; then fi mkdir -p "${install_dir}" -install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF.fd "${install_dir}" +if [ "${ovmf_build}" == "sev" ]; then + install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF.fd "${install_dir}/AMDSEV.fd" +else + install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF.fd "${install_dir}" +fi if [ "${ovmf_build}" == "tdx" ]; then install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF_CODE.fd ${install_dir} install $build_root/$ovmf_dir/"${build_path_fv}"/OVMF_VARS.fd ${install_dir} diff --git a/versions.yaml b/versions.yaml index d95946749..4f6e81e4b 100644 --- a/versions.yaml +++ b/versions.yaml @@ -278,7 +278,7 @@ externals: package_output_dir: "OvmfX64" sev: description: "AmdSev build needed for SEV measured direct boot." - version: "edk2-stable202202" + version: "edk2-stable202302" package: "OvmfPkg/AmdSev/AmdSevX64.dsc" package_output_dir: "AmdSev" tdx: From 76ae7a3abec5a7fb4864141e735a0a29dfb4d920 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Mon, 24 Apr 2023 17:04:39 +0000 Subject: [PATCH 04/10] packaging: adding caching capability for kernel The SEV initrd build requires kernel modules. So, for SEV case, we need to cache kernel modules tarball in addition to kernel tarball. Fixes: #6572 Signed-Off-By: Unmesh Deodhar --- .../local-build/kata-deploy-binaries.sh | 37 ++++++++++++++++++- .../static-build/cache_components_main.sh | 17 ++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 25e05da55..81784379a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -188,6 +188,39 @@ install_initrd_sev() { install_initrd "initrd-sev" "sev" } +#Install kernel component helper +install_cached_kernel_tarball_component() { + local kernel_name=${1} + + install_cached_tarball_component \ + "${kernel_name}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + || return 1 + + if [[ "${kernel_name}" != "kernel-sev" ]]; then + return 0 + fi + + # SEV specific code path + install_cached_tarball_component \ + "${kernel_name}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "kata-static-kernel-sev-modules.tar.xz" \ + "${workdir}/kata-static-kernel-sev-modules.tar.xz" \ + || return 1 + + mkdir -p "${module_dir}" + tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C "${module_dir}" && return 0 + + return 1 +} + #Install kernel asset install_kernel_helper() { local kernel_version_yaml_path="${1}" @@ -199,7 +232,9 @@ install_kernel_helper() { local module_dir="" if [[ "${kernel_name}" == "kernel-sev" ]]; then - job_name="kata-containers-main-${kernel_name}-sev-$(uname -m)" + kernel_version="$(get_from_kata_deps assets.kernel.sev.version)" + default_patches_dir="${repo_root_dir}/tools/packaging/kernel/patches" + module_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kernel-sev/builddir/kata-linux-${kernel_version#v}-${kernel_kata_config_version}/lib/modules/${kernel_version#v}" fi install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0 diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index 0e8a0120f..fc0019c7f 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -33,8 +33,21 @@ cache_kernel_artifacts() { local kernel_tarball_name="kata-static-${KERNEL_FLAVOUR}.tar.xz" local current_kernel_image="$(get_kernel_image_name)" local current_kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")-${current_kernel_kata_config_version}" - create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}" + local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")" + local kernel_modules_tarball_path="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel-sev-modules.tar.xz" + + if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then + current_kernel_version="$(get_from_kata_deps "assets.kernel.sev.version")" + fi + + create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}" + if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then + module_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kernel-sev/builddir/kata-linux-${current_kernel_version#v}-${current_kernel_kata_config_version}/lib/modules/${current_kernel_version#v}" + if [ ! -f "${kernel_modules_tarball_path}" ]; then + tar cvfJ "${kernel_modules_tarball_path}" "${module_dir}/kernel/drivers/virt/coco/efi_secret/" + fi + create_cache_asset "kata-static-kernel-sev-modules.tar.xz" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}" + fi } cache_nydus_artifacts() { From af18806a8d45932f2a99e3a87f9ccdb67469e8e3 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Mon, 24 Apr 2023 21:13:11 +0000 Subject: [PATCH 05/10] static-build: Add caching support to sev ovmf SEV requires special OVMF. Now that we have ability to build this custom OVMF, let's optimize it by caching so that we don't have to build it for every run. Fixes: sev: #6572 Signed-Off-By: Unmesh Deodhar --- tools/packaging/static-build/cache_components_main.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index fc0019c7f..1c46c1e01 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -59,6 +59,7 @@ cache_nydus_artifacts() { cache_ovmf_artifacts() { local current_ovmf_version="$(get_from_kata_deps "externals.ovmf.${OVMF_FLAVOUR}.version")" [ "${OVMF_FLAVOUR}" == "tdx" ] && OVMF_FLAVOUR="tdvf" + [ "${OVMF_FLAVOUR}" == "sev" ] && OVMF_FLAVOUR="ovmf-sev" local ovmf_tarball_name="kata-static-${OVMF_FLAVOUR}.tar.xz" local current_ovmf_image="$(get_ovmf_image_name)" create_cache_asset "${ovmf_tarball_name}" "${current_ovmf_version}" "${current_ovmf_image}" From 813e4c576fbd1f174b7353ecb93abeba3447b9b7 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Thu, 27 Apr 2023 20:53:55 +0000 Subject: [PATCH 06/10] runtimeClasses: add sev runtime class Adding kata-qemu-sev runtime class. Fixes: #6572 Signed-off-by: Unmesh Deodhar --- .../runtimeclasses/kata-runtimeClasses.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml index f65ec6b0a..00197ed3b 100644 --- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml @@ -14,6 +14,19 @@ scheduling: --- kind: RuntimeClass apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-sev +handler: kata-qemu-sev +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 metadata: name: kata-qemu-tdx handler: kata-qemu-tdx From fb9c1fc36e56602702d22b7cfb06fe1a9639e5f7 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Fri, 28 Apr 2023 14:20:55 -0500 Subject: [PATCH 07/10] runtime: Add qemu-sev config Adding config file that can be used with qemu-sev runtime class. Since SEV has limited hotplug support, increase the pod overhead to account for fixed resource usage. Fixes: #6572 Signed-off-by: Unmesh Deodhar --- src/runtime/Makefile | 33 + .../config/configuration-qemu-sev.toml.in | 650 ++++++++++++++++++ 2 files changed, 683 insertions(+) create mode 100644 src/runtime/config/configuration-qemu-sev.toml.in diff --git a/src/runtime/Makefile b/src/runtime/Makefile index f2389c73c..ee3896c77 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -57,6 +57,7 @@ BIN_PREFIX = $(PROJECT_TYPE) PROJECT_DIR = $(PROJECT_TAG) IMAGENAME = $(PROJECT_TAG).img INITRDNAME = $(PROJECT_TAG)-initrd.img +INITRDSEVNAME = $(PROJECT_TAG)-initrd-sev.img TARGET = $(BIN_PREFIX)-runtime RUNTIME_OUTPUT = $(CURDIR)/$(TARGET) @@ -95,6 +96,7 @@ GENERATED_VARS = \ VERSION \ CONFIG_ACRN_IN \ CONFIG_QEMU_IN \ + CONFIG_QEMU_SEV_IN \ CONFIG_QEMU_TDX_IN \ CONFIG_QEMU_GPU_IN \ CONFIG_CLH_IN \ @@ -114,6 +116,7 @@ KERNELDIR := $(PKGDATADIR) IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME) INITRDPATH := $(PKGDATADIR)/$(INITRDNAME) +INITRDSEVPATH := $(PKGDATADIR)/$(INITRDSEVNAME) ROOTFSTYPE_EXT4 := \"ext4\" ROOTFSTYPE_XFS := \"xfs\" @@ -206,6 +209,7 @@ DEFDISABLEBLOCK := false DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p +DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd ifeq ($(ARCH),ppc64le) DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/qemu/virtiofsd @@ -238,6 +242,9 @@ DEFSTATICRESOURCEMGMT ?= false DEFBINDMOUNTS := [] +# Image Service Offload +DEFSERVICEOFFLOAD ?= false + SED = sed CLI_DIR = cmd @@ -288,6 +295,18 @@ ifneq (,$(QEMUCMD)) CONFIGS += $(CONFIG_QEMU_TDX) + CONFIG_FILE_QEMU_SEV = configuration-qemu-sev.toml + CONFIG_QEMU_SEV = config/$(CONFIG_FILE_QEMU_SEV) + CONFIG_QEMU_SEV_IN = $(CONFIG_QEMU_SEV).in + + CONFIG_PATH_QEMU_SEV = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SEV)) + CONFIG_PATHS += $(CONFIG_PATH_QEMU_SEV) + + SYSCONFIG_QEMU_SEV = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SEV)) + SYSCONFIG_PATHS_SEV += $(SYSCONFIG_QEMU_SEV) + + CONFIGS += $(CONFIG_QEMU_SEV) + CONFIG_FILE_QEMU_GPU = configuration-qemu-gpu.toml CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU) CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in @@ -303,6 +322,10 @@ ifneq (,$(QEMUCMD)) KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) KERNELPATH = $(KERNELDIR)/$(KERNELNAME) + KERNELSEVTYPE = compressed + KERNELSEVNAME = $(call MAKE_KERNEL_SEV_NAME,$(KERNELSEVTYPE)) + KERNELSEVPATH = $(KERNELDIR)/$(KERNELSEVNAME) + KERNELTDXTYPE = compressed KERNELTDXNAME = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE)) KERNELTDXPATH = $(KERNELDIR)/$(KERNELTDXNAME) @@ -451,7 +474,9 @@ USER_VARS += SYSCONFIG USER_VARS += IMAGENAME USER_VARS += IMAGEPATH USER_VARS += INITRDNAME +USER_VARS += INITRDSEVNAME USER_VARS += INITRDPATH +USER_VARS += INITRDSEVPATH USER_VARS += DEFROOTFSTYPE USER_VARS += MACHINETYPE USER_VARS += KERNELDIR @@ -461,11 +486,13 @@ USER_VARS += KERNELTYPE_ACRN USER_VARS += KERNELTYPE_CLH USER_VARS += KERNELPATH_ACRN USER_VARS += KERNELPATH +USER_VARS += KERNELSEVPATH USER_VARS += KERNELTDXPATH USER_VARS += KERNELPATH_CLH USER_VARS += KERNELPATH_FC USER_VARS += KERNELVIRTIOFSPATH USER_VARS += FIRMWAREPATH +USER_VARS += FIRMWARESEVPATH USER_VARS += FIRMWARETDVFPATH USER_VARS += FIRMWAREVOLUMEPATH USER_VARS += FIRMWARETDVFVOLUMEPATH @@ -525,6 +552,7 @@ USER_VARS += DEFBLOCKDEVICEAIO_QEMU USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS USER_VARS += DEFSHAREDFS_QEMU_TDX_VIRTIOFS +USER_VARS += DEFSHAREDFS_QEMU_SEV_VIRTIOFS USER_VARS += DEFVIRTIOFSDAEMON USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS USER_VARS += DEFVIRTIOFSCACHESIZE @@ -546,6 +574,7 @@ USER_VARS += DEFSANDBOXCGROUPONLY USER_VARS += DEFSTATICRESOURCEMGMT USER_VARS += DEFSTATICRESOURCEMGMT_FC USER_VARS += DEFBINDMOUNTS +USER_VARS += DEFSERVICEOFFLOAD USER_VARS += DEFVFIOMODE USER_VARS += BUILDFLAGS @@ -630,6 +659,10 @@ define MAKE_KERNEL_VIRTIOFS_NAME $(if $(findstring uncompressed,$1),vmlinux-virtiofs.container,vmlinuz-virtiofs.container) endef +define MAKE_KERNEL_SEV_NAME +$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container) +endef + define MAKE_KERNEL_TDX_NAME $(if $(findstring uncompressed,$1),vmlinux-tdx.container,vmlinuz-tdx.container) endef diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in new file mode 100644 index 000000000..8dba6813e --- /dev/null +++ b/src/runtime/config/configuration-qemu-sev.toml.in @@ -0,0 +1,650 @@ +# Copyright 2022 Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "@CONFIG_QEMU_SEV_IN@" +# XXX: Project: +# XXX: Name: @PROJECT_NAME@ +# XXX: Type: @PROJECT_TYPE@ + +[hypervisor.qemu] +path = "@QEMUPATH@" +kernel = "@KERNELSEVPATH@" +initrd = "@INITRDSEVPATH@" +machine_type = "@MACHINETYPE@" + +# Enable confidential guest support. +# Toggling that setting may trigger different hardware features, ranging +# from memory encryption to both memory and CPU-state encryption and integrity. +# The Kata Containers runtime dynamically detects the available feature set and +# aims at enabling the largest possible one, returning an error if none is +# available, or none is supported by the hypervisor. +# +# Known limitations: +# * Does not work by design: +# - CPU Hotplug +# - Memory Hotplug +# - NVDIMM devices +# +# Default false +confidential_guest = true + +# Enable running QEMU VMM as a non-root user. +# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as +# a non-root random user. See documentation for the limitations of this mode. +# rootless = true + +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + +# List of valid annotations values for the hypervisor +# Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@ + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "@KERNELPARAMS@" + +# Path to the firmware. +# If you want that qemu uses the default firmware leave this option empty +firmware = "@FIRMWARESEVPATH@" + +# Path to the firmware volume. +# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables +# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables +# can be customized per each user while UEFI code is kept same. +firmware_volume = "@FIRMWAREVOLUMEPATH@" + +# Machine accelerators +# comma-separated list of machine accelerators to pass to the hypervisor. +# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` +machine_accelerators="@MACHINEACCELERATORS@" + +# Qemu seccomp sandbox feature +# comma-separated list of seccomp sandbox features to control the syscall access. +# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"` +# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox +# Another note: enabling this feature may reduce performance, you may enable +# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html +#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@" + +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="@CPUFEATURES@" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to @DEFVCPUS@ +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. +default_maxvcpus = @DEFMAXVCPUS@ + +# Bridges can be used to hot plug devices. +# Limitations: +# * Currently only pci bridges are supported +# * Until 30 devices per bridge can be hot plugged. +# * Until 5 PCI bridges can be cold plugged per VM. +# This limitation could be a bug in qemu or in the kernel +# Default number of bridges per SB/VM: +# unspecified or 0 --> will be set to @DEFBRIDGES@ +# > 1 <= 5 --> will be set to the specified number +# > 5 --> will be set to 5 +default_bridges = @DEFBRIDGES@ + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set @DEFMEMSZ@ MiB. +default_memory = @DEFMEMSZ@ +# +# Default memory slots per SB/VM. +# If unspecified then it will be set @DEFMEMSLOTS@. +# This is will determine the times that memory will be hotadded to sandbox/VM. +#memory_slots = @DEFMEMSLOTS@ + +# Default maximum memory in MiB per SB / VM +# unspecified or == 0 --> will be set to the actual amount of physical RAM +# > 0 <= amount of physical RAM --> will be set to the specified number +# > amount of physical RAM --> will be set to the actual amount of physical RAM +default_maxmemory = @DEFMAXMEMSZ@ + +# The size in MiB will be plused to max memory of hypervisor. +# It is the memory address space for the NVDIMM devie. +# If set block storage driver (block_device_driver) to "nvdimm", +# should set memory_offset to the size of block device. +# Default 0 +#memory_offset = 0 + +# Specifies virtio-mem will be enabled or not. +# Please note that this option should be used with the command +# "echo 1 > /proc/sys/vm/overcommit_memory". +# Default false +#enable_virtio_mem = true + +# Disable block device from being used for a container's rootfs. +# In case of a storage driver like devicemapper where a container's +# root file system is backed by a block device, the block device is passed +# directly to the hypervisor for performance reasons. +# This flag prevents the block device from being passed to the hypervisor, +# virtio-fs is used instead to pass the rootfs. +disable_block_device_use = @DEFDISABLEBLOCK@ + +# Shared file system type: +# - virtio-fs (default) +# - virtio-9p +# - virtio-fs-nydus +shared_fs = "@DEFSHAREDFS_QEMU_SEV_VIRTIOFS@" + +# Path to vhost-user-fs daemon. +virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" + +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@ +valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ + +# Default size of DAX cache in MiB +virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ + +# Extra args for virtiofsd daemon +# +# Format example: +# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# Examples: +# Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"] +# +# see `virtiofsd -h` for possible options. +virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ + +# Cache mode: +# +# - none +# Metadata, data, and pathname lookup are not cached in guest. They are +# always fetched from host and any changes are immediately pushed to host. +# +# - auto +# Metadata and pathname lookup cache expires after a configured amount of +# time (default is 1 second). Data is cached while the file is open (close +# to open consistency). +# +# - always +# Metadata, data, and pathname lookup are cached in guest and never expire. +virtio_fs_cache = "@DEFVIRTIOFSCACHE@" + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. This is virtio-scsi, virtio-blk +# or nvdimm. +block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@" + +# Specifies cache-related options will be set to block devices or not. +# Default false +#block_device_cache_set = true + +# Specifies cache-related options for block devices. +# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. +# Default false +#block_device_cache_direct = true + +# Specifies cache-related options for block devices. +# Denotes whether flush requests for the device are ignored. +# Default false +#block_device_cache_noflush = true + +# Enable iothreads (data-plane) to be used. This causes IO to be +# handled in a separate IO thread. This is currently only implemented +# for SCSI. +# +enable_iothreads = @DEFENABLEIOTHREADS@ + +# Enable pre allocation of VM RAM, default false +# Enabling this will result in lower container density +# as all of the memory will be allocated and locked +# This is useful when you want to reserve all the memory +# upfront or in the cases where you want memory latencies +# to be very predictable +# Default false +#enable_mem_prealloc = true + +# Enable huge pages for VM RAM, default false +# Enabling this will result in the VM memory +# being allocated using huge pages. +# This is useful when you want to use vhost-user network +# stacks within the container. This will automatically +# result in memory pre allocation +#enable_hugepages = true + +# Enable vhost-user storage device, default false +# Enabling this will result in some Linux reserved block type +# major range 240-254 being chosen to represent vhost-user devices. +enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@ + +# The base directory specifically used for vhost-user devices. +# Its sub-path "block" is used for block devices; "block/sockets" is +# where we expect vhost-user sockets to live; "block/devices" is where +# simulated block device nodes for vhost-user devices to live. +vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" + +# Enable vIOMMU, default false +# Enabling this will result in the VM having a vIOMMU device +# This will also add the following options to the kernel's +# command line: intel_iommu=on,iommu=pt +#enable_iommu = true + +# Enable IOMMU_PLATFORM, default false +# Enabling this will result in the VM device having iommu_platform=on set +#enable_iommu_platform = true + +# List of valid annotations values for the vhost user store path +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ +valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@ + +# Enable file based guest memory support. The default is an empty string which +# will disable this feature. In the case of virtio-fs, this is enabled +# automatically and '/dev/shm' is used as the backing folder. +# This option will be ignored if VM templating is enabled. +#file_mem_backend = "@DEFFILEMEMBACKEND@" + +# List of valid annotations values for the file_mem_backend annotation +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@ +valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ + +# -pflash can add image file to VM. The arguments of it should be in format +# of ["/path/to/flash0.img", "/path/to/flash1.img"] +pflashes = [] + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. +# +# Default false +#enable_debug = true + +# Disable the customizations done in the runtime when it detects +# that it is running on top a VMM. This will result in the runtime +# behaving as it would when running on bare metal. +# +#disable_nesting_checks = true + +# This is the msize used for 9p shares. It is the number of bytes +# used for 9p packet payload. +#msize_9p = @DEFMSIZE9P@ + +# If false and nvdimm is supported, use nvdimm device to plug guest image. +# Otherwise virtio-block device is used. +# +# nvdimm is not supported when `confidential_guest = true`. +# +# Default is false +#disable_image_nvdimm = true + +# VFIO devices are hotplugged on a bridge by default. +# Enable hotplugging on root bus. This may be required for devices with +# a large PCI bar, as this is a current limitation with hotplugging on +# a bridge. +# Default false +#hotplug_vfio_on_root_bus = true + +# Before hot plugging a PCIe device, you need to add a pcie_root_port device. +# Use this parameter when using some large PCI bar devices, such as Nvidia GPU +# The value means the number of pcie_root_port +# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35" +# Default 0 +#pcie_root_port = 2 + +# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off +# security (vhost-net runs ring0) for network I/O performance. +#disable_vhost_net = true + +# +# Default entropy source. +# The path to a host source of entropy (including a real hardware RNG) +# /dev/urandom and /dev/random are two main options. +# Be aware that /dev/random is a blocking source of entropy. If the host +# runs out of entropy, the VMs boot time will increase leading to get startup +# timeouts. +# The source of entropy /dev/urandom is non-blocking and provides a +# generally acceptable source of entropy. It should work well for pretty much +# all practical purposes. +#entropy_source= "@DEFENTROPYSOURCE@" + +# List of valid annotations values for entropy_source +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVALIDENTROPYSOURCES@ +valid_entropy_sources = @DEFVALIDENTROPYSOURCES@ + +# Path to OCI hook binaries in the *guest rootfs*. +# This does not affect host-side hooks which must instead be added to +# the OCI spec passed to the runtime. +# +# You can create a rootfs with hooks by customizing the osbuilder scripts: +# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder +# +# Hooks must be stored in a subdirectory of guest_hook_path according to their +# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". +# The agent will scan these directories for executable files and add them, in +# lexicographical order, to the lifecycle of the guest container. +# Hooks are executed in the runtime namespace of the guest. See the official documentation: +# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks +# Warnings will be logged if any error is encountered while scanning for hooks, +# but it will not abort container execution. +#guest_hook_path = "/usr/share/oci/hooks" +# +# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. +# Default 0-sized value means unlimited rate. +#rx_rate_limiter_max_rate = 0 +# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block) +# to discipline traffic. +# Default 0-sized value means unlimited rate. +#tx_rate_limiter_max_rate = 0 + +# Set where to save the guest memory dump file. +# If set, when GUEST_PANICKED event occurred, +# guest memeory will be dumped to host filesystem under guest_memory_dump_path, +# This directory will be created automatically if it does not exist. +# +# The dumped file(also called vmcore) can be processed with crash or gdb. +# +# WARNING: +# Dump guest’s memory can take very long depending on the amount of guest memory +# and use much disk space. +#guest_memory_dump_path="/var/crash/kata" + +# If enable paging. +# Basically, if you want to use "gdb" rather than "crash", +# or need the guest-virtual addresses in the ELF vmcore, +# then you should enable paging. +# +# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details +#guest_memory_dump_paging=false + +# Enable swap in the guest. Default false. +# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device +# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness") +# is bigger than 0. +# The size of the swap device should be +# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes. +# If swap_in_bytes is not set, the size should be memory_limit_in_bytes. +# If swap_in_bytes and memory_limit_in_bytes is not set, the size should +# be default_memory. +#enable_guest_swap = true + +# use legacy serial for guest console if available and implemented for architecture. Default false +#use_legacy_serial = true + +# disable applying SELinux on the VMM process (default false) +disable_selinux=@DEFDISABLESELINUX@ + +# disable applying SELinux on the container process +# If set to false, the type `container_t` is applied to the container process by default. +# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built +# with `SELINUX=yes`. +# (default: true) +disable_guest_selinux=@DEFDISABLEGUESTSELINUX@ + +[factory] +# VM templating support. Once enabled, new VMs are created from template +# using vm cloning. They will share the same initial kernel, initramfs and +# agent memory by mapping it readonly. It helps speeding up new container +# creation and saves a lot of memory if there are many kata containers running +# on the same host. +# +# When disabled, new VMs are created from scratch. +# +# Note: Requires "initrd=" to be set ("image=" is not supported). +# +# Default false +#enable_template = true + +# Specifies the path of template. +# +# Default "/run/vc/vm/template" +#template_path = "/run/vc/vm/template" + +# The number of caches of VMCache: +# unspecified or == 0 --> VMCache is disabled +# > 0 --> will be set to the specified number +# +# VMCache is a function that creates VMs as caches before using it. +# It helps speed up new container creation. +# The function consists of a server and some clients communicating +# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. +# The VMCache server will create some VMs and cache them by factory cache. +# It will convert the VM to gRPC format and transport it when gets +# requestion from clients. +# Factory grpccache is the VMCache client. It will request gRPC format +# VM and convert it back to a VM. If VMCache function is enabled, +# kata-runtime will request VM from factory grpccache when it creates +# a new sandbox. +# +# Default 0 +#vm_cache_number = 0 + +# Specify the address of the Unix socket that is used by VMCache. +# +# Default /var/run/kata-containers/cache.sock +#vm_cache_endpoint = "/var/run/kata-containers/cache.sock" + +[agent.@PROJECT_TYPE@] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +#enable_debug = true + +# Enable agent tracing. +# +# If enabled, the agent will generate OpenTelemetry trace spans. +# +# Notes: +# +# - If the runtime also has tracing enabled, the agent spans will be +# associated with the appropriate runtime parent span. +# - If enabled, the runtime will wait for the container to shutdown, +# increasing the container shutdown time slightly. +# +# (default: disabled) +#enable_tracing = true + +# Comma separated list of kernel modules and their parameters. +# These modules will be loaded in the guest kernel using modprobe(8). +# The following example can be used to load two kernel modules with parameters +# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] +# The first word is considered as the module name and the rest as its parameters. +# Container will not be started when: +# * A kernel module is specified and the modprobe command is not installed in the guest +# or it fails loading the module. +# * The module is not available in the guest or it doesn't met the guest kernel +# requirements, like architecture and version. +# +kernel_modules=[] + +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true + +# Agent connection dialing timeout value in seconds +# (default: 90) +dial_timeout = 90 + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +#enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="@DEFNETWORKMODEL_QEMU@" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + +# Apply a custom SELinux security policy to the container process inside the VM. +# This is used when you want to apply a type other than the default `container_t`, +# so general users should not uncomment and apply it. +# (format: "user:role:type") +# Note: You cannot specify MCS policy with the label because the sensitivity levels and +# categories are determined automatically by high-level container runtimes such as containerd. +#guest_selinux_label="@DEFGUESTSELINUXLABEL@" + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# Set the full url to the Jaeger HTTP Thrift collector. +# The default if not set will be "http://localhost:14268/api/traces" +#jaeger_endpoint = "" + +# Sets the username to be used if basic auth is required for Jaeger. +#jaeger_user = "" + +# Sets the password to be used if basic auth is required for Jaeger. +#jaeger_password = "" + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# (default: false) +#disable_new_netns = true + +# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. +# The container cgroups in the host are not created, just one single cgroup per sandbox. +# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. +# The sandbox cgroup is constrained if there is no container type annotation. +# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType +sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ + +# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In +# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful +# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug. +# Compatibility for determining appropriate sandbox (VM) size: +# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O +# does not yet support sandbox sizing annotations. +# - When running single containers using a tool like ctr, container sizing information will be available. +static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT@ + +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=@DEFBINDMOUNTS@ + +# VFIO Mode +# Determines how VFIO devices should be be presented to the container. +# Options: +# +# - vfio +# Matches behaviour of OCI runtimes (e.g. runc) as much as +# possible. VFIO devices will appear in the container as VFIO +# character devices under /dev/vfio. The exact names may differ +# from the host (they need to match the VM's IOMMU group numbers +# rather than the host's) +# +# - guest-kernel +# This is a Kata-specific behaviour that's useful in certain cases. +# The VFIO device is managed by whatever driver in the VM kernel +# claims it. This means it will appear as one or more device nodes +# or network interfaces depending on the nature of the device. +# Using this mode requires specially built workloads that know how +# to locate the relevant device interfaces within the VM. +# +vfio_mode="@DEFVFIOMODE@" + +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# they may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# (default: []) +experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# enable_pprof = true + +# WARNING: All the options in the following section have not been implemented yet. +# This section was added as a placeholder. DO NOT USE IT! +[image] +# Container image service. +# +# Offload the CRI image management service to the Kata agent. +# (default: false) +service_offload = @DEFSERVICEOFFLOAD@ + +# Container image decryption keys provisioning. +# Applies only if service_offload is true. +# Keys can be provisioned locally (e.g. through a special command or +# a local file) or remotely (usually after the guest is remotely attested). +# The provision setting is a complete URL that lets the Kata agent decide +# which method to use in order to fetch the keys. +# +# Keys can be stored in a local file, in a measured and attested initrd: +#provision=data:///local/key/file +# +# Keys could be fetched through a special command or binary from the +# initrd (guest) image, e.g. a firmware call: +#provision=file:///path/to/bin/fetcher/in/guest +# +# Keys can be remotely provisioned. The Kata agent fetches them from e.g. +# a HTTPS URL: +#provision=https://my-key-broker.foo/tenant/ \ No newline at end of file From 4770d3064a008c4760f21b159f3719f4554c4235 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Thu, 4 May 2023 17:08:21 +0000 Subject: [PATCH 08/10] gha: Build and ship SEV kernel. SEV requires custom kernel arguments when building. Thus, adding it to the test and release process. Fixes: #6572 Signed-off-by: Unmesh Deodhar --- .github/workflows/build-kata-static-tarball-amd64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 5942a5d79..5ea73c246 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -19,6 +19,7 @@ jobs: - cloud-hypervisor - firecracker - kernel + - kernel-sev - kernel-dragonball-experimental - kernel-tdx-experimental - kernel-gpu From 45fa36692688d8b419a5136001c8d9284d79b5b4 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Thu, 4 May 2023 17:30:12 +0000 Subject: [PATCH 09/10] gha: Build and ship SEV OVMF SEV requires special OVMF to work. Thus, building that for test and release. Fixes: #6572 Signed-Off-By: Unmesh Deodhar --- .github/workflows/build-kata-static-tarball-amd64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 5ea73c246..e800f9260 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -26,6 +26,7 @@ jobs: - kernel-gpu-snp - kernel-gpu-tdx-experimental - nydus + - ovmf-sev - qemu - qemu-tdx-experimental - rootfs-image From 772d4db2629e1e100b040a5c720c5f92b13a28d7 Mon Sep 17 00:00:00 2001 From: Unmesh Deodhar Date: Thu, 4 May 2023 17:34:13 +0000 Subject: [PATCH 10/10] gha: Build and ship SEV initrd We have code that builds initrd for SEV. thus, adding that to the test and release process. Fixes: #6572 Signed-off-by: Unmesh Deodhar --- .github/workflows/build-kata-static-tarball-amd64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index e800f9260..b49f4ca5c 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -31,6 +31,7 @@ jobs: - qemu-tdx-experimental - rootfs-image - rootfs-initrd + - rootfs-initrd-sev - shim-v2 - tdvf - virtiofsd