From bad859d2f8d51aa72e563c30bac638c7415a5329 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 2 Mar 2022 16:24:42 +1100 Subject: [PATCH 1/3] tools/packaging/kata-deploy/local-build: Add build to gitignore This directory consists entirely of files built during a make kata-tarball, so it should not be committed to the tree. A symbolic link to this directory might be created during 'make tarball', ignore it as well. Signed-off-by: David Gibson [greg: - rearranged the subject to make the subsystem checker happy - also ignore the symbolic link created by `kata-deploy-binaries-in-docker.sh`] Signed-off-by: Greg Kurz --- .gitignore | 1 + tools/packaging/kata-deploy/local-build/.gitignore | 1 + 2 files changed, 2 insertions(+) create mode 100644 tools/packaging/kata-deploy/local-build/.gitignore diff --git a/.gitignore b/.gitignore index 529bab04a..ce97c7e98 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ src/agent/src/version.rs src/agent/kata-agent.service src/agent/protocols/src/*.rs !src/agent/protocols/src/lib.rs +build diff --git a/tools/packaging/kata-deploy/local-build/.gitignore b/tools/packaging/kata-deploy/local-build/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/tools/packaging/kata-deploy/local-build/.gitignore @@ -0,0 +1 @@ +build/ From 1ed7da8fc75426e71cf6d68648ab6ab4576f0a54 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 2 Mar 2022 16:26:22 +1100 Subject: [PATCH 2/3] packaging: Eliminate TTY_OPT and NO_TTY variables in kata-deploy NO_TTY configured whether to add the -t option to docker run. It makes no sense for the caller to configure this, since whether you need it depends on the commands you're running. Since the point here is to run non-interactive build scripts, we don't need -t, or -i either. Signed-off-by: David Gibson Signed-off-by: Greg Kurz --- tools/packaging/kata-deploy/local-build/Makefile | 2 +- .../local-build/kata-deploy-binaries-in-docker.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 68e45d447..f04d9b658 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -16,7 +16,7 @@ endef kata-tarball: | all-parallel merge-builds all-parallel: - ${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1 )) NO_TTY="true" V= + ${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1 )) V= all: cloud-hypervisor-tarball \ firecracker-tarball \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 68fba1b5f..c1fc1538a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -16,10 +16,6 @@ kata_deploy_create="${script_dir}/kata-deploy-binaries.sh" uid=$(id -u ${USER}) gid=$(id -g ${USER}) -TTY_OPT="-i" -NO_TTY="${NO_TTY:-false}" -[ -t 1 ] && [ "${NO_TTY}" == "false" ] && TTY_OPT="-it" - if [ "${script_dir}" != "${PWD}" ]; then ln -sf "${script_dir}/build" "${PWD}/build" fi @@ -34,7 +30,7 @@ docker build -q -t build-kata-deploy \ --build-arg GID=${gid} \ "${script_dir}/dockerbuild/" -docker run ${TTY_OPT} \ +docker run \ -v /var/run/docker.sock:/var/run/docker.sock \ --user ${uid}:${gid} \ --env USER=${USER} -v "${kata_dir}:${kata_dir}" \ From 154c8b03d3763bfe100155105a7db7e889bf2a38 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Fri, 25 Mar 2022 10:30:52 +0100 Subject: [PATCH 3/3] tools/packaging/kata-deploy: Copy install_yq.sh in a dedicated script 'make kata-tarball' sometimes fails early with: cp: cannot create regular file '[...]/tools/packaging/kata-deploy/local-build/dockerbuild/install_yq.sh': File exists This happens because all assets are built in parallel using the same `kata-deploy-binaries-in-docker.sh` script, and thus all try to copy the `install_yq.sh` script to the same location with the `cp` command. This is a well known race condition that cannot be avoided without serialization of `cp` invocations. Move the copying of `install_yq.sh` to a separate script and ensure it is called *before* parallel builds. Make the presence of the copy a prerequisite for each sub-build so that they still can be triggered individually. Update the GH release workflow to also call this script before calling `kata-deploy-binaries-in-docker.sh`. Fixes #3756 Signed-off-by: Greg Kurz --- .github/workflows/release.yaml | 1 + tools/packaging/kata-deploy/local-build/Makefile | 7 +++++-- .../kata-deploy-binaries-in-docker.sh | 4 ---- .../local-build/kata-deploy-copy-yq-installer.sh | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100755 tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ca2db149c..c87d5b3a9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,6 +26,7 @@ jobs: - name: Build ${{ matrix.asset }} run: | + ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}" build_dir=$(readlink -f build) # store-artifact does not work with symlink diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index f04d9b658..ac3a77714 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -15,7 +15,10 @@ endef kata-tarball: | all-parallel merge-builds -all-parallel: +$(MK_DIR)/dockerbuild/install_yq.sh: + $(MK_DIR)/kata-deploy-copy-yq-installer.sh + +all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh ${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1 )) V= all: cloud-hypervisor-tarball \ @@ -26,7 +29,7 @@ all: cloud-hypervisor-tarball \ rootfs-initrd-tarball \ shim-v2-tarball -%-tarball-build: +%-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh $(call BUILD,$*) cloud-hypervisor-tarball: diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index c1fc1538a..4035ff9cb 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -20,10 +20,6 @@ if [ "${script_dir}" != "${PWD}" ]; then ln -sf "${script_dir}/build" "${PWD}/build" fi -install_yq_script_path="${script_dir}/../../../../ci/install_yq.sh" - -cp "${install_yq_script_path}" "${script_dir}/dockerbuild/install_yq.sh" - docker build -q -t build-kata-deploy \ --build-arg IMG_USER="${USER}" \ --build-arg UID=${uid} \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh new file mode 100755 index 000000000..1271fd882 --- /dev/null +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2021 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +script_dir=$(dirname "$(readlink -f "$0")") +install_yq_script_path="${script_dir}/../../../../ci/install_yq.sh" + +cp "${install_yq_script_path}" "${script_dir}/dockerbuild/install_yq.sh"