From 139ed64bf388dc52345ef18706d043ee96ebeef1 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Fri, 22 Jun 2018 17:12:55 +0000 Subject: [PATCH] kernel: Add script to build kernel Today we have instructions to build the kernel but there are a lot of manual steps to get one kernel. This tries to automate the process to setup a kernel for kata. Signed-off-by: Jose Carlos Venegas Munoz --- .ci/setup.sh | 4 + .ci/test.sh | 3 +- Makefile | 3 + kernel/build-kernel.sh | 308 ++++++++++++++++++++++++++++++++++++ kernel/build-kernel_test.sh | 77 +++++++++ 5 files changed, 394 insertions(+), 1 deletion(-) create mode 100755 kernel/build-kernel.sh create mode 100755 kernel/build-kernel_test.sh diff --git a/.ci/setup.sh b/.ci/setup.sh index 89e312c33..e27ca768f 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -14,5 +14,9 @@ source /etc/os-release echo "Setup script for packaging" if [ "$ID" == ubuntu ];then + echo "Install snap dependencies" sudo apt-get install -y snapd snapcraft + + echo "Install kernel dependencies" + sudo -E apt install -y libelf-dev bc gcc fi diff --git a/.ci/test.sh b/.ci/test.sh index e035dd107..bce06ebc0 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -27,10 +27,11 @@ make_target() { return fi popd >> /dev/null - echo "Changes found in $dir" + echo "Changes found in ${dir}" make -f "${toplevel_mk}" "${target}" } make_target test-release-tools "release/" make_target test-packaging-tools "obs-packaging/" make_target test-static-build "static-build/" +make_target test-build-kernel "kernel/" diff --git a/Makefile b/Makefile index 14308d14e..2a43c1dd8 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ test-static-build: test-packaging-tools: @$(MK_DIR)/obs-packaging/build_from_docker.sh +test-build-kernel: + @$(MK_DIR)/kernel/build-kernel_test.sh + $(YQ): @bash -c "source .ci/lib.sh; install_yq" diff --git a/kernel/build-kernel.sh b/kernel/build-kernel.sh new file mode 100755 index 000000000..48de6ba82 --- /dev/null +++ b/kernel/build-kernel.sh @@ -0,0 +1,308 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +description=" +Description: This script is the *ONLY* to build a kernel for development. +" + +set -o errexit +set -o nounset +set -o pipefail + +readonly script_name="$(basename "${BASH_SOURCE[0]}")" +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +#project_name +readonly project_name="kata-containers" +[ -n "${GOPATH:-}" ] || GOPATH="${HOME}/go" +# Fetch the first element from GOPATH as working directory +# as go get only works against the first item in the GOPATH +GOPATH="${GOPATH%%:*}" +# Kernel version to be used +kernel_version="" +# Flag know if need to download the kernel source +download_kernel=false +# The repository where kernel configuration lives +runtime_repository="github.com/${project_name}/runtime" +# The repository where kernel configuration lives +readonly kernel_config_repo="github.com/${project_name}/packaging" +readonly patches_repo="github.com/${project_name}/packaging" +readonly patches_repo_dir="${GOPATH}/src/${patches_repo}" +# Default path to search patches to apply to kernel +readonly default_patches_dir="${patches_repo_dir}/kernel/patches/" +# Default path to search config for kata +readonly default_kernel_config_dir="${GOPATH}/src/${kernel_config_repo}/kernel/configs/" +#Path to kernel directory +kernel_path="" +# +patches_path="" +# +hypervisor_target="" +# +arch_target="" +# +kernel_config_path="" +# destdir +DESTDIR="${DESTDIR:-/}" +#PREFIX= +PREFIX="${PREFIX:-/usr}" + +source "${script_dir}/../scripts/lib.sh" + +usage() { + cat < + +Commands: + +- setup + +- build + +- install + +Options: + + -c : Path to config file to build a the kernel + -h : Display this help. + -k : Path to kernel to build + -p : Path to a directory with patches to apply to kernel. + -v : Kernel version to use if kernel path not provided. +EOT +} + +# Convert architecture to the name used by the Linux kernel build system +arch_to_kernel() { + local -r arch="$1" + + case "$arch" in + aarch64) echo "arm64" ;; + ppc64le) echo "powerpc" ;; + x86_64) echo "$arch" ;; + *) die "unsupported architecture: $arch" ;; + esac +} + +get_kernel() { + local version="${1:-}" + #Remove extra 'v' + version=${version#v} + + local kernel_path=${2:-} + [ -n "${kernel_path}" ] || die "kernel_path not provided" + [ ! -d "${kernel_path}" ] || die "kernel_path already exist" + + major_version=$(echo "${version}" | cut -d. -f1) + kernel_tarball="linux-${version}.tar.xz" + + curl --fail -OL "https://cdn.kernel.org/pub/linux/kernel/v${major_version}.x/sha256sums.asc" + grep "${kernel_tarball}" sha256sums.asc >"${kernel_tarball}.sha256" + + if [ -f "${kernel_tarball}" ] && ! sha256sum -c "${kernel_tarball}.sha256"; then + info "invalid kernel tarball ${kernel_tarball} removing " + rm -f "${kernel_tarball}" + fi + if [ ! -f "${kernel_tarball}" ]; then + info "Download kernel version ${version}" + info "Download kernel" + curl --fail -OL "https://www.kernel.org/pub/linux/kernel/v${major_version}.x/${kernel_tarball}" + else + info "kernel tarball already downloaded" + fi + + sha256sum -c "${kernel_tarball}.sha256" + + tar xf ${kernel_tarball} + + mv "linux-${version}" "${kernel_path}" +} + +get_default_kernel_config() { + local version="${1}" + + local hypervisor="$2" + local kernel_arch="$3" + + [ -n "${version}" ] || die "kernel version not provided" + [ -n "${hypervisor}" ] || die "hypervisor not provided" + [ -n "${kernel_arch}" ] || die "kernel arch not provided" + + major_version=$(echo "${version}" | cut -d. -f1) + minor_version=$(echo "${version}" | cut -d. -f2) + config="${default_kernel_config_dir}/${kernel_arch}_kata_${hypervisor}_${major_version}.${minor_version}.x" + [ -f "${config}" ] || die "failed to find default config ${config}" + echo "${config}" +} + +get_config_version() { + config_version_file="${default_patches_dir}/../kata_config_version" + if [ -f "${config_version_file}" ]; then + cat "${config_version_file}" + else + echo "unknown" + fi +} + +setup_kernel() { + local kernel_path=${1:-} + [ -n "${kernel_path}" ] || die "kernel_path not provided" + if [ -d "$kernel_path" ]; then + info "${kernel_path} already exist" + return + fi + + info "kernel path does not exist, will download kernel" + download_kernel="true" + [ -n "$kernel_version" ] || die "failed to get kernel version: Kernel version is emtpy" + + if [[ "${download_kernel}" == "true" ]]; then + get_kernel "${kernel_version}" "${kernel_path}" + fi + + [ -n "$kernel_path" ] || die "failed to find kernel source path" + + if [ -z "${patches_path}" ]; then + patches_path="${default_patches_dir}" + [ -d "${patches_path}" ] || git clone "https://${patches_repo}.git" "${patches_repo_dir}" + fi + + [ -d "${patches_path}" ] || die " patches path '${patches_path}' does not exist" + + kernel_patches=$(find "${patches_path}" -name '*.patch' -type f) + + pushd "${kernel_path}" >>/dev/null + for p in ${kernel_patches}; do + info "Applying patch $p" + patch -p1 <"$p" + done + + [ -n "${hypervisor_target}" ] || hypervisor_target="kvm" + [ -n "${arch_target}" ] || arch_target="$(uname -m)" + [ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}") + + cp "${kernel_config_path}" ./.config + make oldconfig +} + +build_kernel() { + local kernel_path=${1:-} + [ -n "${kernel_path}" ] || die "kernel_path not provided" + [ -d "${kernel_path}" ] || die "path to kernel does not exist, use ${script_name} setup" + [ -n "${arch_target}" ] || arch_target="$(arch)" + arch_target=$(arch_to_kernel "${arch_target}") + pushd "${kernel_path}" >>/dev/null + make -j $(nproc) ARCH="${arch_target}" + [ -e "arch/${arch_target}/boot/bzImage" ] || [ -e "arch/${arch_target}/boot/Image.gz" ] + [ -e "vmlinux" ] + popd >>/dev/null +} + +install_kata() { + local kernel_path=${1:-} + [ -n "${kernel_path}" ] || die "kernel_path not provided" + [ -d "${kernel_path}" ] || die "path to kernel does not exist, use ${script_name} setup" + pushd "${kernel_path}" >>/dev/null + config_version=$(get_config_version) + [ -n "${config_version}" ] || die "failed to get config version" + install_path=$(readlink -m "${DESTDIR}/${PREFIX}/share/${project_name}") + vmlinuz="vmlinuz-${kernel_version}-${config_version}" + vmlinux="vmlinux-${kernel_version}-${config_version}" + + if [ -e "arch/${arch_target}/boot/bzImage" ]; then + bzImage="arch/${arch_target}/boot/bzImage" + elif [ -e "arch/${arch_target}/boot/Image.gz" ]; then + bzImage="arch/${arch_target}/boot/Image.gz" + else + die "failed to find bzImage" + fi + + install --mode 0644 -D "${bzImage}" "${install_path}/${vmlinuz}" + install --mode 0644 -D "vmlinux" "${install_path}/${vmlinux}" + install --mode 0644 -D ./.config "${install_path}/config-${kernel_version}" + ln -sf "${vmlinuz}" "${install_path}/vmlinuz.container" + ln -sf "${vmlinux}" "${install_path}/vmlinux.container" + ls -la "${install_path}/vmlinux.container" + ls -la "${install_path}/vmlinuz.container" + popd >>/dev/null +} + +main() { + while getopts "a:c:hk:p:t:v:" opt; do + case "$opt" in + a) + arch_target="${OPTARG}" + ;; + c) + kernel_config_path="${OPTARG}" + ;; + + h) + usage + exit 0 + ;; + + k) + kernel_path="${OPTARG}" + ;; + + t) + hypervisor_target="${OPTARG}" + ;; + p) + patches_path="${OPTARG}" + ;; + v) + kernel_version="${OPTARG}" + ;; + esac + done + + shift $(($OPTIND - 1)) + + subcmd="${1:-}" + + [ -z "${subcmd}" ] && usage 1 + + # If not kernel version take it from versions.yaml + if [ -z "$kernel_version" ]; then + kernel_version=$(get_from_kata_deps "assets.kernel.version") + #Remove extra 'v' + kernel_version="${kernel_version#v}" + fi + + if [ -z "${kernel_path}" ]; then + config_version=$(get_config_version) + kernel_path="${PWD}/kata-linux-${kernel_version}-${config_version}" + fi + + case "${subcmd}" in + build) + build_kernel "${kernel_path}" + ;; + install) + build_kernel "${kernel_path}" + install_kata "${kernel_path}" + ;; + setup) + setup_kernel "${kernel_path}" + [ -d "${kernel_path}" ] || die "${kernel_path} does not exist" + echo "Kernel source ready: ${kernel_path} " + ;; + *) + usage 1 + ;; + + esac +} + +main $@ diff --git a/kernel/build-kernel_test.sh b/kernel/build-kernel_test.sh new file mode 100755 index 000000000..d2ae3153d --- /dev/null +++ b/kernel/build-kernel_test.sh @@ -0,0 +1,77 @@ +#!/bin/bash +#Copyright (c) 2018 Intel Corporation +# +#SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly build_kernel_sh="${script_dir}/build-kernel.sh" +readonly tmp_dir=$(mktemp -d -t build-kernel-tmp.XXXXXXXXXX) + +exit_handler() { + rm -rf "$tmp_dir" +} +trap exit_handler EXIT + +OK() { + echo "OK" +} + +FAIL() { + echo "FAIL: $*" + exit -1 +} + +export GOPATH=${GOPATH:-$HOME/go} + +source "${script_dir}/../scripts/lib.sh" + +kata_kernel_version=$(get_from_kata_deps "assets.kernel.version") +kata_kernel_version=${kata_kernel_version/v/} +kernel_dir="kata-linux-${kata_kernel_version}-$(cat ${script_dir}/kata_config_version)" + +check_help() { + echo "Check help works" + out=$(${build_kernel_sh} -h) + [[ ${out} == *"Usage"* ]] + OK +} + +build_kernel() { + echo "Setup a default kernel" + out=$(${build_kernel_sh} setup 2>&1) + [ -f "linux-${kata_kernel_version}.tar.xz" ] || FAIL "tarball does not exist" + [ -d "${kernel_dir}" ] || FAIL "kernel directory does not exist" + OK + + echo "Setup a default again wont download again the kernel" + new_kernel_dir="${PWD}/kernel-kata2" + out=$(${build_kernel_sh} -k "${new_kernel_dir}" setup 2>&1) + [[ ${out} == *"kernel tarball already downloaded"* ]] + [ -f "linux-${kata_kernel_version}.tar.xz" ] || FAIL "tarball does not exist" + [ -d "${new_kernel_dir}" ] || FAIL "kernel directory does not exist" + OK + + echo "Build default kernel" + out=$(${build_kernel_sh} build 2>&1) + [ -e "${kernel_dir}/arch/$(uname -m)/boot/bzImage" ] || FAIL "bzImage not found" + [ -e "${kernel_dir}/vmlinux" ] || FAIL "vmlinux not found" + OK + + echo "Install kernel" + export DESTDIR="${tmp_dir}/kernel-install-path" + out=$(${build_kernel_sh} install 2>&1) + [ -e "${DESTDIR}/usr/share/kata-containers/vmlinux.container" ] + [ -e "${DESTDIR}/usr/share/kata-containers/vmlinuz.container" ] + unset DESTDIR + OK +} + +pushd ${tmp_dir} +check_help +build_kernel +popd