diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index ade4eb9e7..b5c7584fe 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -2,6 +2,10 @@ name: CI | Build kata-static tarball for amd64 on: workflow_call: inputs: + stage: + required: false + type: string + default: test tarball-suffix: required: false type: string @@ -15,8 +19,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + stage: + - ${{ inputs.stage }} asset: - cloud-hypervisor + - cloud-hypervisor-glibc - firecracker - kernel - kernel-sev @@ -32,11 +39,16 @@ jobs: - qemu-snp-experimental - qemu-tdx-experimental - rootfs-image + - rootfs-image-tdx - rootfs-initrd + - rootfs-initrd-mariner - rootfs-initrd-sev - shim-v2 - tdvf - virtiofsd + exclude: + - stage: release + asset: cloud-hypervisor-glibc steps: - name: Login to Kata Containers quay.io if: ${{ inputs.push-to-registry == 'yes' }} diff --git a/.github/workflows/ci-on-push.yaml b/.github/workflows/ci-on-push.yaml index 3951bba08..819a9b33c 100644 --- a/.github/workflows/ci-on-push.yaml +++ b/.github/workflows/ci-on-push.yaml @@ -52,3 +52,7 @@ jobs: registry: ghcr.io repo: ${{ github.repository_owner }}/kata-deploy-ci tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64 + + run-metrics-tests: + needs: build-kata-static-tarball-amd64 + uses: ./.github/workflows/run-launchtimes-metrics.yaml diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml index 8d48b956b..6ff990696 100644 --- a/.github/workflows/release-amd64.yaml +++ b/.github/workflows/release-amd64.yaml @@ -9,6 +9,8 @@ on: jobs: build-kata-static-tarball-amd64: uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml + with: + stage: release kata-deploy: needs: build-kata-static-tarball-amd64 diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml index 2b5e810a3..cd7db8fdf 100644 --- a/.github/workflows/release-arm64.yaml +++ b/.github/workflows/release-arm64.yaml @@ -9,6 +9,8 @@ on: jobs: build-kata-static-tarball-arm64: uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml + with: + stage: release kata-deploy: needs: build-kata-static-tarball-arm64 diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml index ef436b7b8..5f3aaf05f 100644 --- a/.github/workflows/release-s390x.yaml +++ b/.github/workflows/release-s390x.yaml @@ -9,6 +9,8 @@ on: jobs: build-kata-static-tarball-s390x: uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml + with: + stage: release kata-deploy: needs: build-kata-static-tarball-s390x diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index af9b93132..c553ca514 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -83,7 +83,7 @@ jobs: - name: push amd64 static tarball to github run: | tag=$(echo $GITHUB_REF | cut -d/ -f3-) - tarball="kata-static-$tag-x86_64.tar.xz" + tarball="kata-static-$tag-amd64.tar.xz" mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}" pushd $GITHUB_WORKSPACE echo "uploading asset '${tarball}' for tag: ${tag}" @@ -97,7 +97,7 @@ jobs: - name: push arm64 static tarball to github run: | tag=$(echo $GITHUB_REF | cut -d/ -f3-) - tarball="kata-static-$tag-aarch64.tar.xz" + tarball="kata-static-$tag-arm64.tar.xz" mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}" pushd $GITHUB_WORKSPACE echo "uploading asset '${tarball}' for tag: ${tag}" diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 4ed5a2d30..a39c2bbcd 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -17,79 +17,54 @@ jobs: strategy: fail-fast: false matrix: + host_os: + - ubuntu vmm: - clh - dragonball - qemu + include: + - host_os: cbl-mariner + vmm: clh runs-on: ubuntu-latest + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ github.event.pull_request.number }} + KATA_HOST_OS: ${{ matrix.host_os }} + KATA_HYPERVISOR: ${{ matrix.vmm }} steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Download Azure CLI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + run: bash tests/integration/gha-run.sh install-azure-cli - name: Log into the Azure account - run: | - az login \ - --service-principal \ - -u "${{ secrets.AZ_APPID }}" \ - -p "${{ secrets.AZ_PASSWORD }}" \ - --tenant "${{ secrets.AZ_TENANT_ID }}" + run: bash tests/integration/gha-run.sh login-azure + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - name: Create AKS cluster - run: | - az aks create \ - -g "kataCI" \ - -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \ - -s "Standard_D4s_v5" \ - --node-count 1 \ - --generate-ssh-keys + run: bash tests/integration/gha-run.sh create-cluster - name: Install `bats` - run: | - sudo apt-get update - sudo apt-get -y install bats + run: bash tests/integration/gha-run.sh install-bats - name: Install `kubectl` - run: | - sudo az aks install-cli + run: bash tests/integration/gha-run.sh install-kubectl - name: Download credentials for the Kubernetes CLI to use them - run: | - az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64 + run: bash tests/integration/gha-run.sh get-cluster-credentials - name: Run tests - timeout-minutes: 35 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 150s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + timeout-minutes: 60 + run: bash tests/integration/gha-run.sh run-tests-aks - name: Delete AKS cluster if: always() - run: | - az aks delete \ - -g "kataCI" \ - -n "${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64" \ - --yes \ - --no-wait + run: bash tests/integration/gha-run.sh delete-cluster diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index 98a6db610..52ab7f955 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -21,6 +21,10 @@ jobs: - qemu-sev runs-on: sev env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-sev - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-sev diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 541695e0f..535c6de6d 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -21,6 +21,10 @@ jobs: - qemu-snp runs-on: sev-snp env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-snp - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-snp diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 6c5ba0dc1..886b1c026 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -21,6 +21,10 @@ jobs: - qemu-tdx runs-on: tdx env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /etc/rancher/k3s/k3s.yaml steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-tdx - name: Delete kata-deploy if: always() - run: | - kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - sleep 180s - - kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-tdx diff --git a/.github/workflows/run-launchtimes-metrics.yaml b/.github/workflows/run-launchtimes-metrics.yaml new file mode 100644 index 000000000..bfbb9f2eb --- /dev/null +++ b/.github/workflows/run-launchtimes-metrics.yaml @@ -0,0 +1,19 @@ +name: CI | Run launch-times metrics +on: + workflow_call: + +jobs: + launch-times-tests: + runs-on: metrics + env: + GOPATH: ${{ github.workspace }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: run launch times on qemu + run: bash tests/metrics/gha-run.sh run-test-launchtimes-qemu + + - name: run launch times on clh + run: bash tests/metrics/gha-run.sh run-test-launchtimes-clh diff --git a/.github/workflows/snap-release.yaml b/.github/workflows/snap-release.yaml deleted file mode 100644 index 4a12ce8d7..000000000 --- a/.github/workflows/snap-release.yaml +++ /dev/null @@ -1,53 +0,0 @@ -name: Release Kata in snapcraft store -on: - push: - tags: - - '[0-9]+.[0-9]+.[0-9]+*' - -env: - SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.snapcraft_token }} - -jobs: - release-snap: - runs-on: ubuntu-20.04 - steps: - - name: Check out Git repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Snapcraft - run: | - # Required to avoid snapcraft install failure - sudo chown root:root / - - # "--classic" is needed for the GitHub action runner - # environment. - sudo snap install snapcraft --classic - - # Allow other parts to access snap binaries - echo /snap/bin >> "$GITHUB_PATH" - - - name: Build snap - run: | - # Removing man-db, workflow kept failing, fixes: #4480 - sudo apt -y remove --purge man-db - sudo apt-get update - sudo apt-get install -y git git-extras - kata_url="https://github.com/kata-containers/kata-containers" - latest_version=$(git ls-remote --tags ${kata_url} | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1) - current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)" - # Check semantic versioning format (x.y.z) and if the current tag is the latest tag - if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then - # Current version is the latest version, build it - snapcraft snap --debug --destructive-mode - fi - - - name: Upload snap - run: | - snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)" - snap_file="kata-containers_${snap_version}_amd64.snap" - # Upload the snap if it exists - if [ -f ${snap_file} ]; then - snapcraft upload --release=stable ${snap_file} - fi diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml deleted file mode 100644 index ac163fa2e..000000000 --- a/.github/workflows/snap.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: snap CI -on: - pull_request: - types: - - opened - - synchronize - - reopened - - edited - paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ] - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - name: Check out - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Snapcraft - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - # Required to avoid snapcraft install failure - sudo chown root:root / - - # "--classic" is needed for the GitHub action runner - # environment. - sudo snap install snapcraft --classic - - # Allow other parts to access snap binaries - echo /snap/bin >> "$GITHUB_PATH" - - - name: Build snap - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - snapcraft snap --debug --destructive-mode diff --git a/README.md b/README.md index 662899ea6..b53125194 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ - + + +[![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) # Kata Containers @@ -144,8 +146,6 @@ The table below lists the remaining parts of the project: Kata Containers is now [available natively for most distributions](docs/install/README.md#packaged-installation-methods). -However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See -the [components](#components) section for further details. ## Glossary of Terms diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 7f0b3eaa7..2f60ab165 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -587,10 +587,15 @@ $ sudo kata-monitor #### Connect to debug console -Command `kata-runtime exec` is used to connect to the debug console. +You need to start a container for example: +```bash +$ sudo ctr run --runtime io.containerd.kata.v2 -d docker.io/library/ubuntu:latest testdebug +``` + +Then, you can use the command `kata-runtime exec ` to connect to the debug console. ``` -$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd +$ kata-runtime exec testdebug bash-4.2# id uid=0(root) gid=0(root) groups=0(root) bash-4.2# pwd diff --git a/docs/Limitations.md b/docs/Limitations.md index d122e5ca5..74a6acf2d 100644 --- a/docs/Limitations.md +++ b/docs/Limitations.md @@ -147,7 +147,8 @@ these commands is potentially challenging. See issue https://github.com/clearcontainers/runtime/issues/341 and [the constraints challenge](#the-constraints-challenge) for more information. For CPUs resource management see -[CPU constraints](design/vcpu-handling.md). +[CPU constraints(in runtime-go)](design/vcpu-handling-runtime-go.md). +[CPU constraints(in runtime-rs)](design/vcpu-handling-runtime-rs.md). # Architectural limitations diff --git a/docs/design/README.md b/docs/design/README.md index 0c732defd..2fe93b5f6 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -6,7 +6,8 @@ Kata Containers design documents: - [API Design of Kata Containers](kata-api-design.md) - [Design requirements for Kata Containers](kata-design-requirements.md) - [VSocks](VSocks.md) -- [VCPU handling](vcpu-handling.md) +- [VCPU handling(in runtime-go)](vcpu-handling-runtime-go.md) +- [VCPU handling(in runtime-rs)](vcpu-handling-runtime-rs.md) - [VCPU threads pinning](vcpu-threads-pinning.md) - [Host cgroups](host-cgroups.md) - [Agent systemd cgroup](agent-systemd-cgroup.md) diff --git a/docs/design/VSocks.md b/docs/design/VSocks.md index 0271645c2..9375f30f5 100644 --- a/docs/design/VSocks.md +++ b/docs/design/VSocks.md @@ -78,4 +78,4 @@ with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, i the containers are removed automatically. [1]: https://wiki.qemu.org/Features/VirtioVsock -[2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods +[2]: ./vcpu-handling-runtime-go.md#virtual-cpus-and-kubernetes-pods diff --git a/docs/design/vcpu-handling.md b/docs/design/vcpu-handling-runtime-go.md similarity index 100% rename from docs/design/vcpu-handling.md rename to docs/design/vcpu-handling-runtime-go.md diff --git a/docs/design/vcpu-handling-runtime-rs.md b/docs/design/vcpu-handling-runtime-rs.md new file mode 100644 index 000000000..44989ce4e --- /dev/null +++ b/docs/design/vcpu-handling-runtime-rs.md @@ -0,0 +1,51 @@ +# Virtual machine vCPU sizing in Kata Containers 3.0 + +> Preview: +> [Kubernetes(since 1.23)][1] and [Containerd(since 1.6.0-beta4)][2] will help calculate `Sandbox Size` info and pass it to Kata Containers through annotations. +> In order to adapt to this beneficial change and be compatible with the past, we have implemented the new vCPUs handling way in `runtime-rs`, which is slightly different from the original `runtime-go`'s design. + +## When do we need to handle vCPUs size? +vCPUs sizing should be determined by the container workloads. So throughout the life cycle of Kata Containers, there are several points in time when we need to think about how many vCPUs should be at the time. Mainly including the time points of `CreateVM`, `CreateContainer`, `UpdateContainer`, and `DeleteContainer`. +* `CreateVM`: When creating a sandbox, we need to know how many vCPUs to start the VM with. +* `CreateContainer`: When creating a new container in the VM, we may need to hot-plug the vCPUs according to the requirements in container's spec. +* `UpdateContainer`: When receiving the `UpdateContainer` request, we may need to update the vCPU resources according to the new requirements of the container. +* `DeleteContainer`: When a container is removed from the VM, we may need to hot-unplug the vCPUs to reclaim the vCPU resources introduced by the container. + +## On what basis do we calculate the number of vCPUs? +When Kata calculate the number of vCPUs, We have three data sources, the `default_vcpus` and `default_maxvcpus` specified in the configuration file (named `TomlConfig` later in the doc), the `io.kubernetes.cri.sandbox-cpu-quota` and `io.kubernetes.cri.sandbox-cpu-period` annotations passed by the upper layer runtime, and the corresponding CPU resource part in the container's spec for the container when `CreateContainer`/`UpdateContainer`/`DeleteContainer` is requested. + +Our understanding and priority of these resources are as follows, which will affect how we calculate the number of vCPUs later. + +* From `TomlConfig`: + * `default_vcpus`: default number of vCPUs when starting a VM. + * `default_maxvcpus`: maximum number of vCPUs. +* From `Annotation`: + * `InitialSize`: we call the size of the resource passed from the annotations as `InitialSize`. Kubernetes will calculate the sandbox size according to the Pod's statement, which is the `InitialSize` here. This size should be the size we want to prioritize. +* From `Container Spec`: + * The amount of CPU resources that the Container wants to use will be declared through the spec. Including the aforementioned annotations, we mainly consider `cpu quota` and `cpuset` when calculating the number of vCPUs. + * `cpu quota`: `cpu quota` is the most common way to declare the amount of CPU resources. The number of vCPUs introduced by `cpu quota` declared in a container's spec is: `vCPUs = ceiling( quota / period )`. + * `cpuset`: `cpuset` is often used to bind the CPUs that tasks can run on. The number of vCPUs may introduced by `cpuset` declared in a container's spec is the number of CPUs specified in the set that do not overlap with other containers. + + +## How to calculate and adjust the vCPUs size: +There are two types of vCPUs that we need to consider, one is the number of vCPUs when starting the VM (named `Boot Size` in the doc). The second is the number of vCPUs when `CreateContainer`/`UpdateContainer`/`DeleteContainer` request is received (`Real-time Size` in the doc). + +### `Boot Size` +The main considerations are `InitialSize` and `default_vcpus`. There are the following principles: +`InitialSize` has priority over `default_vcpus` declared in `TomlConfig`. +1. When there is such an annotation statement, the originally `default_vcpus` will be modified to the number of vCPUs in the `InitialSize` as the `Boot Size`. (Because not all runtimes support this annotation for the time being, we still keep the `default_cpus` in `TomlConfig`.) +2. When the specs of all containers are aggregated for sandbox size calculation, the method is consistent with the calculation method of `InitialSize` here. + +### `Real-time Size` +When we receive an OCI request, it may be for a single container. But what we have to consider is the number of vCPUs for the entire VM. So we will maintain a list. Every time there is a demand for adjustment, the entire list will be traversed to calculate a value for the number of vCPUs. In addition, there are the following principles: +1. Do not cut computing power and try to keep the number of vCPUs specified by `InitialSize`. + * So the number of vCPUs after will not be less than the `Boot Size`. +2. `cpu quota` takes precedence over `cpuset` and the setting history are took into account. + * We think quota describes the CPU time slice that a cgroup can use, and `cpuset` describes the actual CPU number that a cgroup can use. Quota can better describe the size of the CPU time slice that a cgroup actually wants to use. The `cpuset` only describes which CPUs the cgroup can use, but the cgroup can use the specified CPU but consumes a smaller time slice, so the quota takes precedence over the `cpuset`. + * On the one hand, when both `cpu quota` and `cpuset` are specified, we will calculate the number of vCPUs based on `cpu quota` and ignore `cpuset`. On the other hand, if `cpu quota` was used to control the number of vCPUs in the past, and only `cpuset` was updated during `UpdateContainer`, we will not adjust the number of vCPUs at this time. +3. `StaticSandboxResourceMgmt` controls hotplug. + * Some VMMs and kernels of some architectures do not support hotplugging. We can accommodate this situation through `StaticSandboxResourceMgmt`. When `StaticSandboxResourceMgmt = true` is set, we don't make any further attempts to update the number of vCPUs after booting. + + +[1]: https://github.com/kubernetes/kubernetes/pull/104886 +[2]: https://github.com/containerd/containerd/pull/6155 diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 024f29337..4679f8d46 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -45,6 +45,8 @@ - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md) - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md) - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md) +- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md) + ## Confidential Containers - [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md) - [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md) diff --git a/docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md b/docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md new file mode 100644 index 000000000..d4fd048ec --- /dev/null +++ b/docs/how-to/how-to-run-kata-containers-with-kinds-of-Block-Volumes.md @@ -0,0 +1,78 @@ +# A new way for Kata Containers to use Kinds of Block Volumes + +> **Note:** This guide is only available for runtime-rs with default Hypervisor Dragonball. +> Now, other hypervisors are still ongoing, and it'll be updated when they're ready. + + +## Background + +Currently, there is no widely applicable and convenient method available for users to use some kinds of backend storages, such as File on host based block volume, SPDK based volume or VFIO device based volume for Kata Containers, so we adopt [Proposal: Direct Block Device Assignment](https://github.com/kata-containers/kata-containers/blob/main/docs/design/direct-blk-device-assignment.md) to address it. + +## Solution + +According to the proposal, it requires to use the `kata-ctl direct-volume` command to add a direct assigned block volume device to the Kata Containers runtime. + +And then with the help of method [get_volume_mount_info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L95), get information from JSON file: `(mountinfo.json)` and parse them into structure [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70) which is used to save device-related information. + +We only fill the `mountinfo.json`, such as `device` ,`volume_type`, `fs_type`, `metadata` and `options`, which correspond to the fields in [Direct Volume Info](https://github.com/kata-containers/kata-containers/blob/099b4b0d0e3db31b9054e7240715f0d7f51f9a1c/src/libs/kata-types/src/mount.rs#L70), to describe a device. + +The JSON file `mountinfo.json` placed in a sub-path `/kubelet/kata-test-vol-001/volume001` which under fixed path `/run/kata-containers/shared/direct-volumes/`. +And the full path looks like: `/run/kata-containers/shared/direct-volumes/kubelet/kata-test-vol-001/volume001`, But for some security reasons. it is +encoded as `/run/kata-containers/shared/direct-volumes/L2t1YmVsZXQva2F0YS10ZXN0LXZvbC0wMDEvdm9sdW1lMDAx`. + +Finally, when running a Kata Containers witch `ctr run --mount type=X, src=Y, dst=Z,,options=rbind:rw`, the `type=X` should be specified a proprietary type specifically designed for some kind of volume. + +Now, supported types: + +- `directvol` for direct volume +- `spdkvol` for SPDK volume (TBD) +- `vfiovol` for VFIO device based volume (TBD) + + +## Setup Device and Run a Kata-Containers + +### Direct Block Device Based Volume + +#### create raw block based backend storage + +> **Tips:** raw block based backend storage MUST be formatted with `mkfs`. + +```bash +$ sudo dd if=/dev/zero of=/tmp/stor/rawdisk01.20g bs=1M count=20480 +$ sudo mkfs.ext4 /tmp/stor/rawdisk01.20g +``` + +#### setup direct block device for kata-containers + +```json +{ + "device": "/tmp/stor/rawdisk01.20g", + "volume_type": "directvol", + "fs_type": "ext4", + "metadata":"{}", + "options": [] +} +``` + +```bash +$ sudo ./kata-ctl direct-volume add /kubelet/kata-direct-vol-002/directvol002 "{\"device\": \"/tmp/stor/rawdisk01.20g\", \"volume_type\": \"directvol\", \"fs_type\": \"ext4\", \"metadata\":"{}", \"options\": []}" +$# /kubelet/kata-direct-vol-002/directvol002 <==> /run/kata-containers/shared/direct-volumes/W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx +$ cat W1lMa2F0ZXQva2F0YS10a2F0DAxvbC0wMDEvdm9sdW1lMDAx/mountInfo.json +{"volume_type":"directvol","device":"/tmp/stor/rawdisk01.20g","fs_type":"ext4","metadata":{},"options":[]} +``` + +#### Run a Kata container with direct block device volume + +```bash +$ # type=disrectvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw +$sudo ctr run -t --rm --runtime io.containerd.kata.v2 --mount type=directvol,src=/kubelet/kata-direct-vol-002/directvol002,dst=/disk002,options=rbind:rw "$image" kata-direct-vol-xx05302045 /bin/bash +``` + + +### SPDK Device Based Volume + +TBD + +### VFIO Device Based Volume + +TBD \ No newline at end of file diff --git a/docs/install/README.md b/docs/install/README.md index 8a86bc7c8..5fb1c3a03 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -19,7 +19,6 @@ Packaged installation methods uses your distribution's native package format (su |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------| | [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | -| ~~[Using snap](#snap-installation)~~ | ~~Easy to install~~ | ~~yes~~ | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~ | | [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | | [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | | [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | @@ -42,27 +41,6 @@ Kata packages are provided by official distribution repositories for: | [CentOS](centos-installation-guide.md) | 8 | | [Fedora](fedora-installation-guide.md) | 34 | -### Snap Installation - -> **WARNING:** -> -> The Snap package method is **unmaintained** and only provides an old -> version of Kata Containers: -> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) -> provides Kata Containers -> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) -> but the latest stable Kata Containers release at the time of writing is -> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). -> -> We recommend strongly that you switch to an alternative Kata Containers installation method. -> -> See: https://github.com/kata-containers/kata-containers/issues/6769 -> for further details. - -~~The snap installation is available for all distributions which support `snapd`.~~ - -~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~ - ### Automatic Installation [Use `kata-manager`](/utils/README.md) to automatically install a working Kata Containers system. diff --git a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md index 4cfcb392d..bb1732292 100644 --- a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md +++ b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md @@ -26,7 +26,6 @@ architectures: |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- | | [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes | | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | No | -| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | No | | [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | No | | [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | No | | [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | Yes | @@ -36,8 +35,6 @@ architectures: Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md). ### Official packages `ToDo` -### Snap Installation -`ToDo` ### Automatic Installation `ToDo` ### Manual Installation diff --git a/docs/install/snap-installation-guide.md b/docs/install/snap-installation-guide.md deleted file mode 100644 index acf2e637e..000000000 --- a/docs/install/snap-installation-guide.md +++ /dev/null @@ -1,82 +0,0 @@ -# Kata Containers snap package - -> **WARNING:** -> -> The Snap package method is **unmaintained** and only provides an old -> version of Kata Containers: -> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) -> provides Kata Containers -> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) -> but the latest stable Kata Containers release at the time of writing is -> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). -> -> We recommend strongly that you switch to an alternative Kata Containers installation method. -> -> See: https://github.com/kata-containers/kata-containers/issues/6769 -> for further details. - -## Install Kata Containers - -Kata Containers can be installed in any Linux distribution that supports -[snapd](https://docs.snapcraft.io/installing-snapd). - -Run the following command to install **Kata Containers**: - -> **WARNING:** -> -> The Snap package method is **unmaintained** and only provides an old -> version of Kata Containers: -> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) -> provides Kata Containers -> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) -> but the latest stable Kata Containers release at the time of writing is -> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). -> -> We recommend strongly that you switch to an alternative Kata Containers installation method. -> -> See: https://github.com/kata-containers/kata-containers/issues/6769 -> for further details. - -```sh -$ sudo snap install kata-containers --stable --classic -``` - -## Configure Kata Containers - -By default Kata Containers snap image is mounted at `/snap/kata-containers` as a -read-only file system, therefore default configuration file can not be edited. -Fortunately Kata Containers supports loading a configuration file from another -path than the default. - -```sh -$ sudo mkdir -p /etc/kata-containers -$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/ -$ $EDITOR /etc/kata-containers/configuration.toml -``` - -## Integration with shim v2 Container Engines - -The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the -`containerd-shim-kata-v2` binary to allow Kata Containers to be created. -Run the following command to create a symbolic link to the shim v2 binary. - -```sh -$ sudo ln -sf /snap/kata-containers/current/usr/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2 -``` - -Once the symbolic link has been created and the engine daemon configured, `io.containerd.kata.v2` -can be used as runtime. - -Read the following documents to know how to run Kata Containers 2.x with `containerd`. - -* [How to use Kata Containers and Containerd](../how-to/containerd-kata.md) -* [Install Kata Containers with containerd](./container-manager/containerd/containerd-install.md) - - -## Remove Kata Containers snap package - -Run the following command to remove the Kata Containers snap: - -```sh -$ sudo snap remove kata-containers -``` diff --git a/snap/local/README.md b/snap/local/README.md deleted file mode 100644 index 4b449ef17..000000000 --- a/snap/local/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Kata Containers snap image - -This directory contains the resources needed to build the Kata Containers -[snap][1] image. - -## Initial setup - -Kata Containers can be installed in any Linux distribution that supports -[snapd](https://docs.snapcraft.io/installing-snapd). For this example, we -assume Ubuntu as your base distro. -```sh -$ sudo apt-get --no-install-recommends install -y apt-utils ca-certificates snapd snapcraft -``` - -## Install snap - -You can install the Kata Containers snap from the [snapcraft store][8] or by running the following command: - -```sh -$ sudo snap install kata-containers --classic -``` - -## Build and install snap image - -Run the command below which will use the packaging Makefile to build the snap image: - -```sh -$ make -C tools/packaging snap -``` - -> **Warning:** -> -> By default, `snapcraft` will create a clean virtual machine -> environment to build the snap in using the `multipass` tool. -> -> However, `multipass` is silently disabled when `--destructive-mode` is -> used. -> -> Since building the Kata Containers package currently requires -> `--destructive-mode`, the snap will be built using the host -> environment. To avoid parts of the build auto-detecting additional -> features to enable (for example for QEMU), we recommend that you -> only run the snap build in a minimal host environment. - -To install the resulting snap image, snap must be put in [classic mode][3] and the -security confinement must be disabled (`--classic`). Also since the resulting snap -has not been signed the verification of signature must be omitted (`--dangerous`). - -```sh -$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap" -``` - -Replace `${version}` with the current version of Kata Containers and `${arch}` with -the system architecture. - -## Configure Kata Containers - -By default Kata Containers snap image is mounted at `/snap/kata-containers` as a -read-only file system, therefore default configuration file can not be edited. -Fortunately [`kata-runtime`][4] supports loading a configuration file from another -path than the default. - -```sh -$ sudo mkdir -p /etc/kata-containers -$ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/ -$ $EDITOR /etc/kata-containers/configuration.toml -``` - -## Integration with docker and Kubernetes - -The path to the runtime provided by the Kata Containers snap image is -`/snap/kata-containers/current/usr/bin/kata-runtime`. You should use it to -run Kata Containers with [docker][9] and [Kubernetes][10]. - -## Remove snap - -You can remove the Kata Containers snap by running the following command: - -```sh -$ sudo snap remove kata-containers -``` - -## Limitations - -The [miniOS image][2] is not included in the snap image as it is not possible for -QEMU to open a guest RAM backing store on a read-only filesystem. Fortunately, -you can start Kata Containers with a Linux initial RAM disk (initrd) that is -included in the snap image. If you want to use the miniOS image instead of initrd, -then a new configuration file can be [created](#configure-kata-containers) -and [configured][7]. - -[1]: https://docs.snapcraft.io/snaps/intro -[2]: ../../docs/design/architecture/README.md#root-filesystem-image -[3]: https://docs.snapcraft.io/reference/confinement#classic -[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration -[5]: https://docs.docker.com/engine/reference/commandline/dockerd -[6]: ../../docs/install/docker/ubuntu-docker-install.md -[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image -[8]: https://snapcraft.io/kata-containers -[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker -[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes diff --git a/snap/local/snap-common.sh b/snap/local/snap-common.sh deleted file mode 100644 index 0a2a18e15..000000000 --- a/snap/local/snap-common.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2022 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 - -# Description: Idempotent script to be sourced by all parts in a -# snapcraft config file. - -set -o errexit -set -o nounset -set -o pipefail - -# XXX: Bash-specific code. zsh doesn't support this option and that *does* -# matter if this script is run sourced... since it'll be using zsh! ;) -[ -n "$BASH_VERSION" ] && set -o errtrace - -[ -n "${DEBUG:-}" ] && set -o xtrace - -die() -{ - echo >&2 "ERROR: $0: $*" -} - -[ -n "${SNAPCRAFT_STAGE:-}" ] ||\ - die "must be sourced from a snapcraft config file" - -snap_yq_version=3.4.1 - -snap_common_install_yq() -{ - export yq="${SNAPCRAFT_STAGE}/bin/yq" - - local yq_pkg - yq_pkg="github.com/mikefarah/yq" - - local yq_url - yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}" - curl -o "${yq}" -L "${yq_url}" - chmod +x "${yq}" -} - -# Function that should be called for each snap "part" in -# snapcraft.yaml. -snap_common_main() -{ - # Architecture - arch="$(uname -m)" - - case "${arch}" in - aarch64) - goarch="arm64" - qemu_arch="${arch}" - ;; - - ppc64le) - goarch="ppc64le" - qemu_arch="ppc64" - ;; - - s390x) - goarch="${arch}" - qemu_arch="${arch}" - ;; - - x86_64) - goarch="amd64" - qemu_arch="${arch}" - ;; - - *) die "unsupported architecture: ${arch}" ;; - esac - - dpkg_arch=$(dpkg --print-architecture) - - # golang - # - # We need the O/S name in golang format, but since we don't - # know if the godeps part has run, we don't know if golang is - # available yet, hence fall back to a standard system command. - goos="$(go env GOOS &>/dev/null || true)" - [ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]') - - export GOROOT="${SNAPCRAFT_STAGE}" - export GOPATH="${GOROOT}/gopath" - export GO111MODULE="auto" - - mkdir -p "${GOPATH}/bin" - export PATH="${GOPATH}/bin:${PATH}" - - # Proxy - export http_proxy="${http_proxy:-}" - export https_proxy="${https_proxy:-}" - - # Binaries - mkdir -p "${SNAPCRAFT_STAGE}/bin" - - export PATH="$PATH:${SNAPCRAFT_STAGE}/bin" - - # YAML query tool - export yq="${SNAPCRAFT_STAGE}/bin/yq" - - # Kata paths - export kata_dir=$(printf "%s/src/github.com/%s/%s" \ - "${GOPATH}" \ - "${SNAPCRAFT_PROJECT_NAME}" \ - "${SNAPCRAFT_PROJECT_NAME}") - - export versions_file="${kata_dir}/versions.yaml" - - [ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq -} - -snap_common_main diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml deleted file mode 100644 index 4bbb8e0f1..000000000 --- a/snap/snapcraft.yaml +++ /dev/null @@ -1,170 +0,0 @@ -name: kata-containers -website: https://github.com/kata-containers/kata-containers -summary: Build lightweight VMs that seamlessly plug into the containers ecosystem -description: | - Kata Containers is an open source project and community working to build a - standard implementation of lightweight Virtual Machines (VMs) that feel and - perform like containers, but provide the workload isolation and security - advantages of VMs -confinement: classic -adopt-info: metadata -base: core20 - -parts: - metadata: - plugin: nil - prime: - - -* - build-packages: - - git - - git-extras - override-pull: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - version="9999" - - if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then - version=$(echo ${GITHUB_REF:-} | cut -d/ -f3) - git checkout ${version} - fi - - snapcraftctl set-grade "stable" - snapcraftctl set-version "${version}" - - mkdir -p $(dirname ${kata_dir}) - ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir} - - docker: - after: [metadata] - plugin: nil - prime: - - -* - build-packages: - - ca-certificates - - containerd - - curl - - gnupg - - lsb-release - - runc - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\ - sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - distro_codename=$(lsb_release -cs) - echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get -y update - sudo apt-get -y install docker-ce docker-ce-cli containerd.io - - echo "Unmasking docker service" - sudo -E systemctl unmask docker.service || true - sudo -E systemctl unmask docker.socket || true - echo "Adding $USER into docker group" - sudo -E gpasswd -a $USER docker - echo "Starting docker" - # docker may fail to start using "fd://" in docker.service - sudo sed -i 's/fd:\/\//unix:\/\//g' /lib/systemd/system/docker.service - sudo systemctl daemon-reload - sudo -E systemctl start docker || true - - image: - after: [docker] - plugin: nil - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make rootfs-image-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-image.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - - sudo -E NO_TTY=true make rootfs-initrd-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-rootfs-initrd.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - - runtime: - after: [docker] - plugin: nil - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make shim-v2-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-shim-v2.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - mkdir -p "${SNAPCRAFT_PART_INSTALL}/usr/bin" - ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/containerd-shim-kata-v2" "${SNAPCRAFT_PART_INSTALL}/usr/bin/containerd-shim-kata-v2" - ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-runtime" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-runtime" - ln -sf "${SNAPCRAFT_PART_INSTALL}/opt/kata/bin/kata-collect-data.sh" "${SNAPCRAFT_PART_INSTALL}/usr/bin/kata-collect-data.sh" - - kernel: - after: [docker] - plugin: nil - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make kernel-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - qemu: - plugin: make - after: [docker] - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make qemu-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-qemu.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - virtiofsd: - plugin: nil - after: [docker] - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make virtiofsd-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-virtiofsd.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - - cloud-hypervisor: - plugin: nil - after: [docker] - override-build: | - source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh" - - if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then - cd "${SNAPCRAFT_PROJECT_DIR}" - sudo -E NO_TTY=true make cloud-hypervisor-tarball - - tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz" - - tar -xvJpf "${tarfile}" -C "${SNAPCRAFT_PART_INSTALL}" - fi - -apps: - runtime: - command: usr/bin/kata-runtime - shim: - command: usr/bin/containerd-shim-kata-v2 - collect-data: - command: usr/bin/kata-collect-data.sh diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index b13e9eeb9..f902cd041 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -2146,6 +2146,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -2285,6 +2286,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -2645,6 +2647,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "objc" version = "0.2.7" @@ -3887,6 +3898,13 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] + [[package]] name = "salsa20" version = "0.10.2" @@ -4384,6 +4402,19 @@ dependencies = [ "slog-scope", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.20", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -4646,6 +4677,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", diff --git a/src/agent/rustjail/src/process.rs b/src/agent/rustjail/src/process.rs index 0e7fe73ef..cdecae130 100644 --- a/src/agent/rustjail/src/process.rs +++ b/src/agent/rustjail/src/process.rs @@ -161,7 +161,7 @@ impl Process { pub fn notify_term_close(&mut self) { let notify = self.term_exit_notifier.clone(); - notify.notify_one(); + notify.notify_waiters(); } pub fn close_stdin(&mut self) { diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index 042acd0ae..de131faf0 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -81,7 +81,8 @@ cfg_if! { // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00 pub const ACPI_DEV_PATH: &str = "/devices/LNXSYSTM"; -pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu"; +pub const SYSFS_CPU_PATH: &str = "/sys/devices/system/cpu"; +pub const SYSFS_CPU_ONLINE_PATH: &str = "/sys/devices/system/cpu/online"; pub const SYSFS_MEMORY_BLOCK_SIZE_PATH: &str = "/sys/devices/system/memory/block_size_bytes"; pub const SYSFS_MEMORY_HOTPLUG_PROBE_PATH: &str = "/sys/devices/system/memory/probe"; diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index f5e9d271b..280b0d95a 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -89,6 +89,27 @@ impl Handle { .await?; } + // we need to update the link's interface name, thus we should rename the existed link whose name + // is the same with the link's request name, otherwise, it would update the link failed with the + // name conflicted. + let mut new_link = None; + if link.name() != iface.name { + if let Ok(link) = self.find_link(LinkFilter::Name(iface.name.as_str())).await { + // update the existing interface name with a temporary name, otherwise + // it would failed to udpate this interface with an existing name. + let mut request = self.handle.link().set(link.index()); + request.message_mut().header = link.header.clone(); + + request + .name(format!("{}_temp", link.name())) + .up() + .execute() + .await?; + + new_link = Some(link); + } + } + // Update link let mut request = self.handle.link().set(link.index()); request.message_mut().header = link.header.clone(); @@ -101,6 +122,14 @@ impl Handle { .execute() .await?; + // swap the updated iface's name. + if let Some(nlink) = new_link { + let mut request = self.handle.link().set(nlink.index()); + request.message_mut().header = nlink.header.clone(); + + request.name(link.name()).up().execute().await?; + } + Ok(()) } diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index fd43baed1..9049cd5c2 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -673,15 +673,16 @@ impl AgentService { let cid = req.container_id; let eid = req.exec_id; - let mut term_exit_notifier = Arc::new(tokio::sync::Notify::new()); + let term_exit_notifier; let reader = { let s = self.sandbox.clone(); let mut sandbox = s.lock().await; let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?; + term_exit_notifier = p.term_exit_notifier.clone(); + if p.term_master.is_some() { - term_exit_notifier = p.term_exit_notifier.clone(); p.get_reader(StreamType::TermMaster) } else if stdout { if p.parent_stdout.is_some() { diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 9ed7f9720..24b678747 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -12,6 +12,7 @@ use crate::pci; use crate::uevent::{Uevent, UeventMatcher}; use crate::watcher::BindWatcher; use anyhow::{anyhow, Context, Result}; +use kata_types::cpu::CpuSet; use libc::pid_t; use oci::{Hook, Hooks}; use protocols::agent::OnlineCPUMemRequest; @@ -25,6 +26,7 @@ use std::collections::HashMap; use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use std::{thread, time}; use tokio::sync::mpsc::{channel, Receiver, Sender}; @@ -265,12 +267,12 @@ impl Sandbox { pub fn online_cpu_memory(&self, req: &OnlineCPUMemRequest) -> Result<()> { if req.nb_cpus > 0 { // online cpus - online_cpus(&self.logger, req.nb_cpus as i32)?; + online_cpus(&self.logger, req.nb_cpus as i32).context("online cpus")?; } if !req.cpu_only { // online memory - online_memory(&self.logger)?; + online_memory(&self.logger).context("online memory")?; } if req.nb_cpus == 0 { @@ -434,23 +436,33 @@ fn online_resources(logger: &Logger, path: &str, pattern: &str, num: i32) -> Res // max wait for all CPUs to online will use 50 * 100 = 5 seconds. const ONLINE_CPUMEM_WATI_MILLIS: u64 = 50; -const ONLINE_CPUMEM_MAX_RETRIES: u32 = 100; +const ONLINE_CPUMEM_MAX_RETRIES: i32 = 100; #[instrument] fn online_cpus(logger: &Logger, num: i32) -> Result { - let mut onlined_count: i32 = 0; + let mut onlined_cpu_count = onlined_cpus().context("onlined cpu count")?; + // for some vmms, like dragonball, they will online cpus for us + // so check first whether agent need to do the online operation + if onlined_cpu_count >= num { + return Ok(num); + } for i in 0..ONLINE_CPUMEM_MAX_RETRIES { - let r = online_resources( + // online num resources + online_resources( logger, - SYSFS_CPU_ONLINE_PATH, + SYSFS_CPU_PATH, r"cpu[0-9]+", - num - onlined_count, - ); + num - onlined_cpu_count, + ) + .context("online cpu resource")?; - onlined_count += r?; - if onlined_count == num { - info!(logger, "online {} CPU(s) after {} retries", num, i); + onlined_cpu_count = onlined_cpus().context("onlined cpu count")?; + if onlined_cpu_count >= num { + info!( + logger, + "Currently {} onlined CPU(s) after {} retries", onlined_cpu_count, i + ); return Ok(num); } thread::sleep(time::Duration::from_millis(ONLINE_CPUMEM_WATI_MILLIS)); @@ -465,10 +477,18 @@ fn online_cpus(logger: &Logger, num: i32) -> Result { #[instrument] fn online_memory(logger: &Logger) -> Result<()> { - online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1)?; + online_resources(logger, SYSFS_MEMORY_ONLINE_PATH, r"memory[0-9]+", -1) + .context("online memory resource")?; Ok(()) } +fn onlined_cpus() -> Result { + let content = + fs::read_to_string(SYSFS_CPU_ONLINE_PATH).context("read sysfs cpu online file")?; + let online_cpu_set = CpuSet::from_str(content.trim())?; + Ok(online_cpu_set.len() as i32) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock index c5be35bfb..0ed990c0b 100644 --- a/src/dragonball/Cargo.lock +++ b/src/dragonball/Cargo.lock @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -209,11 +209,12 @@ dependencies = [ [[package]] name = "dbs-address-space" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc" +checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322" dependencies = [ "arc-swap", + "lazy_static", "libc", "nix 0.23.2", "thiserror", @@ -247,9 +248,9 @@ dependencies = [ [[package]] name = "dbs-boot" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108" +checksum = "5466a92f75aa928a9103dcb2088f6d1638ef9da8945fad7389a73864dfa0182c" dependencies = [ "dbs-arch", "kvm-bindings", @@ -300,9 +301,9 @@ dependencies = [ [[package]] name = "dbs-upcall" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f" +checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb" dependencies = [ "anyhow", "dbs-utils", @@ -330,9 +331,9 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668" +checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8" dependencies = [ "byteorder", "caps", @@ -346,7 +347,7 @@ dependencies = [ "kvm-ioctls", "libc", "log", - "nix 0.23.2", + "nix 0.24.3", "nydus-api", "nydus-blobfs", "nydus-rafs", @@ -444,13 +445,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -482,7 +483,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -618,9 +619,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -670,7 +671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -841,7 +842,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1044,7 +1045,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1120,16 +1121,16 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1167,18 +1168,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" dependencies = [ "proc-macro2", "quote", @@ -1187,9 +1188,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1422,7 +1423,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1610,7 +1611,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -1619,13 +1629,28 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -1634,42 +1659,84 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "xattr" version = "0.2.3" diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 5036d7272..b0e03047d 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -12,16 +12,16 @@ edition = "2018" [dependencies] arc-swap = "1.5.0" bytes = "1.1.0" -dbs-address-space = "0.2.0" +dbs-address-space = "0.3.0" dbs-allocator = "0.1.0" dbs-arch = "0.2.0" -dbs-boot = "0.3.0" +dbs-boot = "0.4.0" dbs-device = "0.2.0" dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] } dbs-legacy-devices = "0.1.0" -dbs-upcall = { version = "0.2.0", optional = true } +dbs-upcall = { version = "0.3.0", optional = true } dbs-utils = "0.2.0" -dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] } +dbs-virtio-devices = { version = "0.3.1", optional = true, features = ["virtio-mmio"] } kvm-bindings = "0.6.0" kvm-ioctls = "0.12.0" lazy_static = "1.2" @@ -55,3 +55,5 @@ virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] # virtio-fs only work on atomic-guest-memory virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] +virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"] +virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"] diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index a271d04cb..247daf10d 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -19,6 +19,8 @@ use crate::vmm::Vmm; use self::VmConfigError::*; use self::VmmActionError::MachineConfig; +#[cfg(feature = "virtio-balloon")] +pub use crate::device_manager::balloon_dev_mgr::{BalloonDeviceConfigInfo, BalloonDeviceError}; #[cfg(feature = "virtio-blk")] pub use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, @@ -27,6 +29,8 @@ pub use crate::device_manager::blk_dev_mgr::{ pub use crate::device_manager::fs_dev_mgr::{ FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo, }; +#[cfg(feature = "virtio-mem")] +pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError}; #[cfg(feature = "virtio-net")] pub use crate::device_manager::virtio_net_dev_mgr::{ VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, @@ -34,7 +38,6 @@ pub use crate::device_manager::virtio_net_dev_mgr::{ }; #[cfg(feature = "virtio-vsock")] pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; - #[cfg(feature = "hotplug")] pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo}; @@ -97,6 +100,20 @@ pub enum VmmActionError { /// The action `ResizeVcpu` Failed #[error("vcpu resize error : {0}")] ResizeVcpu(#[source] VcpuResizeError), + + /// Cannot access address space. + #[error("Cannot access address space.")] + AddressSpaceNotInitialized, + + #[cfg(feature = "virtio-mem")] + /// Mem device related errors. + #[error("virtio-mem device error: {0}")] + Mem(#[source] MemDeviceError), + + #[cfg(feature = "virtio-balloon")] + /// Balloon device related errors. + #[error("virtio-balloon device error: {0}")] + Balloon(#[source] BalloonDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -172,6 +189,15 @@ pub enum VmmAction { #[cfg(feature = "hotplug")] /// Resize Vcpu number in the guest. ResizeVcpu(VcpuResizeInfo), + + #[cfg(feature = "virtio-mem")] + /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input. + InsertMemDevice(MemDeviceConfigInfo), + + #[cfg(feature = "virtio-balloon")] + /// Add a new balloon device or update one that already exists using the `BalloonDeviceConfig` + /// as input. + InsertBalloonDevice(BalloonDeviceConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -274,6 +300,12 @@ impl VmmService { } #[cfg(feature = "hotplug")] VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg), + #[cfg(feature = "virtio-mem")] + VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg), + #[cfg(feature = "virtio-balloon")] + VmmAction::InsertBalloonDevice(balloon_cfg) => { + self.add_balloon_device(vmm, event_mgr, balloon_cfg) + } }; debug!("send vmm response: {:?}", response); @@ -626,12 +658,6 @@ impl VmmService { #[cfg(feature = "hotplug")] fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult { - if !cfg!(target_arch = "x86_64") { - // TODO: Arm need to support vcpu hotplug. issue: #6010 - warn!("This arch do not support vm resize!"); - return Ok(VmmData::Empty); - } - if !cfg!(feature = "dbs-upcall") { warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature."); return Ok(VmmData::Empty); @@ -654,6 +680,62 @@ impl VmmService { Ok(VmmData::Empty) } + + #[cfg(feature = "virtio-mem")] + fn add_mem_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: MemDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; + + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady + } else { + VmmActionError::StartMicroVm(e) + } + })?; + + vm.device_manager_mut() + .mem_manager + .insert_or_update_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Mem) + } + + #[cfg(feature = "virtio-balloon")] + fn add_balloon_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: BalloonDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; + + if config.size_mib != 0 { + info!("add_balloon_device: wait prealloc"); + vm.stop_prealloc().map_err(VmmActionError::StartMicroVm)?; + } + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady + } else { + VmmActionError::StartMicroVm(e) + } + })?; + + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Balloon) + } } fn handle_cpu_topology( @@ -1462,4 +1544,84 @@ mod tests { t.check_request(); } } + + #[cfg(feature = "virtio-mem")] + #[test] + fn test_vmm_action_insert_mem_device() { + skip_if_not_root!(); + + let tests = &mut [ + // hotplug unready + TestData::new( + VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()), + InstanceState::Running, + &|result| { + assert!(matches!( + result, + Err(VmmActionError::StartMicroVm( + StartMicroVmError::UpcallMissVsock + )) + )); + let err_string = format!("{}", result.unwrap_err()); + let expected_err = String::from( + "failed to boot the VM: \ + the upcall client needs a virtio-vsock device for communication", + ); + assert_eq!(err_string, expected_err); + }, + ), + // success + TestData::new( + VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()), + InstanceState::Uninitialized, + &|result| { + assert!(result.is_ok()); + }, + ), + ]; + + for t in tests.iter_mut() { + t.check_request(); + } + } + + #[cfg(feature = "virtio-balloon")] + #[test] + fn test_vmm_action_insert_balloon_device() { + skip_if_not_root!(); + + let tests = &mut [ + // hotplug unready + TestData::new( + VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()), + InstanceState::Running, + &|result| { + assert!(matches!( + result, + Err(VmmActionError::StartMicroVm( + StartMicroVmError::UpcallMissVsock + )) + )); + let err_string = format!("{}", result.unwrap_err()); + let expected_err = String::from( + "failed to boot the VM: \ + the upcall client needs a virtio-vsock device for communication", + ); + assert_eq!(err_string, expected_err); + }, + ), + // success + TestData::new( + VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()), + InstanceState::Uninitialized, + &|result| { + assert!(result.is_ok()); + }, + ), + ]; + + for t in tests.iter_mut() { + t.check_request(); + } + } } diff --git a/src/dragonball/src/device_manager/balloon_dev_mgr.rs b/src/dragonball/src/device_manager/balloon_dev_mgr.rs new file mode 100644 index 000000000..b0ee2bd37 --- /dev/null +++ b/src/dragonball/src/device_manager/balloon_dev_mgr.rs @@ -0,0 +1,419 @@ +// Copyright 2020 Alibaba Cloud. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dbs_virtio_devices as virtio; +use serde_derive::{Deserialize, Serialize}; +use slog::{error, info}; +use virtio::balloon::{Balloon, BalloonConfig}; +use virtio::Error as VirtIoError; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::DbsMmioV2Device; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = false; + +/// Errors associated with `BalloonDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum BalloonDeviceError { + /// The balloon device was already used. + #[error("the virtio-balloon ID was already added to a different device")] + BalloonDeviceAlreadyExists, + + /// Cannot perform the requested operation after booting the microVM. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// guest memory error + #[error("failed to access guest memory, {0}")] + GuestMemoryError(#[source] vm_memory::mmap::Error), + + /// create balloon device error + #[error("failed to create virtio-balloon device, {0}")] + CreateBalloonDevice(#[source] virtio::Error), + + /// hotplug balloon device error + #[error("cannot hotplug virtio-balloon device, {0}")] + HotplugDeviceFailed(#[source] DeviceMgrError), + + /// create mmio device error + #[error("cannot create virtio-balloon mmio device, {0}")] + CreateMmioDevice(#[source] DeviceMgrError), + + /// Cannot initialize a balloon device or add a device to the MMIO Bus. + #[error("failure while registering balloon device: {0}")] + RegisterBalloonDevice(#[source] DeviceMgrError), + + /// resize balloon device error + #[error("failure while resizing virtio-balloon device, {0}")] + ResizeFailed(#[source] VirtIoError), + + /// The balloon device id doesn't exist. + #[error("invalid balloon device id '{0}'")] + InvalidDeviceId(String), + + /// balloon device does not exist + #[error("balloon device does not exist")] + NotExist, + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a virtio-balloon device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +pub struct BalloonDeviceConfigInfo { + /// Unique identifier of the balloon device + pub balloon_id: String, + /// Resize balloon size in mib + pub size_mib: u64, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, + /// VIRTIO_BALLOON_F_DEFLATE_ON_OOM + pub f_deflate_on_oom: bool, + /// VIRTIO_BALLOON_F_REPORTING + pub f_reporting: bool, +} + +impl ConfigItem for BalloonDeviceConfigInfo { + type Err = BalloonDeviceError; + + fn id(&self) -> &str { + &self.balloon_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> { + if self.balloon_id.as_str() == other.balloon_id.as_str() { + Err(BalloonDeviceError::BalloonDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Balloon Device Info +pub type BalloonDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for BalloonDeviceInfo { + type Err = BalloonDeviceError; + + fn id(&self) -> &str { + &self.config.balloon_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> { + if self.config.balloon_id.as_str() == other.config.balloon_id.as_str() { + Err(BalloonDeviceError::BalloonDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Balloon Devices Configs +#[derive(Clone)] +pub struct BalloonDeviceMgr { + /// A list of `BalloonDeviceConfig` objects. + info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl BalloonDeviceMgr { + /// Inserts `balloon_cfg` in the virtio-balloon device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + pub fn insert_or_update_device( + &mut self, + mut ctx: DeviceOpContext, + balloon_cfg: BalloonDeviceConfigInfo, + ) -> std::result::Result<(), BalloonDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + error!(ctx.logger(), "hotplug feature has been disabled."; + "subsystem" => "balloon_dev_mgr",); + return Err(BalloonDeviceError::UpdateNotAllowedPostBoot); + } + + let epoll_mgr = ctx + .get_epoll_mgr() + .map_err(BalloonDeviceError::DeviceManager)?; + + // If the id of the drive already exists in the list, the operation is update. + if let Some(index) = self.get_index_of_balloon_dev(&balloon_cfg.balloon_id) { + // Update an existing balloon device + if ctx.is_hotplug { + info!(ctx.logger(), "resize virtio balloon size to {:?}", balloon_cfg.size_mib; "subsystem" => "balloon_dev_mgr"); + self.update_balloon_size(index, balloon_cfg.size_mib)?; + } + self.info_list.insert_or_update(&balloon_cfg)?; + } else { + // Create a new balloon device + if !self.info_list.is_empty() { + error!(ctx.logger(), "only support one balloon device!"; "subsystem" => "balloon_dev_mgr"); + return Err(BalloonDeviceError::BalloonDeviceAlreadyExists); + } + + if !ctx.is_hotplug { + self.info_list.insert_or_update(&balloon_cfg)?; + return Ok(()); + } + + info!(ctx.logger(), "hotplug balloon device: {}", balloon_cfg.balloon_id; "subsystem" => "balloon_dev_mgr"); + let device = Box::new( + virtio::balloon::Balloon::new( + epoll_mgr, + BalloonConfig { + f_deflate_on_oom: balloon_cfg.f_deflate_on_oom, + f_reporting: balloon_cfg.f_reporting, + }, + ) + .map_err(BalloonDeviceError::CreateBalloonDevice)?, + ); + + let mmio_dev = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + device, + &mut ctx, + balloon_cfg.use_shared_irq.unwrap_or(self.use_shared_irq), + balloon_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BalloonDeviceError::CreateMmioDevice)?; + ctx.insert_hotplug_mmio_device(&mmio_dev, None) + .map_err(|e| { + error!( + ctx.logger(), + "hotplug balloon device {} error: {}", + &balloon_cfg.balloon_id, e; + "subsystem" => "balloon_dev_mgr" + ); + BalloonDeviceError::HotplugDeviceFailed(e) + })?; + let index = self.info_list.insert_or_update(&balloon_cfg)?; + self.info_list[index].set_device(mmio_dev); + } + Ok(()) + } + + /// Attaches all virtio-balloon devices from the BalloonDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), BalloonDeviceError> { + let epoll_mgr = ctx + .get_epoll_mgr() + .map_err(BalloonDeviceError::DeviceManager)?; + + for info in self.info_list.iter_mut() { + info!(ctx.logger(), "attach balloon device: {}", info.config.balloon_id; "subsystem" => "balloon_dev_mgr"); + + let device = Balloon::new( + epoll_mgr.clone(), + BalloonConfig { + f_deflate_on_oom: info.config.f_deflate_on_oom, + f_reporting: info.config.f_reporting, + }, + ) + .map_err(BalloonDeviceError::CreateBalloonDevice)?; + let mmio_dev = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BalloonDeviceError::RegisterBalloonDevice)?; + info.set_device(mmio_dev); + } + + Ok(()) + } + + fn update_balloon_size( + &self, + index: usize, + size_mib: u64, + ) -> std::result::Result<(), BalloonDeviceError> { + let device = self.info_list[index] + .device + .as_ref() + .ok_or_else(|| BalloonDeviceError::NotExist)?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(balloon_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return balloon_dev + .set_size(size_mib) + .map_err(BalloonDeviceError::ResizeFailed); + } + } + Ok(()) + } + + fn get_index_of_balloon_dev(&self, balloon_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.balloon_id.eq(balloon_id)) + } +} + +impl Default for BalloonDeviceMgr { + /// Create a new `BalloonDeviceMgr` object.. + fn default() -> Self { + BalloonDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + impl Default for BalloonDeviceConfigInfo { + fn default() -> Self { + BalloonDeviceConfigInfo { + balloon_id: "".to_string(), + size_mib: 0, + use_generic_irq: None, + use_shared_irq: None, + f_deflate_on_oom: false, + f_reporting: false, + } + } + } + + #[test] + fn test_balloon_config_check_conflicts() { + let config = BalloonDeviceConfigInfo::default(); + let mut config2 = BalloonDeviceConfigInfo::default(); + assert!(config.check_conflicts(&config2).is_err()); + config2.balloon_id = "dummy_balloon".to_string(); + assert!(config.check_conflicts(&config2).is_ok()); + } + + #[test] + fn test_create_balloon_devices_configs() { + let mgr = BalloonDeviceMgr::default(); + assert_eq!(mgr.info_list.len(), 0); + assert_eq!(mgr.get_index_of_balloon_dev(""), None); + } + + #[test] + fn test_balloon_insert_or_update_device() { + //Init vm for test. + let mut vm = create_vm_for_test(); + + // Test for standard config + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + } + + #[test] + fn test_balloon_attach_device() { + //Init vm and insert balloon config for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + // Test for standard config + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + assert!(vm + .device_manager_mut() + .balloon_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + } + + #[test] + fn test_balloon_update_device() { + //Init vm for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + assert!(vm + .device_manager_mut() + .balloon_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + assert!(vm + .device_manager() + .balloon_manager + .update_balloon_size(0, 200) + .is_ok()); + } +} diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index f7cdfa9b0..0fe10cf24 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -871,6 +871,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), None, false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); let dummy_file = TempFile::new().unwrap(); @@ -907,6 +909,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), None, false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); vm.device_manager_mut() diff --git a/src/dragonball/src/device_manager/mem_dev_mgr.rs b/src/dragonball/src/device_manager/mem_dev_mgr.rs new file mode 100644 index 000000000..2bb68ae80 --- /dev/null +++ b/src/dragonball/src/device_manager/mem_dev_mgr.rs @@ -0,0 +1,733 @@ +// Copyright 2020 Alibaba Cloud. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::io; +use std::sync::{Arc, Mutex}; + +use dbs_address_space::{ + AddressSpace, AddressSpaceError, AddressSpaceRegion, MPOL_MF_MOVE, MPOL_PREFERRED, USABLE_END, +}; +use dbs_utils::epoll_manager::EpollManager; +use dbs_virtio_devices as virtio; +use kvm_bindings::kvm_userspace_memory_region; +use kvm_ioctls::VmFd; +use nix::sys::mman; +use serde_derive::{Deserialize, Serialize}; +use slog::{debug, error, info, warn}; +use virtio::mem::{Mem, MemRegionFactory}; +use virtio::Error as VirtIoError; +use vm_memory::{ + Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestRegionMmap, GuestUsize, MmapRegion, +}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::DbsMmioV2Device; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::vm::VmConfigInfo; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = false; + +const HUGE_PAGE_2M: usize = 0x200000; + +// max numa node ids on host +const MAX_NODE: u32 = 64; + +/// Errors associated with `MemDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum MemDeviceError { + /// The mem device was already used. + #[error("the virtio-mem ID was already added to a different device")] + MemDeviceAlreadyExists, + + /// Cannot perform the requested operation after booting the microVM. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// insert mem device error + #[error("cannot add virtio-mem device, {0}")] + InsertDeviceFailed(#[source] DeviceMgrError), + + /// create mem device error + #[error("cannot create virito-mem device, {0}")] + CreateMemDevice(#[source] DeviceMgrError), + + /// create mmio device error + #[error("cannot create virito-mem mmio device, {0}")] + CreateMmioDevice(#[source] DeviceMgrError), + + /// resize mem device error + #[error("failure while resizing virtio-mem device, {0}")] + ResizeFailed(#[source] VirtIoError), + + /// mem device does not exist + #[error("mem device does not exist")] + DeviceNotExist, + + /// address space region error + #[error("address space region error, {0}")] + AddressSpaceRegion(#[source] AddressSpaceError), + + /// Cannot initialize a mem device or add a device to the MMIO Bus. + #[error("failure while registering mem device: {0}")] + RegisterMemDevice(#[source] DeviceMgrError), + + /// The mem device id doesn't exist. + #[error("invalid mem device id '{0}'")] + InvalidDeviceId(String), + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a virtio-mem device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +pub struct MemDeviceConfigInfo { + /// Unique identifier of the pmem device + pub mem_id: String, + /// Memory size mib + pub size_mib: u64, + /// Memory capacity mib + pub capacity_mib: u64, + /// Use multi_region or not + pub multi_region: bool, + /// host numa node id + pub host_numa_node_id: Option, + /// guest numa node id + pub guest_numa_node_id: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl ConfigItem for MemDeviceConfigInfo { + type Err = MemDeviceError; + + fn id(&self) -> &str { + &self.mem_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> { + if self.mem_id.as_str() == other.mem_id.as_str() { + Err(MemDeviceError::MemDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Mem Device Info +pub type MemDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for MemDeviceInfo { + type Err = MemDeviceError; + + fn id(&self) -> &str { + &self.config.mem_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> { + if self.config.mem_id.as_str() == other.config.mem_id.as_str() { + Err(MemDeviceError::MemDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Mem Devices Configs +#[derive(Clone)] +pub struct MemDeviceMgr { + /// A list of `MemDeviceConfig` objects. + info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl MemDeviceMgr { + /// Inserts `mem_cfg` in the virtio-mem device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + pub fn insert_or_update_device( + &mut self, + mut ctx: DeviceOpContext, + mem_cfg: MemDeviceConfigInfo, + ) -> std::result::Result<(), MemDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + error!(ctx.logger(), "hotplug feature has been disabled."; + "subsystem" => "virito-mem"); + return Err(MemDeviceError::UpdateNotAllowedPostBoot); + } + + let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?; + + // If the id of the drive already exists in the list, the operation is update. + if let Some(index) = self.get_index_of_mem_dev(&mem_cfg.mem_id) { + // Update an existing memory device + if ctx.is_hotplug { + info!( + ctx.logger(), + "update memory device: {}, size: 0x{:x}MB.", + mem_cfg.mem_id, + mem_cfg.size_mib; + "subsystem" => "virito-mem" + ); + self.update_memory_size(index, mem_cfg.size_mib)?; + } + self.info_list.insert_or_update(&mem_cfg)?; + } else { + // Create a new memory device + if !ctx.is_hotplug { + self.info_list.insert_or_update(&mem_cfg)?; + return Ok(()); + } + + info!( + ctx.logger(), + "hot-add memory device: {}, size: 0x{:x}MB.", mem_cfg.mem_id, mem_cfg.size_mib; + "subsystem" => "virito-mem" + ); + + let device = Self::create_memory_device(&mem_cfg, &ctx, &epoll_mgr) + .map_err(MemDeviceError::CreateMemDevice)?; + let mmio_device = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + &mut ctx, + mem_cfg.use_shared_irq.unwrap_or(self.use_shared_irq), + mem_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(MemDeviceError::CreateMmioDevice)?; + + #[cfg(not(test))] + ctx.insert_hotplug_mmio_device(&mmio_device, None) + .map_err(|e| { + error!( + ctx.logger(), + "failed to hot-add virtio-mem device {}, {}", &mem_cfg.mem_id, e; + "subsystem" => "virito-mem" + ); + MemDeviceError::InsertDeviceFailed(e) + })?; + + let index = self.info_list.insert_or_update(&mem_cfg)?; + self.info_list[index].set_device(mmio_device); + } + + Ok(()) + } + + /// Attaches all virtio-mem devices from the MemDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), MemDeviceError> { + let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?; + + for info in self.info_list.iter_mut() { + let config = &info.config; + info!( + ctx.logger(), + "attach virtio-mem device {}, size 0x{:x}.", config.mem_id, config.size_mib; + "subsystem" => "virito-mem" + ); + // Ignore virtio-mem device with zero memory capacity. + if config.size_mib == 0 { + debug!( + ctx.logger(), + "ignore zero-sizing memory device {}.", config.mem_id; + "subsystem" => "virito-mem" + ); + continue; + } + + let device = Self::create_memory_device(config, ctx, &epoll_mgr) + .map_err(MemDeviceError::CreateMemDevice)?; + let mmio_device = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + ctx, + config.use_shared_irq.unwrap_or(self.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(MemDeviceError::RegisterMemDevice)?; + + info.set_device(mmio_device); + } + + Ok(()) + } + + fn get_index_of_mem_dev(&self, mem_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.mem_id.eq(mem_id)) + } + + fn create_memory_device( + config: &MemDeviceConfigInfo, + ctx: &DeviceOpContext, + epoll_mgr: &EpollManager, + ) -> std::result::Result, DeviceMgrError> { + let factory = Arc::new(Mutex::new(MemoryRegionFactory::new( + ctx, + config.mem_id.clone(), + config.host_numa_node_id, + )?)); + + let mut capacity_mib = config.capacity_mib; + if capacity_mib == 0 { + capacity_mib = *USABLE_END >> 20; + } + // get boot memory size for calculate alignment + let boot_mem_size = { + let boot_size = (ctx.get_vm_config()?.mem_size_mib << 20) as u64; + // increase 1G memory because of avoiding mmio hole + match boot_size { + x if x > dbs_boot::layout::MMIO_LOW_START => x + (1 << 30), + _ => boot_size, + } + }; + + virtio::mem::Mem::new( + config.mem_id.clone(), + capacity_mib, + config.size_mib, + config.multi_region, + config.guest_numa_node_id, + epoll_mgr.clone(), + factory, + boot_mem_size, + ) + .map_err(DeviceMgrError::Virtio) + } + + /// Removes all virtio-mem devices + pub fn remove_devices(&self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + for info in self.info_list.iter() { + if let Some(device) = &info.device { + DeviceManager::destroy_mmio_virtio_device(device.clone(), ctx)?; + } + } + + Ok(()) + } + + fn update_memory_size( + &self, + index: usize, + size_mib: u64, + ) -> std::result::Result<(), MemDeviceError> { + let device = self.info_list[index] + .device + .as_ref() + .ok_or_else(|| MemDeviceError::DeviceNotExist)?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(mem_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return mem_dev + .set_requested_size(size_mib) + .map_err(MemDeviceError::ResizeFailed); + } + } + Ok(()) + } +} + +impl Default for MemDeviceMgr { + /// Create a new `MemDeviceMgr` object.. + fn default() -> Self { + MemDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +struct MemoryRegionFactory { + mem_id: String, + vm_as: GuestAddressSpaceImpl, + address_space: AddressSpace, + vm_config: VmConfigInfo, + vm_fd: Arc, + logger: Arc, + host_numa_node_id: Option, + instance_id: String, +} + +impl MemoryRegionFactory { + fn new( + ctx: &DeviceOpContext, + mem_id: String, + host_numa_node_id: Option, + ) -> Result { + let vm_as = ctx.get_vm_as()?; + let address_space = ctx.get_address_space()?; + let vm_config = ctx.get_vm_config()?; + let logger = Arc::new(ctx.logger().new(slog::o!())); + + let shared_info = ctx.shared_info.read().unwrap(); + let instance_id = shared_info.id.clone(); + + Ok(MemoryRegionFactory { + mem_id, + vm_as, + address_space, + vm_config, + vm_fd: ctx.vm_fd.clone(), + logger, + host_numa_node_id, + instance_id, + }) + } + + fn configure_anon_mem(&self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> { + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_DONTFORK, + ) + } + .map_err(VirtIoError::Madvise)?; + + Ok(()) + } + + fn configure_numa(&self, mmap_reg: &MmapRegion, node_id: u32) -> Result<(), VirtIoError> { + let nodemask = 1_u64 + .checked_shl(node_id) + .ok_or(VirtIoError::InvalidInput)?; + let res = unsafe { + libc::syscall( + libc::SYS_mbind, + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + MPOL_PREFERRED, + &nodemask as *const u64, + MAX_NODE, + MPOL_MF_MOVE, + ) + }; + if res < 0 { + warn!( + self.logger, + "failed to mbind memory to host_numa_node_id {}: this may affect performance", + node_id; + "subsystem" => "virito-mem" + ); + } + Ok(()) + } + + fn configure_thp(&mut self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> { + debug!( + self.logger, + "Setting MADV_HUGEPAGE on AddressSpaceRegion addr {:x?} len {:x?}", + mmap_reg.as_ptr(), + mmap_reg.size(); + "subsystem" => "virito-mem" + ); + + // Safe because we just create the MmapRegion + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_HUGEPAGE, + ) + } + .map_err(VirtIoError::Madvise)?; + + Ok(()) + } + + fn map_to_kvm( + &mut self, + slot: u32, + reg: &Arc, + mmap_reg: &MmapRegion, + ) -> Result<(), VirtIoError> { + let host_addr = mmap_reg.as_ptr() as u64; + + let flags = 0u32; + + let mem_region = kvm_userspace_memory_region { + slot, + guest_phys_addr: reg.start_addr().raw_value(), + memory_size: reg.len(), + userspace_addr: host_addr, + flags, + }; + + // Safe because the user mem region is just created, and kvm slot is allocated + // by resource allocator. + unsafe { self.vm_fd.set_user_memory_region(mem_region) } + .map_err(VirtIoError::SetUserMemoryRegion)?; + + Ok(()) + } +} + +impl MemRegionFactory for MemoryRegionFactory { + fn create_region( + &mut self, + guest_addr: GuestAddress, + region_len: GuestUsize, + kvm_slot: u32, + ) -> std::result::Result, VirtIoError> { + // create address space region + let mem_type = self.vm_config.mem_type.as_str(); + let mut mem_file_path = self.vm_config.mem_file_path.clone(); + let mem_file_name = format!( + "/virtiomem_{}_{}", + self.instance_id.as_str(), + self.mem_id.as_str() + ); + mem_file_path.push_str(mem_file_name.as_str()); + let region = Arc::new( + AddressSpaceRegion::create_default_memory_region( + guest_addr, + region_len, + self.host_numa_node_id, + mem_type, + mem_file_path.as_str(), + false, + true, + ) + .map_err(|e| { + error!(self.logger, "failed to insert address space region: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + guest_addr.0, region_len + ), + )) + })?, + ); + info!( + self.logger, + "VM: mem_type: {} mem_file_path: {}, numa_node_id: {:?} file_offset: {:?}", + mem_type, + mem_file_path, + self.host_numa_node_id, + region.file_offset(); + "subsystem" => "virito-mem" + ); + + let mmap_region = MmapRegion::build( + region.file_offset().cloned(), + region_len as usize, + region.prot_flags(), + region.perm_flags(), + ) + .map_err(VirtIoError::NewMmapRegion)?; + let host_addr: u64 = mmap_region.as_ptr() as u64; + + // thp + if mem_type == "hugeanon" || mem_type == "hugeshmem" { + self.configure_thp(&mmap_region)?; + } + + // Handle numa + if let Some(numa_node_id) = self.host_numa_node_id { + self.configure_numa(&mmap_region, numa_node_id)?; + } + + // add to guest memory mapping + self.map_to_kvm(kvm_slot, ®ion, &mmap_region)?; + + info!( + self.logger, + "kvm set user memory region: slot: {}, flags: {}, guest_phys_addr: {:X}, memory_size: {}, userspace_addr: {:X}", + kvm_slot, + 0, + guest_addr.raw_value(), + region_len, + host_addr; + "subsystem" => "virito-mem" + ); + + // All value should be valid. + let memory_region = Arc::new( + GuestRegionMmap::new(mmap_region, guest_addr).map_err(VirtIoError::InsertMmap)?, + ); + + let vm_as_new = self + .vm_as + .memory() + .insert_region(memory_region.clone()) + .map_err(VirtIoError::InsertMmap)?; + self.vm_as.lock().unwrap().replace(vm_as_new); + self.address_space.insert_region(region).map_err(|e| { + error!(self.logger, "failed to insert address space region: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + guest_addr.0, region_len + ), + )) + })?; + + Ok(memory_region) + } + + fn restore_region_addr( + &self, + guest_addr: GuestAddress, + ) -> std::result::Result<*mut u8, VirtIoError> { + let memory = self.vm_as.memory(); + // NOTE: We can't clone `GuestRegionMmap` reference directly!!! + // + // Since an important role of the member `mapping` (type is + // `MmapRegion`) in `GuestRegionMmap` is to mmap the memory during + // construction and munmap the memory during drop. However, when the + // life time of cloned data is over, the drop operation will be + // performed, which will munmap the origional mmap memory, which will + // cause some memory in dragonall to be inaccessable. And remember the + // data structure that was cloned is still alive now, when its life time + // is over, it will perform the munmap operation again, which will cause + // a memory exception! + memory + .get_host_address(guest_addr) + .map_err(VirtIoError::GuestMemory) + } + + fn get_host_numa_node_id(&self) -> Option { + self.host_numa_node_id + } + + fn set_host_numa_node_id(&mut self, host_numa_node_id: Option) { + self.host_numa_node_id = host_numa_node_id; + } +} + +#[cfg(test)] +mod tests { + use vm_memory::GuestMemoryRegion; + + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + impl Default for MemDeviceConfigInfo { + fn default() -> Self { + MemDeviceConfigInfo { + mem_id: "".to_string(), + size_mib: 0, + capacity_mib: 1024, + multi_region: true, + host_numa_node_id: None, + guest_numa_node_id: None, + use_generic_irq: None, + use_shared_irq: None, + } + } + } + + #[test] + fn test_mem_config_check_conflicts() { + let config = MemDeviceConfigInfo::default(); + let mut config2 = MemDeviceConfigInfo::default(); + assert!(config.check_conflicts(&config2).is_err()); + config2.mem_id = "dummy_mem".to_string(); + assert!(config.check_conflicts(&config2).is_ok()); + } + + #[test] + fn test_create_mem_devices_configs() { + let mgr = MemDeviceMgr::default(); + assert_eq!(mgr.info_list.len(), 0); + assert_eq!(mgr.get_index_of_mem_dev(""), None); + } + + #[test] + fn test_mem_insert_or_update_device() { + // Init vm for test. + let mut vm = create_vm_for_test(); + + // We don't need to use virtio-mem before start vm + // Test for standard config with hotplug + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + true, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + + let dummy_mem_device = MemDeviceConfigInfo::default(); + vm.device_manager_mut() + .mem_manager + .insert_or_update_device(device_op_ctx, dummy_mem_device) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + } + + #[test] + fn test_mem_attach_device() { + // Init vm and insert mem config for test. + let mut vm = create_vm_for_test(); + let dummy_mem_device = MemDeviceConfigInfo::default(); + vm.device_manager_mut() + .mem_manager + .info_list + .insert_or_update(&dummy_mem_device) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + + // Test for standard config + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + false, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + vm.device_manager_mut() + .mem_manager + .attach_devices(&mut device_op_ctx) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + } + + #[test] + fn test_mem_create_region() { + let vm = create_vm_for_test(); + let ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + true, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + let mem_id = String::from("mem0"); + let guest_addr = GuestAddress(0x1_0000_0000); + let region_len = 0x1000_0000; + let kvm_slot = 2; + + // no vfio manager, no numa node + let mut factory = MemoryRegionFactory::new(&ctx, mem_id, None).unwrap(); + let region_opt = factory.create_region(guest_addr, region_len, kvm_slot); + assert_eq!(region_opt.unwrap().len(), region_len); + } +} diff --git a/src/dragonball/src/device_manager/memory_region_handler.rs b/src/dragonball/src/device_manager/memory_region_handler.rs index 2be149ef9..fbf5aa20c 100644 --- a/src/dragonball/src/device_manager/memory_region_handler.rs +++ b/src/dragonball/src/device_manager/memory_region_handler.rs @@ -55,6 +55,7 @@ impl DeviceVirtioRegionHandler { None, file_offset, region.flags(), + region.prot(), false, )); diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 766c5eef9..b007270c9 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::io; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; @@ -45,9 +45,10 @@ use dbs_upcall::{ use dbs_virtio_devices::vsock::backend::VsockInnerConnector; use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::api::v1::InstanceInfo; use crate::error::StartMicroVmError; use crate::resource_manager::ResourceManager; -use crate::vm::{KernelConfigInfo, Vm}; +use crate::vm::{KernelConfigInfo, Vm, VmConfigInfo}; use crate::IoManagerCached; /// Virtual machine console device manager. @@ -89,6 +90,18 @@ mod memory_region_handler; #[cfg(feature = "virtio-fs")] pub use self::memory_region_handler::*; +#[cfg(feature = "virtio-mem")] +/// Device manager for virtio-mem devices. +pub mod mem_dev_mgr; +#[cfg(feature = "virtio-mem")] +use self::mem_dev_mgr::MemDeviceMgr; + +#[cfg(feature = "virtio-balloon")] +/// Device manager for virtio-balloon devices. +pub mod balloon_dev_mgr; +#[cfg(feature = "virtio-balloon")] +use self::balloon_dev_mgr::BalloonDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -248,6 +261,8 @@ pub struct DeviceOpContext { upcall_client: Option>>, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec>, + vm_config: Option, + shared_info: Arc>, } impl DeviceOpContext { @@ -257,6 +272,8 @@ impl DeviceOpContext { vm_as: Option, address_space: Option, is_hotplug: bool, + vm_config: Option, + shared_info: Arc>, ) -> Self { let irq_manager = device_mgr.irq_manager.clone(); let res_manager = device_mgr.res_manager.clone(); @@ -282,11 +299,21 @@ impl DeviceOpContext { upcall_client: None, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec::new(), + vm_config, + shared_info, } } pub(crate) fn create_boot_ctx(vm: &Vm, epoll_mgr: Option) -> Self { - Self::new(epoll_mgr, vm.device_manager(), None, None, false) + Self::new( + epoll_mgr, + vm.device_manager(), + None, + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ) } pub(crate) fn get_vm_as(&self) -> Result { @@ -296,6 +323,27 @@ impl DeviceOpContext { } } + pub(crate) fn get_vm_config(&self) -> Result { + match self.vm_config.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn get_address_space(&self) -> Result { + match self.address_space.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn get_epoll_mgr(&self) -> Result { + match self.epoll_mgr.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + pub(crate) fn logger(&self) -> &slog::Logger { &self.logger } @@ -386,6 +434,8 @@ impl DeviceOpContext { Some(vm_as), vm.vm_address_space().cloned(), true, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); ctx.upcall_client = vm.upcall_client().clone(); ctx @@ -463,7 +513,7 @@ pub struct DeviceManager { res_manager: Arc, vm_fd: Arc, pub(crate) logger: slog::Logger, - + pub(crate) shared_info: Arc>, pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, #[cfg(target_arch = "aarch64")] @@ -481,6 +531,12 @@ pub struct DeviceManager { #[cfg(feature = "virtio-fs")] fs_manager: Arc>, + + #[cfg(feature = "virtio-mem")] + pub(crate) mem_manager: MemDeviceMgr, + + #[cfg(feature = "virtio-balloon")] + pub(crate) balloon_manager: BalloonDeviceMgr, } impl DeviceManager { @@ -490,6 +546,7 @@ impl DeviceManager { res_manager: Arc, epoll_manager: EpollManager, logger: &slog::Logger, + shared_info: Arc>, ) -> Self { DeviceManager { io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))), @@ -498,6 +555,7 @@ impl DeviceManager { res_manager, vm_fd, logger: logger.new(slog::o!()), + shared_info, con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, @@ -511,6 +569,10 @@ impl DeviceManager { virtio_net_manager: VirtioNetDeviceMgr::default(), #[cfg(feature = "virtio-fs")] fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), + #[cfg(feature = "virtio-mem")] + mem_manager: MemDeviceMgr::default(), + #[cfg(feature = "virtio-balloon")] + balloon_manager: BalloonDeviceMgr::default(), } } @@ -636,9 +698,9 @@ impl DeviceManager { vm_as: GuestAddressSpaceImpl, epoll_mgr: EpollManager, kernel_config: &mut KernelConfigInfo, - com1_sock_path: Option, dmesg_fifo: Option>, address_space: Option<&AddressSpace>, + vm_config: &VmConfigInfo, ) -> std::result::Result<(), StartMicroVmError> { let mut ctx = DeviceOpContext::new( Some(epoll_mgr), @@ -646,8 +708,12 @@ impl DeviceManager { Some(vm_as), address_space.cloned(), false, + Some(vm_config.clone()), + self.shared_info.clone(), ); + let com1_sock_path = vm_config.serial_path.clone(); + self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; @@ -710,6 +776,8 @@ impl DeviceManager { Some(vm_as), address_space.cloned(), true, + None, + self.shared_info.clone(), ); #[cfg(feature = "virtio-blk")] @@ -874,6 +942,24 @@ impl DeviceManager { ) } + /// Create an Virtio MMIO transport layer device for the virtio backend device with configure + /// change notification enabled. + pub fn create_mmio_virtio_device_with_device_change_notification( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + let features = DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY; + DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(features), + use_shared_irq, + use_generic_irq, + ) + } + /// Create an Virtio MMIO transport layer device for the virtio backend device with specified /// features. pub fn create_mmio_virtio_device_with_features( @@ -1019,6 +1105,7 @@ mod tests { use vm_memory::{GuestAddress, MmapRegion}; use super::*; + #[cfg(target_arch = "x86_64")] use crate::vm::CpuTopology; impl DeviceManager { @@ -1029,6 +1116,10 @@ mod tests { let epoll_manager = EpollManager::default(); let res_manager = Arc::new(ResourceManager::new(None)); let logger = slog_scope::logger().new(slog::o!()); + let shared_info = Arc::new(RwLock::new(InstanceInfo::new( + String::from("dragonball"), + String::from("1"), + ))); DeviceManager { vm_fd: Arc::clone(&vm_fd), @@ -1047,10 +1138,15 @@ mod tests { virtio_net_manager: VirtioNetDeviceMgr::default(), #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), + #[cfg(feature = "virtio-mem")] + mem_manager: MemDeviceMgr::default(), + #[cfg(feature = "virtio-balloon")] + balloon_manager: BalloonDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), logger, + shared_info, } } } @@ -1090,7 +1186,7 @@ mod tests { }, vpmu_feature: 0, }; - vm.set_vm_config(vm_config); + vm.set_vm_config(vm_config.clone()); vm.init_guest_memory().unwrap(); vm.setup_interrupt_controller().unwrap(); let vm_as = vm.vm_as().cloned().unwrap(); @@ -1116,8 +1212,8 @@ mod tests { event_mgr.epoll_manager(), &mut cmdline, None, - None, address_space.as_ref(), + &vm_config, ) .unwrap(); let guard = mgr.io_manager.load(); @@ -1141,6 +1237,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), vm.vm_address_space().cloned(), true, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); let guest_addr = GuestAddress(0x200000000000); diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 7ac725209..4682a37bd 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -192,6 +192,11 @@ pub enum StartMicroVmError { /// Virtio-fs errors. #[error("virtio-fs errors: {0}")] FsDeviceError(#[source] device_manager::fs_dev_mgr::FsDeviceError), + + #[cfg(feature = "virtio-balloon")] + /// Virtio-balloon errors. + #[error("virtio-balloon errors: {0}")] + BalloonDeviceError(#[source] device_manager::balloon_dev_mgr::BalloonDeviceError), } /// Errors associated with starting the instance. diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 8f39af519..41200d010 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -374,11 +374,17 @@ impl VcpuManager { entry_addr: GuestAddress, ) -> Result<()> { info!("create boot vcpus"); - self.create_vcpus( - self.vcpu_config.boot_vcpu_count, - Some(request_ts), - Some(entry_addr), - )?; + let boot_vcpu_count = if cfg!(target_arch = "aarch64") { + // On aarch64, kvm doesn't allow to call KVM_CREATE_VCPU ioctl after vm has been booted + // because of vgic check. To support vcpu hotplug/hotunplug feature, we should create + // all the vcpufd at booting procedure. + // SetVmConfiguration API will ensure max_vcpu_count >= boot_vcpu_count, so it is safe + // to directly use max_vcpu_count here. + self.vcpu_config.max_vcpu_count + } else { + self.vcpu_config.boot_vcpu_count + }; + self.create_vcpus(boot_vcpu_count, Some(request_ts), Some(entry_addr))?; Ok(()) } @@ -1213,7 +1219,10 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); // test start boot vcpus assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); @@ -1267,8 +1276,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); + assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // invalid cpuid for pause let cpu_indexes = vec![2]; @@ -1304,9 +1319,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // invalid cpuid for exit let cpu_indexes = vec![2]; @@ -1330,9 +1350,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // exit all success assert!(vcpu_manager.exit_all_vcpus().is_ok()); @@ -1351,9 +1376,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // invalid cpuid for exit let cpu_indexes = vec![2]; @@ -1377,9 +1407,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // revalidate all success assert!(vcpu_manager.revalidate_all_vcpus_cache().is_ok()); @@ -1395,9 +1430,14 @@ mod tests { assert!(vcpu_manager .create_boot_vcpus(TimestampUs::default(), GuestAddress(0)) .is_ok()); + #[cfg(target_arch = "x86_64")] assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3); assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok()); + #[cfg(target_arch = "aarch64")] + assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2); // set vcpus in hotplug action let cpu_ids = vec![0]; diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index edc753245..fe8f23207 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -6,70 +6,35 @@ // Use of this source code is governed by a BSD-style license that can be // found in the THIRD-PARTY file. -use std::collections::HashMap; -use std::fmt::Debug; use std::ops::Deref; +use std::sync::MutexGuard; use dbs_arch::gic::GICDevice; use dbs_arch::pmu::initialize_pmu; -use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel}; +use dbs_arch::{MMIODeviceInfo, VpmuFeatureLevel}; +use dbs_boot::fdt_utils::*; use dbs_boot::InitrdConfig; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; -use linux_loader::loader::Cmdline; -use vm_memory::{GuestAddressSpace, GuestMemory}; +use linux_loader::cmdline::{Cmdline, Error as CmdlineError}; +use vm_memory::GuestAddressSpace; use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, StartMicroVmError}; use crate::event_manager::EventManager; +use crate::vcpu::VcpuManager; -/// Configures the system and should be called once per vm before starting vcpu threads. -/// For aarch64, we only setup the FDT. -/// -/// # Arguments -/// -/// * `guest_mem` - The memory to be used by the guest. -/// * `cmdline` - The kernel commandline. -/// * `vcpu_mpidr` - Array of MPIDR register values per vcpu. -/// * `device_info` - A hashmap containing the attached devices for building FDT device nodes. -/// * `gic_device` - The GIC device. -/// * `initrd` - Information about an optional initrd. -#[allow(clippy::borrowed_box)] -fn configure_system( - guest_mem: &M, - cmdline: &str, - vcpu_mpidr: Vec, - device_info: Option<&HashMap<(DeviceType, String), T>>, - gic_device: &Box, - initrd: &Option, - vpmu_feature: &VpmuFeatureLevel, -) -> super::Result<()> { - dbs_boot::fdt::create_fdt( - guest_mem, - vcpu_mpidr, - cmdline, - device_info, - gic_device, - initrd, - vpmu_feature, - ) - .map_err(Error::BootSystem)?; - Ok(()) -} - -#[cfg(target_arch = "aarch64")] impl Vm { /// Gets a reference to the irqchip of the VM - #[allow(clippy::borrowed_box)] - pub fn get_irqchip(&self) -> &Box { - self.irqchip_handle.as_ref().unwrap() + pub fn get_irqchip(&self) -> &dyn GICDevice { + self.irqchip_handle.as_ref().unwrap().as_ref() } /// Creates the irq chip in-kernel device model. pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> { - let vcpu_count = self.vm_config.vcpu_count; + let vcpu_count = self.vm_config.max_vcpu_count; self.irqchip_handle = Some( dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) @@ -99,12 +64,11 @@ impl Vm { /// Initialize the virtual machine instance. /// /// It initialize the virtual machine instance by: - /// 1) initialize virtual machine global state and configuration. - /// 2) create system devices, such as interrupt controller. - /// 3) create and start IO devices, such as serial, console, block, net, vsock etc. - /// 4) create and initialize vCPUs. - /// 5) configure CPU power management features. - /// 6) load guest kernel image. + /// 1) Initialize virtual machine reset event fd. + /// 2) Create and initialize vCPUs. + /// 3) Create and initialize interrupt controller. + /// 4) Create and initialize vPMU device. + /// 5) Create and initialize devices, such as virtio, block, net, vsock, vfio etc. pub fn init_microvm( &mut self, epoll_mgr: EpollManager, @@ -139,6 +103,52 @@ impl Vm { Ok(()) } + /// Generate fdt information about VM. + fn get_fdt_vm_info<'a>( + &'a self, + vm_memory: &'a GuestMemoryImpl, + cmdline: &'a str, + initrd_config: Option<&'a InitrdConfig>, + vcpu_manager: &'a MutexGuard, + ) -> FdtVmInfo { + let guest_memory = vm_memory.memory(); + let vcpu_mpidr = vcpu_manager + .vcpus() + .into_iter() + .map(|cpu| cpu.get_mpidr()) + .collect(); + let vm_config = self.vm_config(); + let mut vcpu_boot_onlined = vec![1; vm_config.vcpu_count as usize]; + vcpu_boot_onlined.resize(vm_config.max_vcpu_count as usize, 0); + let vpmu_feature = vcpu_manager.vpmu_feature(); + // This configuration is used for passing cache information into guest. + // TODO: dragonball-sandbox #274; kata-containers #6969 + let cache_passthrough_enabled = false; + let fdt_vcpu_info = FdtVcpuInfo::new( + vcpu_mpidr, + vcpu_boot_onlined, + vpmu_feature, + cache_passthrough_enabled, + ); + + FdtVmInfo::new(guest_memory, cmdline, initrd_config, fdt_vcpu_info) + } + + // This method is used for passing cache/numa information into guest + // TODO: dragonball-sandbox #274,#275; kata-containers #6969 + /// Generate fdt information about cache/numa + fn get_fdt_numa_info(&self) -> FdtNumaInfo { + FdtNumaInfo::default() + } + + /// Generate fdt information about devices + fn get_fdt_device_info(&self) -> FdtDeviceInfo { + FdtDeviceInfo::new( + self.device_manager().get_mmio_device_info(), + self.get_irqchip(), + ) + } + /// Execute system architecture specific configurations. /// /// 1) set guest kernel boot parameters @@ -150,24 +160,23 @@ impl Vm { initrd: Option, ) -> std::result::Result<(), StartMicroVmError> { let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; - let vpmu_feature = vcpu_manager.vpmu_feature(); - let vcpu_mpidr = vcpu_manager - .vcpus() - .into_iter() - .map(|cpu| cpu.get_mpidr()) - .collect(); - let guest_memory = vm_memory.memory(); + let cmdline_cstring = cmdline + .as_cstring() + .map_err(StartMicroVmError::ProcessCommandlne)?; + let fdt_vm_info = self.get_fdt_vm_info( + vm_memory, + cmdline_cstring + .to_str() + .map_err(|_| StartMicroVmError::ProcessCommandlne(CmdlineError::InvalidAscii))?, + initrd.as_ref(), + &vcpu_manager, + ); + let fdt_numa_info = self.get_fdt_numa_info(); + let fdt_device_info = self.get_fdt_device_info(); - configure_system( - guest_memory, - cmdline.as_cstring().unwrap().to_str().unwrap(), - vcpu_mpidr, - self.device_manager.get_mmio_device_info(), - self.get_irqchip(), - &initrd, - &vpmu_feature, - ) - .map_err(StartMicroVmError::ConfigureSystem) + dbs_boot::fdt::create_fdt(fdt_vm_info, fdt_numa_info, fdt_device_info) + .map(|_| ()) + .map_err(|e| StartMicroVmError::ConfigureSystem(Error::BootSystem(e))) } pub(crate) fn register_events( diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 0fab58e05..323db0a86 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -241,6 +241,7 @@ impl Vm { resource_manager.clone(), epoll_manager.clone(), &logger, + api_shared_info.clone(), ); Ok(Vm { @@ -413,6 +414,19 @@ impl Vm { (dragonball_version, instance_id) } + + pub(crate) fn stop_prealloc(&mut self) -> std::result::Result<(), StartMicroVmError> { + if self.address_space.is_initialized() { + return self + .address_space + .wait_prealloc(true) + .map_err(StartMicroVmError::AddressManagerError); + } + + Err(StartMicroVmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + )) + } } impl Vm { @@ -481,7 +495,6 @@ impl Vm { ) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: initializing devices ..."); - let com1_sock_path = self.vm_config.serial_path.clone(); let kernel_config = self .kernel_config .as_mut() @@ -503,9 +516,9 @@ impl Vm { vm_as.clone(), epoll_manager, kernel_config, - com1_sock_path, self.dmesg_fifo.take(), self.address_space.address_space(), + &self.vm_config, )?; info!(self.logger, "VM: start devices"); @@ -888,6 +901,7 @@ impl Vm { #[cfg(test)] pub mod tests { + #[cfg(target_arch = "x86_64")] use kvm_ioctls::VcpuExit; use linux_loader::cmdline::Cmdline; use test_utils::skip_if_not_root; diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 322a3625e..5da16917f 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -508,6 +508,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 598cde620..f0a0f4471 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -27,6 +27,7 @@ thiserror = "1.0" toml = "0.5.8" oci = { path = "../oci" } +safe-path = { path = "../safe-path" } [dev-dependencies] tempfile = "3" diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 3af0563c1..f094ddd70 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -316,6 +316,10 @@ pub const KATA_ANNO_CFG_VFIO_MODE: &str = "io.katacontainers.config.runtime.vfio pub const KATA_ANNO_CFG_HYPERVISOR_PREFETCH_FILES_LIST: &str = "io.katacontainers.config.hypervisor.prefetch_files.list"; +/// A sandbox annotation for sandbox level volume sharing with host. +pub const KATA_ANNO_CFG_SANDBOX_BIND_MOUNTS: &str = + "io.katacontainers.config.runtime.sandbox_bind_mounts"; + /// A helper structure to query configuration information by check annotations. #[derive(Debug, Default, Deserialize)] pub struct Annotation { @@ -950,6 +954,16 @@ impl Annotation { KATA_ANNO_CFG_VFIO_MODE => { config.runtime.vfio_mode = value.to_string(); } + KATA_ANNO_CFG_SANDBOX_BIND_MOUNTS => { + let args: Vec = value + .to_string() + .split_ascii_whitespace() + .map(str::to_string) + .collect(); + for arg in args { + config.runtime.sandbox_bind_mounts.push(arg.to_string()); + } + } _ => { warn!(sl!(), "Annotation {} not enabled", key); } diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 067ff6776..dddd3adc5 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -8,7 +8,8 @@ use std::path::Path; use super::default; use crate::config::{ConfigOps, TomlConfig}; -use crate::{eother, resolve_path, validate_path}; +use crate::mount::split_bind_mounts; +use crate::{eother, validate_path}; /// Type of runtime VirtContainer. pub const RUNTIME_NAME_VIRTCONTAINER: &str = "virt_container"; @@ -146,7 +147,14 @@ impl ConfigOps for Runtime { } for bind in conf.runtime.sandbox_bind_mounts.iter_mut() { - resolve_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?; + // Split the bind mount, canonicalize the path and then append rw mode to it. + let (real_path, mode) = split_bind_mounts(bind); + match Path::new(real_path).canonicalize() { + Err(e) => return Err(eother!("sandbox bind mount `{}` is invalid: {}", bind, e)), + Ok(path) => { + *bind = format!("{}{}", path.display(), mode); + } + } } Ok(()) @@ -176,7 +184,12 @@ impl ConfigOps for Runtime { } for bind in conf.runtime.sandbox_bind_mounts.iter() { - validate_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?; + // Just validate the real_path. + let (real_path, _mode) = split_bind_mounts(bind); + validate_path!( + real_path.to_owned(), + "sandbox bind mount `{}` is invalid: {}" + )?; } Ok(()) diff --git a/src/libs/kata-types/src/cpu.rs b/src/libs/kata-types/src/cpu.rs index 0020de097..e47681f62 100644 --- a/src/libs/kata-types/src/cpu.rs +++ b/src/libs/kata-types/src/cpu.rs @@ -26,7 +26,7 @@ pub enum Error { } /// Assigned CPU resources for a Linux container. -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub struct LinuxContainerCpuResources { shares: u64, period: u64, diff --git a/src/libs/kata-types/src/mount.rs b/src/libs/kata-types/src/mount.rs index f66e828bd..83bcca3ea 100644 --- a/src/libs/kata-types/src/mount.rs +++ b/src/libs/kata-types/src/mount.rs @@ -5,7 +5,7 @@ // use anyhow::{anyhow, Context, Result}; -use std::{collections::HashMap, path::PathBuf}; +use std::{collections::HashMap, fs, path::PathBuf}; /// Prefix to mark a volume as Kata special. pub const KATA_VOLUME_TYPE_PREFIX: &str = "kata:"; @@ -25,6 +25,15 @@ pub const KATA_MOUNT_INFO_FILE_NAME: &str = "mountInfo.json"; /// KATA_DIRECT_VOLUME_ROOT_PATH is the root path used for concatenating with the direct-volume mount info file path pub const KATA_DIRECT_VOLUME_ROOT_PATH: &str = "/run/kata-containers/shared/direct-volumes"; +/// SANDBOX_BIND_MOUNTS_DIR is for sandbox bindmounts +pub const SANDBOX_BIND_MOUNTS_DIR: &str = "sandbox-mounts"; + +/// SANDBOX_BIND_MOUNTS_RO is for sandbox bindmounts with readonly +pub const SANDBOX_BIND_MOUNTS_RO: &str = ":ro"; + +/// SANDBOX_BIND_MOUNTS_RO is for sandbox bindmounts with readwrite +pub const SANDBOX_BIND_MOUNTS_RW: &str = ":rw"; + /// Information about a mount. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] pub struct Mount { @@ -71,6 +80,27 @@ pub struct DirectVolumeMountInfo { pub options: Vec, } +/// join_path joins user provided volumepath with kata direct-volume root path +/// the volume_path is base64-encoded and then safely joined to the end of path prefix +pub fn join_path(prefix: &str, volume_path: &str) -> Result { + if volume_path.is_empty() { + return Err(anyhow!("volume path must not be empty")); + } + let b64_encoded_path = base64::encode(volume_path.as_bytes()); + + Ok(safe_path::scoped_join(prefix, b64_encoded_path)?) +} + +/// get DirectVolume mountInfo from mountinfo.json. +pub fn get_volume_mount_info(volume_path: &str) -> Result { + let mount_info_file_path = + join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?.join(KATA_MOUNT_INFO_FILE_NAME); + let mount_info_file = fs::read_to_string(mount_info_file_path)?; + let mount_info: DirectVolumeMountInfo = serde_json::from_str(&mount_info_file)?; + + Ok(mount_info) +} + /// Check whether a mount type is a marker for Kata specific volume. pub fn is_kata_special_volume(ty: &str) -> bool { ty.len() > KATA_VOLUME_TYPE_PREFIX.len() && ty.starts_with(KATA_VOLUME_TYPE_PREFIX) @@ -128,6 +158,28 @@ impl NydusExtraOptions { serde_json::from_slice(&extra_options_buf).context("deserialize nydus's extraoption") } } + +/// sandbox bindmount format: /path/to/dir, or /path/to/dir:ro[:rw] +/// the real path is without suffix ":ro" or ":rw". +pub fn split_bind_mounts(bindmount: &str) -> (&str, &str) { + let (real_path, mode) = if bindmount.ends_with(SANDBOX_BIND_MOUNTS_RO) { + ( + bindmount.trim_end_matches(SANDBOX_BIND_MOUNTS_RO), + SANDBOX_BIND_MOUNTS_RO, + ) + } else if bindmount.ends_with(SANDBOX_BIND_MOUNTS_RW) { + ( + bindmount.trim_end_matches(SANDBOX_BIND_MOUNTS_RW), + SANDBOX_BIND_MOUNTS_RW, + ) + } else { + // default bindmount format + (bindmount, "") + }; + + (real_path, mode) +} + #[cfg(test)] mod tests { use super::*; @@ -137,6 +189,18 @@ mod tests { assert!(!is_kata_special_volume("kata:")); } + #[test] + fn test_split_bind_mounts() { + let test01 = "xxx0:ro"; + let test02 = "xxx2:rw"; + let test03 = "xxx3:is"; + let test04 = "xxx4"; + assert_eq!(split_bind_mounts(test01), ("xxx0", ":ro")); + assert_eq!(split_bind_mounts(test02), ("xxx2", ":rw")); + assert_eq!(split_bind_mounts(test03), ("xxx3:is", "")); + assert_eq!(split_bind_mounts(test04), ("xxx4", "")); + } + #[test] fn test_is_kata_guest_mount_volume() { assert!(is_kata_guest_mount_volume("kata:guest-mount:nfs")); diff --git a/src/libs/kata-types/src/utils/u32_set.rs b/src/libs/kata-types/src/utils/u32_set.rs index 3742e4d54..44c55a163 100644 --- a/src/libs/kata-types/src/utils/u32_set.rs +++ b/src/libs/kata-types/src/utils/u32_set.rs @@ -13,7 +13,7 @@ use crate::Error; /// /// The `U32Set` may be used to save CPUs parsed from a CPU list file or NUMA nodes parsed from /// a NUMA node list file. -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub struct U32Set(Vec); impl U32Set { diff --git a/src/libs/logging/Cargo.toml b/src/libs/logging/Cargo.toml index c7cac4d7b..4d19d90c6 100644 --- a/src/libs/logging/Cargo.toml +++ b/src/libs/logging/Cargo.toml @@ -15,6 +15,7 @@ serde_json = "1.0.73" # (by stopping the compiler from removing log calls). slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] } slog-json = "2.4.0" +slog-term = "2.9.0" slog-async = "2.7.0" slog-scope = "4.4.0" diff --git a/src/libs/logging/src/lib.rs b/src/libs/logging/src/lib.rs index 29325c6bd..b66217459 100644 --- a/src/libs/logging/src/lib.rs +++ b/src/libs/logging/src/lib.rs @@ -39,6 +39,28 @@ const LOG_LEVELS: &[(&str, slog::Level)] = &[ const DEFAULT_SUBSYSTEM: &str = "root"; +// Creates a logger which prints output as human readable text to the terminal +pub fn create_term_logger(level: slog::Level) -> (slog::Logger, slog_async::AsyncGuard) { + let term_drain = slog_term::term_compact().fuse(); + + // Ensure only a unique set of key/value fields is logged + let unique_drain = UniqueDrain::new(term_drain).fuse(); + + // Allow runtime filtering of records by log level + let filter_drain = RuntimeLevelFilter::new(unique_drain, level).fuse(); + + // Ensure the logger is thread-safe + let (async_drain, guard) = slog_async::Async::new(filter_drain) + .thread_name("slog-async-logger".into()) + .build_with_guard(); + + // Add some "standard" fields + let logger = slog::Logger::root(async_drain.fuse(), o!("subsystem" => DEFAULT_SUBSYSTEM)); + + (logger, guard) +} + +// Creates a logger which prints output as JSON // XXX: 'writer' param used to make testing possible. pub fn create_logger( name: &str, diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index 3ad755256..9ed7c0a04 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -366,7 +366,8 @@ message OnlineCPUMemRequest { // resources are connected asynchronously and the agent returns immediately. bool wait = 1; - // NbCpus specifies the number of CPUs that were added and the agent has to online. + // NbCpus specifies the number of CPUs that should be onlined in the guest. + // Special value 0 means agent will skip this check. uint32 nb_cpus = 2; // CpuOnly specifies whether only online CPU or not. diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index c98e45f0e..358776fab 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -241,6 +241,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -644,11 +655,12 @@ dependencies = [ [[package]] name = "dbs-address-space" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc" +checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322" dependencies = [ "arc-swap", + "lazy_static", "libc", "nix 0.23.2", "thiserror", @@ -682,9 +694,9 @@ dependencies = [ [[package]] name = "dbs-boot" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a74a8c05a1674d3032e610b4f201c7440c345559bad3dfe6b455ce195785108" +checksum = "5466a92f75aa928a9103dcb2088f6d1638ef9da8945fad7389a73864dfa0182c" dependencies = [ "dbs-arch", "kvm-bindings", @@ -735,9 +747,9 @@ dependencies = [ [[package]] name = "dbs-upcall" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f" +checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb" dependencies = [ "anyhow", "dbs-utils", @@ -765,9 +777,9 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668" +checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8" dependencies = [ "byteorder", "caps", @@ -781,7 +793,7 @@ dependencies = [ "kvm-ioctls", "libc", "log", - "nix 0.23.2", + "nix 0.24.3", "nydus-api", "nydus-blobfs", "nydus-rafs", @@ -827,6 +839,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dlv-list" version = "0.3.0" @@ -1212,6 +1245,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -1481,6 +1523,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path 0.1.0", "serde", "serde_json", "slog", @@ -1618,6 +1661,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -1845,7 +1889,16 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ "libc", ] @@ -2433,6 +2486,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.8", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.7.1" @@ -2844,6 +2908,19 @@ dependencies = [ "slog-scope", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.20", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -2962,6 +3039,17 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.2.0" @@ -3042,6 +3130,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index 185912145..bbc401f64 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -3,3 +3,4 @@ members = [ "crates/shim", "crates/shim-ctl", ] + diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index 4c7d3db05..e9c2b8c19 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -309,3 +309,12 @@ experimental=@DEFAULTEXPFEATURES@ # - When running single containers using a tool like ctr, container sizing information will be available. static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@ +# If specified, sandbox_bind_mounts identifieds host paths to be mounted(ro, rw) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +# Now it supports three kinds of bind mount format: +# - "/path/to", default readonly mode. +# - "/path/to:ro", readonly mode. +# - "/path/to:rw", readwrite mode. +sandbox_bind_mounts=@DEFBINDMOUNTS@ diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index d06da15ea..13ba4085d 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -117,5 +117,6 @@ impl_agent!( get_ip_tables | crate::GetIPTablesRequest | crate::GetIPTablesResponse | None, set_ip_tables | crate::SetIPTablesRequest | crate::SetIPTablesResponse | None, get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None, - resize_volume | crate::ResizeVolumeRequest | crate::Empty | None + resize_volume | crate::ResizeVolumeRequest | crate::Empty | None, + online_cpu_mem | crate::OnlineCPUMemRequest | crate::Empty | None ); diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 172095ceb..ca9c8f172 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -336,7 +336,7 @@ impl From for agent::UpdateContainerRequest { fn from(from: UpdateContainerRequest) -> Self { Self { container_id: from.container_id, - resources: from_option(Some(from.resources)), + resources: from_option(from.resources), ..Default::default() } } diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index ea3bab78f..43449ca59 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -54,6 +54,7 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { // sandbox async fn create_sandbox(&self, req: CreateSandboxRequest) -> Result; async fn destroy_sandbox(&self, req: Empty) -> Result; + async fn online_cpu_mem(&self, req: OnlineCPUMemRequest) -> Result; // network async fn add_arp_neighbors(&self, req: AddArpNeighborRequest) -> Result; diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index da6e14430..1ba7efd5e 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -201,7 +201,7 @@ pub struct ListProcessesRequest { #[derive(PartialEq, Clone, Default)] pub struct UpdateContainerRequest { pub container_id: String, - pub resources: oci::LinuxResources, + pub resources: Option, pub mounts: Vec, } diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index eb613aad3..6a61faea4 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -32,7 +32,7 @@ kata-types = { path = "../../../libs/kata-types" } logging = { path = "../../../libs/logging" } shim-interface = { path = "../../../libs/shim-interface" } -dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs","dbs-upcall"] } +dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs", "dbs-upcall"] } ch-config = { path = "ch-config", optional = true } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 9cf5bcd8c..a54a6a43e 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -494,6 +494,10 @@ impl CloudHypervisorInner { Ok(()) } + pub(crate) async fn resize_vcpu(&self, old_vcpu: u32, new_vcpu: u32) -> Result<(u32, u32)> { + Ok((old_vcpu, new_vcpu)) + } + pub(crate) async fn get_pids(&self) -> Result> { Ok(Vec::::new()) } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index a4b8b05ff..dd95413fc 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -114,6 +114,11 @@ impl Hypervisor for CloudHypervisor { inner.cleanup().await } + async fn resize_vcpu(&self, old_vcpu: u32, new_vcpu: u32) -> Result<(u32, u32)> { + let inner = self.inner.read().await; + inner.resize_vcpu(old_vcpu, new_vcpu).await + } + async fn get_pids(&self) -> Result> { let inner = self.inner.read().await; inner.get_pids().await diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index c633cf918..62d9f7e52 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -8,19 +8,21 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::{anyhow, Context, Result}; use kata_sys_util::rand::RandomBytes; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, RwLock}; +use super::{ + util::{get_host_path, get_virt_drive_name}, + Device, DeviceConfig, DeviceType, +}; use crate::{ BlockConfig, BlockDevice, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, }; -use super::{ - util::{get_host_path, get_virt_drive_name}, - Device, DeviceConfig, -}; pub type ArcMutexDevice = Arc>; +const DEVICE_TYPE_BLOCK: &str = "b"; + /// block_index and released_block_index are used to search an available block index /// in Sandbox. /// @@ -75,35 +77,27 @@ impl DeviceManager { }) } - pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result { - let device_id = if let Some(dev) = self.find_device(device_config).await { - dev - } else { - self.create_device(device_config) - .await - .context("failed to create device")? - }; - Ok(device_id) - } - - pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> { + async fn try_add_device(&mut self, device_id: &str) -> Result<()> { // find the device let device = self .devices .get(device_id) .context("failed to find device")?; - let mut device_guard = device.lock().await; + // attach device + let mut device_guard = device.lock().await; let result = device_guard.attach(self.hypervisor.as_ref()).await; + // handle attach error if let Err(e) = result { - if let DeviceConfig::BlockCfg(config) = device_guard.get_device_info().await { - self.shared_info.release_device_index(config.index); + if let DeviceType::Block(device) = device_guard.get_device_info().await { + self.shared_info.release_device_index(device.config.index); }; drop(device_guard); self.devices.remove(device_id); return Err(e); } + Ok(()) } @@ -120,66 +114,97 @@ impl DeviceManager { } Err(e) => Err(e), }; + + // if detach success, remove it from device manager if result.is_ok() { drop(device_guard); - // if detach success, remove it from device manager self.devices.remove(device_id); } + return result; } + Err(anyhow!( "device with specified ID hasn't been created. {}", device_id )) } - pub async fn get_device_info(&self, device_id: &str) -> Result { + async fn get_device_info(&self, device_id: &str) -> Result { if let Some(dev) = self.devices.get(device_id) { return Ok(dev.lock().await.get_device_info().await); } + Err(anyhow!( "device with specified ID hasn't been created. {}", device_id )) } - async fn find_device(&self, device_config: &DeviceConfig) -> Option { + async fn find_device(&self, host_path: String) -> Option { for (device_id, dev) in &self.devices { match dev.lock().await.get_device_info().await { - DeviceConfig::BlockCfg(config) => match device_config { - DeviceConfig::BlockCfg(ref config_new) => { - if config_new.path_on_host == config.path_on_host { - return Some(device_id.to_string()); - } + DeviceType::Block(device) => { + if device.config.path_on_host == host_path { + return Some(device_id.to_string()); } - _ => { - continue; - } - }, + } _ => { // TODO: support find other device type continue; } } } + None } - async fn create_device(&mut self, device_config: &DeviceConfig) -> Result { + fn get_dev_virt_path(&mut self, dev_type: &str) -> Result> { + let virt_path = if dev_type == DEVICE_TYPE_BLOCK { + // generate virt path + let current_index = self.shared_info.declare_device_index()?; + let drive_name = get_virt_drive_name(current_index as i32)?; + let virt_path_name = format!("/dev/{}", drive_name); + + Some((current_index, virt_path_name)) + } else { + // only dev_type is block, otherwise, it's useless. + None + }; + + Ok(virt_path) + } + + async fn new_device(&mut self, device_config: &DeviceConfig) -> Result { // device ID must be generated by manager instead of device itself // in case of ID collision let device_id = self.new_device_id()?; let dev: ArcMutexDevice = match device_config { - DeviceConfig::BlockCfg(config) => self - .create_block_device(config, device_id.clone()) - .await - .context("failed to create device")?, + DeviceConfig::BlockCfg(config) => { + // try to find the device, found and just return id. + if let Some(dev_id_matched) = self.find_device(config.path_on_host.clone()).await { + info!( + sl!(), + "device with host path:{:?} found. just return device id: {:?}", + config.path_on_host.clone(), + dev_id_matched + ); + + return Ok(dev_id_matched); + } + + self.create_block_device(config, device_id.clone()) + .await + .context("failed to create device")? + } _ => { return Err(anyhow!("invliad device type")); } }; + // register device to devices self.devices.insert(device_id.clone(), dev.clone()); + Ok(device_id) } @@ -204,17 +229,23 @@ impl DeviceManager { _ => "".to_string(), }; block_config.driver_option = block_driver; - // generate virt path - let current_index = self.shared_info.declare_device_index()?; - block_config.index = current_index; - let drive_name = get_virt_drive_name(current_index as i32)?; - block_config.virt_path = format!("/dev/{}", drive_name); - // if the path on host is empty, we need to get device host path from the device major and minor number - // Otherwise, it might be rawfile based block device, the host path is already passed from the runtime, so we don't need to do anything here - if block_config.path_on_host.is_empty() { - block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor) - .context("failed to get host path")?; + + // generate block device index and virt path + // safe here, Block device always has virt_path. + if let Some(virt_path) = self.get_dev_virt_path(DEVICE_TYPE_BLOCK)? { + block_config.index = virt_path.0; + block_config.virt_path = virt_path.1; } + + // if the path on host is empty, we need to get device host path from the device major and minor number + // Otherwise, it might be rawfile based block device, the host path is already passed from the runtime, + // so we don't need to do anything here + if block_config.path_on_host.is_empty() { + block_config.path_on_host = + get_host_path(DEVICE_TYPE_BLOCK.to_owned(), config.major, config.minor) + .context("failed to get host path")?; + } + Ok(Arc::new(Mutex::new(BlockDevice::new( device_id, block_config, @@ -237,3 +268,36 @@ impl DeviceManager { Err(anyhow!("ID are exhausted")) } } + +// Many scenarios have similar steps when adding devices. so to reduce duplicated code, +// we should create a common method abstracted and use it in various scenarios. +// do_handle_device: +// (1) new_device with DeviceConfig and return device_id; +// (2) try_add_device with device_id and do really add device; +// (3) return device info of device's info; +pub async fn do_handle_device( + d: &RwLock, + dev_info: &DeviceConfig, +) -> Result { + let device_id = d + .write() + .await + .new_device(dev_info) + .await + .context("failed to create deviec")?; + + d.write() + .await + .try_add_device(&device_id) + .await + .context("failed to add deivce")?; + + let device_info = d + .read() + .await + .get_device_info(&device_id) + .await + .context("failed to get device info")?; + + Ok(device_info) +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs index ff3a579a8..63fe40022 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs @@ -7,7 +7,7 @@ use std::{fs, path::Path, process::Command}; use crate::device::Device; -use crate::device::DeviceConfig; +use crate::device::DeviceType; use crate::Hypervisor as hypervisor; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use anyhow::anyhow; @@ -166,7 +166,7 @@ impl Device for VfioConfig { todo!() } - async fn get_device_info(&self) -> DeviceConfig { + async fn get_device_info(&self) -> DeviceType { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs index d778c4459..a105672d5 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs @@ -5,7 +5,7 @@ // use crate::device::Device; -use crate::device::DeviceConfig; +use crate::device::DeviceType; use crate::Hypervisor as hypervisor; use anyhow::Result; use async_trait::async_trait; @@ -47,7 +47,7 @@ impl Device for VhostUserConfig { todo!() } - async fn get_device_info(&self) -> DeviceConfig { + async fn get_device_info(&self) -> DeviceType { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs index 2ff98a1e7..da5d50ea7 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs @@ -6,7 +6,7 @@ pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio"; use crate::device::Device; -use crate::device::{DeviceConfig, DeviceType}; +use crate::device::DeviceType; use crate::Hypervisor as hypervisor; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -98,8 +98,8 @@ impl Device for BlockDevice { Ok(Some(self.config.index)) } - async fn get_device_info(&self) -> DeviceConfig { - DeviceConfig::BlockCfg(self.config.clone()) + async fn get_device_info(&self) -> DeviceType { + DeviceType::Block(self.clone()) } async fn increase_attach_count(&mut self) -> Result { diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs index d2d3cc762..0a97845e7 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs @@ -67,6 +67,9 @@ pub struct ShareFsDeviceConfig { /// queue_num: queue number pub queue_num: u64, + + /// options: virtiofs device's config options. + pub options: Vec, } #[derive(Debug, Clone)] diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs index d341d9a12..d4996a3e6 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs @@ -53,7 +53,7 @@ pub trait Device: Send + Sync { // detach is to unplug device from VM async fn detach(&mut self, h: &dyn hypervisor) -> Result>; // get_device_info returns device config - async fn get_device_info(&self) -> DeviceConfig; + async fn get_device_info(&self) -> DeviceType; // increase_attach_count is used to increase the attach count for a device // return values: // * true: no need to do real attach when current attach count is zero, skip following actions. diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 45b77f09e..09afc4c1a 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -13,7 +13,7 @@ use crate::{ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use dragonball::{ - api::v1::{BlockDeviceConfigInfo, BootSourceConfig}, + api::v1::{BlockDeviceConfigInfo, BootSourceConfig, VcpuResizeInfo}, vm::VmConfigInfo, }; use kata_sys_util::mount; @@ -327,6 +327,53 @@ impl DragonballInner { } } + // check if resizing info is valid + // the error in this function is not ok to be tolerated, the container boot will fail + fn precheck_resize_vcpus(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + // old_vcpus > 0, safe for conversion + let current_vcpus = old_vcpus; + + // a non-zero positive is required + if new_vcpus == 0 { + return Err(anyhow!("resize vcpu error: 0 vcpu resizing is invalid")); + } + + // cannot exceed maximum value + if new_vcpus > self.config.cpu_info.default_maxvcpus { + return Err(anyhow!("resize vcpu error: cannot greater than maxvcpus")); + } + + Ok((current_vcpus, new_vcpus)) + } + + // do the check before resizing, returns Result<(old, new)> + pub(crate) async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + if old_vcpus == new_vcpus { + info!( + sl!(), + "resize_vcpu: no need to resize vcpus because old_vcpus is equal to new_vcpus" + ); + return Ok((new_vcpus, new_vcpus)); + } + + let (old_vcpus, new_vcpus) = self.precheck_resize_vcpus(old_vcpus, new_vcpus)?; + info!( + sl!(), + "check_resize_vcpus passed, passing new_vcpus = {:?} to vmm", new_vcpus + ); + + let cpu_resize_info = VcpuResizeInfo { + vcpu_count: Some(new_vcpus as u8), + }; + self.vmm_instance + .resize_vcpu(&cpu_resize_info) + .context(format!( + "failed to do_resize_vcpus on new_vcpus={:?}", + new_vcpus + ))?; + Ok((old_vcpus, new_vcpus)) + } + pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) { self.config = config; } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 94861536c..10e13b578 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -157,7 +157,11 @@ impl DragonballInner { .context("insert vsock") } - fn parse_inline_virtiofs_args(&self, fs_cfg: &mut FsDeviceConfigInfo) -> Result<()> { + fn parse_inline_virtiofs_args( + &self, + fs_cfg: &mut FsDeviceConfigInfo, + options: &mut Vec, + ) -> Result<()> { let mut debug = false; let mut opt_list = String::new(); @@ -169,8 +173,8 @@ impl DragonballInner { sl!(), "args: {:?}", &self.config.shared_fs.virtio_fs_extra_args ); - let args = &self.config.shared_fs.virtio_fs_extra_args; - let _ = go_flag::parse_args_with_warnings::(args, None, |flags| { + let mut args = self.config.shared_fs.virtio_fs_extra_args.clone(); + let _ = go_flag::parse_args_with_warnings::(&args, None, |flags| { flags.add_flag("d", &mut debug); flags.add_flag("thread-pool-size", &mut fs_cfg.thread_pool_size); flags.add_flag("drop-sys-resource", &mut fs_cfg.drop_sys_resource); @@ -178,6 +182,9 @@ impl DragonballInner { }) .with_context(|| format!("parse args: {:?}", args))?; + // more options parsed for inline virtio-fs' custom config + args.append(options); + if debug { warn!( sl!(), @@ -202,6 +209,7 @@ impl DragonballInner { "xattr" => fs_cfg.xattr = true, "no_xattr" => fs_cfg.xattr = false, "cache_symlinks" => {} // inline virtiofs always cache symlinks + "no_readdir" => fs_cfg.no_readdir = true, "trace" => warn!( sl!(), "Inline virtiofs \"-o trace\" option not supported yet, ignored." @@ -234,16 +242,25 @@ impl DragonballInner { xattr: true, ..Default::default() }; - self.do_add_fs_device(&config.fs_type, &mut fs_cfg) + + let mut options = config.options.clone(); + self.do_add_fs_device(&config.fs_type, &mut fs_cfg, &mut options) } - fn do_add_fs_device(&self, fs_type: &str, fs_cfg: &mut FsDeviceConfigInfo) -> Result<()> { + fn do_add_fs_device( + &self, + fs_type: &str, + fs_cfg: &mut FsDeviceConfigInfo, + options: &mut Vec, + ) -> Result<()> { match fs_type { VIRTIO_FS => { fs_cfg.mode = String::from("vhostuser"); } INLINE_VIRTIO_FS => { - self.parse_inline_virtiofs_args(fs_cfg)?; + // All parameters starting with --patch-fs do not need to be processed, these are the parameters required by patch fs + options.retain(|x| !x.starts_with("--patch-fs")); + self.parse_inline_virtiofs_args(fs_cfg, options)?; } _ => { return Err(anyhow!( @@ -311,8 +328,12 @@ mod tests { "--drop-sys-resource".to_string(), "-d".to_string(), ]; + + let mut options: Vec = Vec::new(); dragonball.config.shared_fs.virtio_fs_cache = "auto".to_string(); - dragonball.parse_inline_virtiofs_args(&mut fs_cfg).unwrap(); + dragonball + .parse_inline_virtiofs_args(&mut fs_cfg, &mut options) + .unwrap(); assert!(!fs_cfg.no_open); assert!(fs_cfg.xattr); diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index c6df95cc9..707294937 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -77,6 +77,12 @@ impl Hypervisor for Dragonball { inner.save_vm().await } + // returns Result<(old_vcpus, new_vcpus)> + async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + let inner = self.inner.read().await; + inner.resize_vcpu(old_vcpus, new_vcpus).await + } + async fn add_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.add_device(device).await diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 4c40b40c5..ad3977eca 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -16,8 +16,8 @@ use crossbeam_channel::{unbounded, Receiver, Sender}; use dragonball::{ api::v1::{ BlockDeviceConfigInfo, BootSourceConfig, FsDeviceConfigInfo, FsMountConfigInfo, - InstanceInfo, InstanceState, VirtioNetDeviceConfigInfo, VmmAction, VmmActionError, VmmData, - VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, + InstanceInfo, InstanceState, VcpuResizeInfo, VirtioNetDeviceConfigInfo, VmmAction, + VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, }, vm::VmConfigInfo, Vmm, @@ -248,6 +248,12 @@ impl VmmInstance { Ok(()) } + pub fn resize_vcpu(&self, cfg: &VcpuResizeInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::ResizeVcpu(cfg.clone()))) + .with_context(|| format!("Failed to resize_vm(hotplug vcpu), cfg: {:?}", cfg))?; + Ok(()) + } + pub fn pause(&self) -> Result<()> { todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 2001433e5..2386d0924 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -78,6 +78,7 @@ pub trait Hypervisor: Send + Sync { async fn pause_vm(&self) -> Result<()>; async fn save_vm(&self) -> Result<()>; async fn resume_vm(&self) -> Result<()>; + async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)>; // returns (old_vcpus, new_vcpus) // device manager async fn add_device(&self, device: DeviceType) -> Result<()>; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 456bf6373..4b1f7cae3 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -104,6 +104,11 @@ impl QemuInner { todo!() } + pub(crate) async fn resize_vcpu(&self, _old_vcpus: u32, _new_vcpus: u32) -> Result<(u32, u32)> { + info!(sl!(), "QemuInner::resize_vcpu()"); + todo!() + } + pub(crate) async fn get_pids(&self) -> Result> { info!(sl!(), "QemuInner::get_pids()"); todo!() diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 77217f153..eb657dc2d 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -118,6 +118,11 @@ impl Hypervisor for Qemu { inner.cleanup().await } + async fn resize_vcpu(&self, old_vcpus: u32, new_vcpus: u32) -> Result<(u32, u32)> { + let inner = self.inner.read().await; + inner.resize_vcpu(old_vcpus, new_vcpus).await + } + async fn get_pids(&self) -> Result> { let inner = self.inner.read().await; inner.get_pids().await diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 9847ce61b..22ffda48b 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -41,4 +41,5 @@ logging = { path = "../../../libs/logging" } oci = { path = "../../../libs/oci" } actix-rt = "2.7.0" persist = { path = "../persist"} + [features] diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs index b7f515d7f..831e30c0f 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/mod.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs @@ -26,6 +26,8 @@ use oci::LinuxResources; use persist::sandbox_persist::Persist; use tokio::sync::RwLock; +use crate::ResourceUpdateOp; + const OS_ERROR_NO_SUCH_PROCESS: i32 = 3; pub struct CgroupArgs { @@ -149,29 +151,51 @@ impl CgroupsResource { &self, cid: &str, linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, h: &dyn Hypervisor, ) -> Result<()> { - let resource = self.calc_resource(linux_resources); - let changed = self.update_resources(cid, resource).await; + let new_resources = self.calc_resource(linux_resources); + let old_resources = self.update_resources(cid, new_resources.clone(), op).await; - if !changed { - return Ok(()); - } - - self.do_update_cgroups(h).await - } - - async fn update_resources(&self, cid: &str, new_resource: Resources) -> bool { - let mut resources = self.resources.write().await; - let old_resource = resources.insert(cid.to_owned(), new_resource.clone()); - - if let Some(old_resource) = old_resource { - if old_resource == new_resource { - return false; + if let Some(old_resource) = old_resources.clone() { + if old_resource == new_resources { + return Ok(()); } } - true + match self.do_update_cgroups(h).await { + Err(e) => { + // if update failed, we should roll back the records in resources + let mut resources = self.resources.write().await; + match op { + ResourceUpdateOp::Add => { + resources.remove(cid); + } + ResourceUpdateOp::Update | ResourceUpdateOp::Del => { + if let Some(old_resource) = old_resources { + resources.insert(cid.to_owned(), old_resource); + } + } + } + Err(e) + } + Ok(()) => Ok(()), + } + } + + async fn update_resources( + &self, + cid: &str, + new_resource: Resources, + op: ResourceUpdateOp, + ) -> Option { + let mut resources = self.resources.write().await; + match op { + ResourceUpdateOp::Add | ResourceUpdateOp::Update => { + resources.insert(cid.to_owned(), new_resource.clone()) + } + ResourceUpdateOp::Del => resources.remove(cid), + } } async fn do_update_cgroups(&self, h: &dyn Hypervisor) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs new file mode 100644 index 000000000..661e1a5e3 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cpu_mem/cpu.rs @@ -0,0 +1,188 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + cmp, + collections::{HashMap, HashSet}, + convert::TryFrom, + sync::Arc, +}; + +use agent::{Agent, OnlineCPUMemRequest}; +use anyhow::{Context, Ok, Result}; +use hypervisor::Hypervisor; +use kata_types::{config::TomlConfig, cpu::LinuxContainerCpuResources}; +use oci::LinuxCpu; +use tokio::sync::RwLock; + +use crate::ResourceUpdateOp; + +#[derive(Default, Debug, Clone)] +pub struct CpuResource { + /// Current number of vCPUs + pub(crate) current_vcpu: Arc>, + + /// Default number of vCPUs + pub(crate) default_vcpu: u32, + + /// CpuResource of each container + pub(crate) container_cpu_resources: Arc>>, +} + +impl CpuResource { + pub fn new(config: Arc) -> Result { + let hypervisor_name = config.runtime.hypervisor_name.clone(); + let hypervisor_config = config + .hypervisor + .get(&hypervisor_name) + .context(format!("failed to get hypervisor {}", hypervisor_name))?; + Ok(Self { + current_vcpu: Arc::new(RwLock::new(hypervisor_config.cpu_info.default_vcpus as u32)), + default_vcpu: hypervisor_config.cpu_info.default_vcpus as u32, + container_cpu_resources: Arc::new(RwLock::new(HashMap::new())), + }) + } + + pub(crate) async fn update_cpu_resources( + &self, + cid: &str, + linux_cpus: Option<&LinuxCpu>, + op: ResourceUpdateOp, + hypervisor: &dyn Hypervisor, + agent: &dyn Agent, + ) -> Result<()> { + self.update_container_cpu_resources(cid, linux_cpus, op) + .await + .context("update container cpu resources")?; + let vcpu_required = self + .calc_cpu_resources() + .await + .context("calculate vcpus required")?; + + if vcpu_required == self.current_vcpu().await { + return Ok(()); + } + + let curr_vcpus = self + .do_update_cpu_resources(vcpu_required, op, hypervisor, agent) + .await?; + self.update_current_vcpu(curr_vcpus).await; + Ok(()) + } + + async fn current_vcpu(&self) -> u32 { + let current_vcpu = self.current_vcpu.read().await; + *current_vcpu + } + + async fn update_current_vcpu(&self, new_vcpus: u32) { + let mut current_vcpu = self.current_vcpu.write().await; + *current_vcpu = new_vcpus; + } + + // update container_cpu_resources field + async fn update_container_cpu_resources( + &self, + cid: &str, + linux_cpus: Option<&LinuxCpu>, + op: ResourceUpdateOp, + ) -> Result<()> { + if let Some(cpu) = linux_cpus { + let container_resource = LinuxContainerCpuResources::try_from(cpu)?; + let mut resources = self.container_cpu_resources.write().await; + match op { + ResourceUpdateOp::Add => { + resources.insert(cid.to_owned(), container_resource); + } + ResourceUpdateOp::Update => { + let resource = resources.insert(cid.to_owned(), container_resource.clone()); + if let Some(old_container_resource) = resource { + // the priority of cpu-quota is higher than cpuset when determine the number of vcpus. + // we should better ignore the resource update when update cpu only by cpuset if cpu-quota + // has been set previously. + if old_container_resource.quota() > 0 && container_resource.quota() < 0 { + resources.insert(cid.to_owned(), old_container_resource); + } + } + } + ResourceUpdateOp::Del => { + resources.remove(cid); + } + } + } + + Ok(()) + } + + // calculates the total required vcpus by adding each container's requirements within the pod + async fn calc_cpu_resources(&self) -> Result { + let mut total_vcpu = 0; + let mut cpuset_vcpu: HashSet = HashSet::new(); + + let resources = self.container_cpu_resources.read().await; + for (_, cpu_resource) in resources.iter() { + let vcpu = cpu_resource.get_vcpus().unwrap_or(0) as u32; + cpuset_vcpu.extend(cpu_resource.cpuset().iter()); + total_vcpu += vcpu; + } + + // contrained only by cpuset + if total_vcpu == 0 && !cpuset_vcpu.is_empty() { + info!(sl!(), "(from cpuset)get vcpus # {:?}", cpuset_vcpu); + return Ok(cpuset_vcpu.len() as u32); + } + + info!( + sl!(), + "(from cfs_quota&cfs_period)get vcpus count {}", total_vcpu + ); + Ok(total_vcpu) + } + + // do hotplug and hot-unplug the vcpu + async fn do_update_cpu_resources( + &self, + new_vcpus: u32, + op: ResourceUpdateOp, + hypervisor: &dyn Hypervisor, + agent: &dyn Agent, + ) -> Result { + let old_vcpus = self.current_vcpu().await; + + // when adding vcpus, ignore old_vcpus > new_vcpus + // when deleting vcpus, ignore old_vcpus < new_vcpus + if (op == ResourceUpdateOp::Add && old_vcpus > new_vcpus) + || (op == ResourceUpdateOp::Del && old_vcpus < new_vcpus) + { + return Ok(old_vcpus); + } + + // do not reduce computing power + // the number of vcpus would not be lower than the default size + let new_vcpus = cmp::max(new_vcpus, self.default_vcpu); + + let (old, new) = hypervisor + .resize_vcpu(old_vcpus, new_vcpus) + .await + .context("resize vcpus")?; + + if old < new { + let add = new - old; + info!(sl!(), "request to onlineCpuMem with {:?} cpus", add); + + agent + .online_cpu_mem(OnlineCPUMemRequest { + wait: false, + nb_cpus: new, + cpu_only: true, + }) + .await + .context("online vcpus")?; + } + + Ok(new) + } +} diff --git a/src/runtime-rs/crates/runtimes/src/static_resource.rs b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs similarity index 82% rename from src/runtime-rs/crates/runtimes/src/static_resource.rs rename to src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs index 453ce85b3..53eccc52b 100644 --- a/src/runtime-rs/crates/runtimes/src/static_resource.rs +++ b/src/runtime-rs/crates/resource/src/cpu_mem/initial_size.rs @@ -13,17 +13,16 @@ use kata_types::{ cpu::LinuxContainerCpuResources, k8s::container_type, }; -// static resource that StaticResourceManager needs, this is the spec for the +// initial resource that InitialSizeManager needs, this is the spec for the // sandbox/container's workload #[derive(Clone, Copy, Debug)] -struct StaticResource { +struct InitialSize { vcpu: u32, mem_mb: u32, } -// generate static resource(vcpu and memory in MiB) from spec's information -// used for static resource management -impl TryFrom<&oci::Spec> for StaticResource { +// generate initial resource(vcpu and memory in MiB) from spec's information +impl TryFrom<&oci::Spec> for InitialSize { type Error = anyhow::Error; fn try_from(spec: &oci::Spec) -> Result { let mut vcpu: u32 = 0; @@ -65,31 +64,32 @@ impl TryFrom<&oci::Spec> for StaticResource { } info!( sl!(), - "static resource mgmt result: vcpu={}, mem_mb={}", vcpu, mem_mb + "(from PodSandbox's annotation / SingleContainer's spec) initial size: vcpu={}, mem_mb={}", vcpu, mem_mb ); Ok(Self { vcpu, mem_mb }) } } -// StaticResourceManager is responsible for static resource management +// InitialSizeManager is responsible for initial vcpu/mem management // -// static resource management sizing information is optionally provided, either by +// inital vcpu/mem management sizing information is optionally provided, either by // upper layer runtime (containerd / crio) or by the container spec itself (when it // is a standalone single container such as the one started with *docker run*) // // the sizing information uses three values, cpu quota, cpu period and memory limit, -// and with above values it calculates the # vcpus and memory for the workload and -// add them to default value of the config +// and with above values it calculates the # vcpus and memory for the workload +// +// if the workload # of vcpus and memory is invalid for vmms, we still use default +// value in toml_config #[derive(Clone, Copy, Debug)] -pub struct StaticResourceManager { - resource: StaticResource, +pub struct InitialSizeManager { + resource: InitialSize, } -impl StaticResourceManager { +impl InitialSizeManager { pub fn new(spec: &oci::Spec) -> Result { Ok(Self { - resource: StaticResource::try_from(spec) - .context("failed to construct static resource")?, + resource: InitialSize::try_from(spec).context("failed to construct static resource")?, }) } @@ -100,8 +100,13 @@ impl StaticResourceManager { .hypervisor .get_mut(hypervisor_name) .context("failed to get hypervisor config")?; - hv.cpu_info.default_vcpus += self.resource.vcpu as i32; - hv.memory_info.default_memory += self.resource.mem_mb; + + if self.resource.vcpu > 0 { + hv.cpu_info.default_vcpus = self.resource.vcpu as i32 + } + if self.resource.mem_mb > 0 { + hv.memory_info.default_memory = self.resource.mem_mb; + } Ok(()) } } @@ -151,7 +156,7 @@ mod tests { struct TestData<'a> { desc: &'a str, input: InputData, - result: StaticResource, + result: InitialSize, } fn get_test_data() -> Vec> { @@ -163,7 +168,7 @@ mod tests { quota: None, memory: None, }, - result: StaticResource { vcpu: 0, mem_mb: 0 }, + result: InitialSize { vcpu: 0, mem_mb: 0 }, }, TestData { desc: "normal resource limit", @@ -173,7 +178,7 @@ mod tests { quota: Some(220_000), memory: Some(1024 * 1024 * 512), }, - result: StaticResource { + result: InitialSize { vcpu: 3, mem_mb: 512, }, @@ -183,7 +188,7 @@ mod tests { } #[test] - fn test_static_resource_mgmt_sandbox() { + fn test_initial_size_sandbox() { let tests = get_test_data(); // run tests @@ -210,22 +215,22 @@ mod tests { ..Default::default() }; - let static_resource = StaticResource::try_from(&spec); + let initial_size = InitialSize::try_from(&spec); assert!( - static_resource.is_ok(), + initial_size.is_ok(), "test[{}]: {:?} should be ok", i, d.desc ); - let static_resource = static_resource.unwrap(); + let initial_size = initial_size.unwrap(); assert_eq!( - static_resource.vcpu, d.result.vcpu, + initial_size.vcpu, d.result.vcpu, "test[{}]: {:?} vcpu should be {}", i, d.desc, d.result.vcpu, ); assert_eq!( - static_resource.mem_mb, d.result.mem_mb, + initial_size.mem_mb, d.result.mem_mb, "test[{}]: {:?} memory should be {}", i, d.desc, d.result.mem_mb, ); @@ -233,7 +238,7 @@ mod tests { } #[test] - fn test_static_resource_mgmt_container() { + fn test_initial_size_container() { let tests = get_test_data(); // run tests @@ -261,22 +266,22 @@ mod tests { ..Default::default() }; - let static_resource = StaticResource::try_from(&spec); + let initial_size = InitialSize::try_from(&spec); assert!( - static_resource.is_ok(), + initial_size.is_ok(), "test[{}]: {:?} should be ok", i, d.desc ); - let static_resource = static_resource.unwrap(); + let initial_size = initial_size.unwrap(); assert_eq!( - static_resource.vcpu, d.result.vcpu, + initial_size.vcpu, d.result.vcpu, "test[{}]: {:?} vcpu should be {}", i, d.desc, d.result.vcpu, ); assert_eq!( - static_resource.mem_mb, d.result.mem_mb, + initial_size.mem_mb, d.result.mem_mb, "test[{}]: {:?} memory should be {}", i, d.desc, d.result.mem_mb, ); diff --git a/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs new file mode 100644 index 000000000..f2984cd1c --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cpu_mem/mod.rs @@ -0,0 +1,8 @@ +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub mod cpu; +pub mod initial_size; diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs index b118e2e38..4e4aae9e8 100644 --- a/src/runtime-rs/crates/resource/src/lib.rs +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -22,6 +22,7 @@ pub mod rootfs; pub mod share_fs; pub mod volume; pub use manager::ResourceManager; +pub mod cpu_mem; use kata_types::config::hypervisor::SharedFsInfo; @@ -30,3 +31,10 @@ pub enum ResourceConfig { Network(NetworkConfig), ShareFs(SharedFsInfo), } + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ResourceUpdateOp { + Add, + Del, + Update, +} diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index a022f722f..d79de40cd 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -4,9 +4,8 @@ // SPDX-License-Identifier: Apache-2.0 // -use crate::network::NetworkConfig; -use crate::resource_persist::ResourceState; -use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig}; +use std::sync::Arc; + use agent::types::Device; use agent::{Agent, Storage}; use anyhow::Result; @@ -17,9 +16,13 @@ use kata_types::config::TomlConfig; use kata_types::mount::Mount; use oci::{Linux, LinuxResources}; use persist::sandbox_persist::Persist; -use std::sync::Arc; use tokio::sync::RwLock; +use crate::network::NetworkConfig; +use crate::resource_persist::ResourceState; +use crate::ResourceUpdateOp; +use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig}; + pub struct ManagerArgs { pub sid: String, pub agent: Arc, @@ -110,13 +113,14 @@ impl ResourceManager { inner.dump().await } - pub async fn update_cgroups( + pub async fn update_linux_resource( &self, cid: &str, linux_resources: Option<&LinuxResources>, - ) -> Result<()> { + op: ResourceUpdateOp, + ) -> Result> { let inner = self.inner.read().await; - inner.update_cgroups(cid, linux_resources).await + inner.update_linux_resource(cid, linux_resources, op).await } pub async fn cleanup(&self) -> Result<()> { diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 7e50485ae..b60925c11 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -4,30 +4,34 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::{sync::Arc, thread, vec}; +use std::{sync::Arc, thread}; -use crate::{network::NetworkConfig, resource_persist::ResourceState}; use agent::{types::Device, Agent, Storage}; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; use hypervisor::{ - device::{device_manager::DeviceManager, DeviceConfig}, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, DeviceType, + }, BlockConfig, Hypervisor, }; use kata_types::config::TomlConfig; use kata_types::mount::Mount; -use oci::{Linux, LinuxResources}; +use oci::{Linux, LinuxCpu, LinuxResources}; use persist::sandbox_persist::Persist; use tokio::{runtime, sync::RwLock}; use crate::{ cgroups::{CgroupArgs, CgroupsResource}, + cpu_mem::cpu::CpuResource, manager::ManagerArgs, - network::{self, Network}, + network::{self, Network, NetworkConfig}, + resource_persist::ResourceState, rootfs::{RootFsResource, Rootfs}, - share_fs::{self, ShareFs}, + share_fs::{self, sandbox_bind_mounts::SandboxBindMounts, ShareFs}, volume::{Volume, VolumeResource}, - ResourceConfig, + ResourceConfig, ResourceUpdateOp, }; pub(crate) struct ResourceManagerInner { @@ -42,6 +46,7 @@ pub(crate) struct ResourceManagerInner { pub rootfs_resource: RootFsResource, pub volume_resource: VolumeResource, pub cgroups_resource: CgroupsResource, + pub cpu_resource: CpuResource, } impl ResourceManagerInner { @@ -51,12 +56,12 @@ impl ResourceManagerInner { hypervisor: Arc, toml_config: Arc, ) -> Result { - let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; - // create device manager let dev_manager = DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?; + let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; + let cpu_resource = CpuResource::new(toml_config.clone())?; Ok(Self { sid: sid.to_string(), toml_config, @@ -68,6 +73,7 @@ impl ResourceManagerInner { rootfs_resource: RootFsResource::new(), volume_resource: VolumeResource::new(), cgroups_resource, + cpu_resource, }) } @@ -97,6 +103,12 @@ impl ResourceManagerInner { .setup_device_before_start_vm(self.hypervisor.as_ref()) .await .context("setup share fs device before start vm")?; + + // setup sandbox bind mounts: setup = true + self.handle_sandbox_bindmounts(true) + .await + .context("failed setup sandbox bindmounts")?; + Some(share_fs) } else { None @@ -249,6 +261,7 @@ impl ResourceManagerInner { spec, self.device_manager.as_ref(), &self.sid, + self.agent.clone(), ) .await } @@ -258,42 +271,23 @@ impl ResourceManagerInner { for d in linux.devices.iter() { match d.r#type.as_str() { "b" => { - let device_info = DeviceConfig::BlockCfg(BlockConfig { + let dev_info = DeviceConfig::BlockCfg(BlockConfig { major: d.major, minor: d.minor, ..Default::default() }); - let device_id = self - .device_manager - .write() - .await - .new_device(&device_info) - .await - .context("failed to create deviec")?; - self.device_manager - .write() + let device_info = do_handle_device(&self.device_manager, &dev_info) .await - .try_add_device(&device_id) - .await - .context("failed to add deivce")?; - - // get complete device information - let dev_info = self - .device_manager - .read() - .await - .get_device_info(&device_id) - .await - .context("failed to get device info")?; + .context("do handle device")?; // create agent device - if let DeviceConfig::BlockCfg(config) = dev_info { + if let DeviceType::Block(device) = device_info { let agent_device = Device { - id: device_id.clone(), + id: device.device_id.clone(), container_path: d.path.clone(), - field_type: config.driver_option, - vm_path: config.virt_path, + field_type: device.config.driver_option, + vm_path: device.config.virt_path, ..Default::default() }; devices.push(agent_device); @@ -308,14 +302,20 @@ impl ResourceManagerInner { Ok(devices) } - pub async fn update_cgroups( - &self, - cid: &str, - linux_resources: Option<&LinuxResources>, - ) -> Result<()> { - self.cgroups_resource - .update_cgroups(cid, linux_resources, self.hypervisor.as_ref()) - .await + async fn handle_sandbox_bindmounts(&self, setup: bool) -> Result<()> { + let bindmounts = self.toml_config.runtime.sandbox_bind_mounts.clone(); + if bindmounts.is_empty() { + info!(sl!(), "sandbox bindmounts empty, just skip it."); + return Ok(()); + } + + let sb_bindmnt = SandboxBindMounts::new(self.sid.clone(), bindmounts)?; + + if setup { + sb_bindmnt.setup_sandbox_bind_mounts() + } else { + sb_bindmnt.cleanup_sandbox_bind_mounts() + } } pub async fn cleanup(&self) -> Result<()> { @@ -324,6 +324,12 @@ impl ResourceManagerInner { .delete() .await .context("delete cgroup")?; + + // cleanup sandbox bind mounts: setup = false + self.handle_sandbox_bindmounts(false) + .await + .context("failed to cleanup sandbox bindmounts")?; + // clean up share fs mount if let Some(share_fs) = &self.share_fs { share_fs @@ -340,6 +346,57 @@ impl ResourceManagerInner { self.rootfs_resource.dump().await; self.volume_resource.dump().await; } + + pub async fn update_linux_resource( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + op: ResourceUpdateOp, + ) -> Result> { + let linux_cpus = || -> Option<&LinuxCpu> { linux_resources.as_ref()?.cpu.as_ref() }(); + + // if static_sandbox_resource_mgmt, we will not have to update sandbox's cpu or mem resource + if !self.toml_config.runtime.static_sandbox_resource_mgmt { + self.cpu_resource + .update_cpu_resources( + cid, + linux_cpus, + op, + self.hypervisor.as_ref(), + self.agent.as_ref(), + ) + .await?; + } + + // we should firstly update the vcpus and mems, and then update the host cgroups + self.cgroups_resource + .update_cgroups(cid, linux_resources, op, self.hypervisor.as_ref()) + .await?; + + // update the linux resources for agent + self.agent_linux_resources(linux_resources) + } + + fn agent_linux_resources( + &self, + linux_resources: Option<&LinuxResources>, + ) -> Result> { + let mut resources = match linux_resources { + Some(linux_resources) => linux_resources.clone(), + None => { + return Ok(None); + } + }; + + // clear the cpuset + // for example, if there are only 5 vcpus now, and the cpuset in LinuxResources is 0-2,6, guest os will report + // error when creating the container. so we choose to clear the cpuset here. + if let Some(cpu) = &mut resources.cpu { + cpu.cpus = String::new(); + } + + Ok(Some(resources)) + } } #[async_trait] @@ -386,6 +443,7 @@ impl Persist for ResourceManagerInner { ) .await?, toml_config: Arc::new(TomlConfig::default()), + cpu_resource: CpuResource::default(), }) } } diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs index 0e161fcb3..bcb31d957 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs @@ -10,7 +10,10 @@ use agent::Storage; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use hypervisor::{ - device::{device_manager::DeviceManager, DeviceConfig}, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, DeviceType, + }, BlockConfig, }; use kata_types::mount::Mount; @@ -46,18 +49,10 @@ impl BlockRootfs { ..Default::default() }; - let device_id = d - .write() + // create and insert block device into Kata VM + let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone())) .await - .new_device(&DeviceConfig::BlockCfg(block_device_config.clone())) - .await - .context("failed to create deviec")?; - - d.write() - .await - .try_add_device(device_id.as_str()) - .await - .context("failed to add deivce")?; + .context("do handle device failed.")?; let mut storage = Storage { fs_type: rootfs.fs_type.clone(), @@ -66,17 +61,11 @@ impl BlockRootfs { ..Default::default() }; - // get complete device information - let dev_info = d - .read() - .await - .get_device_info(device_id.as_str()) - .await - .context("failed to get device info")?; - - if let DeviceConfig::BlockCfg(config) = dev_info { - storage.driver = config.driver_option; - storage.source = config.virt_path; + let mut device_id: String = "".to_owned(); + if let DeviceType::Block(device) = device_info { + storage.driver = device.config.driver_option; + storage.source = device.config.virt_path; + device_id = device.device_id; } Ok(Self { diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs index 12bb64420..4d70a6c7b 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs @@ -18,6 +18,7 @@ pub use utils::{ mod virtio_fs_share_mount; use virtio_fs_share_mount::VirtiofsShareMount; pub use virtio_fs_share_mount::EPHEMERAL_PATH; +pub mod sandbox_bind_mounts; use std::{collections::HashMap, fmt::Debug, path::PathBuf, sync::Arc}; diff --git a/src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs b/src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs new file mode 100644 index 000000000..13bd28103 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs @@ -0,0 +1,155 @@ +// Copyright (c) 2023 Alibaba Cloud +// Copyright (c) 2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +// Note: +// sandbox_bind_mounts supports kinds of mount patterns, for example: +// (1) "/path/to", with default readonly mode. +// (2) "/path/to:ro", same as (1). +// (3) "/path/to:rw", with readwrite mode. +// +// sandbox_bind_mounts: ["/path/to", "/path/to:rw", "/mnt/to:ro"] +// + +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context, Result}; + +use super::utils::{do_get_host_path, mkdir_with_permissions}; +use kata_sys_util::{fs::get_base_name, mount}; +use kata_types::mount::{SANDBOX_BIND_MOUNTS_DIR, SANDBOX_BIND_MOUNTS_RO, SANDBOX_BIND_MOUNTS_RW}; + +#[derive(Clone, Default, Debug)] +pub struct SandboxBindMounts { + sid: String, + host_mounts_path: PathBuf, + sandbox_bindmounts: Vec, +} + +impl SandboxBindMounts { + pub fn new(sid: String, sandbox_bindmounts: Vec) -> Result { + // /run/kata-containers/shared/sandboxes//rw/passthrough/sandbox-mounts + let bindmounts_path = + do_get_host_path(SANDBOX_BIND_MOUNTS_DIR, sid.as_str(), "", true, false); + let host_mounts_path = PathBuf::from(bindmounts_path); + + Ok(SandboxBindMounts { + sid, + host_mounts_path, + sandbox_bindmounts, + }) + } + + fn parse_sandbox_bind_mounts<'a>(&self, bindmnt_src: &'a str) -> Result<(&'a str, &'a str)> { + // get the bindmount's r/w mode + let bindmount_mode = if bindmnt_src.ends_with(SANDBOX_BIND_MOUNTS_RW) { + SANDBOX_BIND_MOUNTS_RW + } else { + SANDBOX_BIND_MOUNTS_RO + }; + + // get the true bindmount from the string + let bindmount = bindmnt_src.trim_end_matches(bindmount_mode); + + Ok((bindmount_mode, bindmount)) + } + + pub fn setup_sandbox_bind_mounts(&self) -> Result<()> { + let mut mounted_list: Vec = Vec::new(); + let mut mounted_map: HashMap = HashMap::new(); + for src in &self.sandbox_bindmounts { + let (bindmount_mode, bindmount) = self + .parse_sandbox_bind_mounts(src) + .context("parse sandbox bind mounts failed")?; + + // get the basename of the canonicalized mount path mnt_name: dirX + let mnt_name = get_base_name(bindmount)? + .into_string() + .map_err(|e| anyhow!("failed to get base name {:?}", e))?; + + // if repeated mounted, do umount it and return error + if mounted_map.insert(mnt_name.clone(), true).is_some() { + for p in &mounted_list { + nix::mount::umount(p) + .context("mounted_map insert one repeated mounted, do umount it")?; + } + + return Err(anyhow!( + "sandbox-bindmounts: path {} is already specified.", + bindmount + )); + } + + // mount_dest: /run/kata-containers/shared/sandboxes//rw/passthrough/sandbox-mounts/dirX + let mount_dest = self.host_mounts_path.clone().join(mnt_name.as_str()); + mkdir_with_permissions(self.host_mounts_path.clone().to_path_buf(), 0o750).context( + format!( + "create host mounts path {:?}", + self.host_mounts_path.clone() + ), + )?; + + info!( + sl!(), + "sandbox-bindmounts mount_src: {:?} => mount_dest: {:?}", bindmount, &mount_dest + ); + + // mount -o bind,ro host_shared mount_dest + // host_shared: ${bindmount} + mount::bind_mount_unchecked(Path::new(bindmount), &mount_dest, true).map_err(|e| { + for p in &mounted_list { + nix::mount::umount(p).unwrap_or_else(|x| { + format!("do umount failed: {:?}", x); + }); + } + e + })?; + + // default sandbox bind mounts mode is ro. + if bindmount_mode == SANDBOX_BIND_MOUNTS_RO { + info!(sl!(), "sandbox readonly bind mount."); + // dest_ro: /run/kata-containers/shared/sandboxes//ro/passthrough/sandbox-mounts + let mount_dest_ro = + do_get_host_path(SANDBOX_BIND_MOUNTS_DIR, &self.sid, "", true, true); + let sandbox_bindmounts_ro = [mount_dest_ro, mnt_name.clone()].join("/"); + + mount::bind_remount(sandbox_bindmounts_ro, true) + .context("remount ro directory with ro permission")?; + } + + mounted_list.push(mount_dest); + } + + Ok(()) + } + + pub fn cleanup_sandbox_bind_mounts(&self) -> Result<()> { + for src in &self.sandbox_bindmounts { + let parsed_mnts = self + .parse_sandbox_bind_mounts(src) + .context("parse sandbox bind mounts")?; + + let mnt_name = get_base_name(parsed_mnts.1)? + .into_string() + .map_err(|e| anyhow!("failed to convert to string{:?}", e))?; + + // /run/kata-containers/shared/sandboxes//passthrough/rw/sandbox-mounts/dir + let mnt_dest = self.host_mounts_path.join(mnt_name.as_str()); + mount::umount_timeout(mnt_dest, 0).context("umount bindmount failed")?; + } + + if fs::metadata(self.host_mounts_path.clone())?.is_dir() { + fs::remove_dir_all(self.host_mounts_path.clone()).context(format!( + "remove sandbox bindmount point {:?}.", + self.host_mounts_path.clone() + ))?; + } + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs index 81ab8b7fd..9a5676bdf 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -56,6 +56,7 @@ pub(crate) async fn prepare_virtiofs( fs_type: fs_type.to_string(), queue_size: 0, queue_num: 0, + options: vec![], }, }; h.add_device(DeviceType::ShareFs(share_fs_device)) diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs index c93cbec54..3300c74ef 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -4,13 +4,27 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::path::{Path, PathBuf}; +use std::{ + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; use anyhow::Result; use kata_sys_util::mount; use super::*; +pub(crate) fn mkdir_with_permissions(path_target: PathBuf, mode: u32) -> Result<()> { + let new_path = &path_target; + std::fs::create_dir_all(new_path) + .context(format!("unable to create new path: {:?}", new_path))?; + + // mode format: 0o750, ... + std::fs::set_permissions(new_path, std::fs::Permissions::from_mode(mode))?; + + Ok(()) +} + pub(crate) fn ensure_dir_exist(path: &Path) -> Result<()> { if !path.exists() { std::fs::create_dir_all(path).context(format!("failed to create directory {:?}", path))?; diff --git a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs index 2cea9904d..6f875d29e 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs @@ -12,7 +12,6 @@ use kata_types::k8s::is_watchable_mount; use kata_types::mount; use nix::sys::stat::stat; use std::fs; -use std::os::unix::fs::PermissionsExt; use std::path::Path; const WATCHABLE_PATH_NAME: &str = "watchable"; @@ -21,7 +20,10 @@ pub const EPHEMERAL_PATH: &str = "/run/kata-containers/sandbox/ephemeral"; use super::{ get_host_rw_shared_path, - utils::{self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path}, + utils::{ + self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path, + mkdir_with_permissions, + }, ShareFsMount, ShareFsMountResult, ShareFsRootfsConfig, ShareFsVolumeConfig, KATA_GUEST_SHARE_DIR, PASSTHROUGH_FS_DIR, }; @@ -79,13 +81,11 @@ impl ShareFsMount for VirtiofsShareMount { .join(PASSTHROUGH_FS_DIR) .join(WATCHABLE_PATH_NAME); - fs::create_dir_all(&watchable_host_path).context(format!( - "unable to create watchable path: {:?}", - &watchable_host_path, + mkdir_with_permissions(watchable_host_path.clone(), 0o750).context(format!( + "unable to create watchable path {:?}", + watchable_host_path ))?; - fs::set_permissions(watchable_host_path, fs::Permissions::from_mode(0o750))?; - // path: /run/kata-containers/shared/containers/passthrough/watchable/config-map-name let file_name = Path::new(&guest_path) .file_name() diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs index 14f09fc81..ad1c01f17 100644 --- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs @@ -4,29 +4,32 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::Result; +use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use std::{collections::HashMap, fs, path::Path}; +use nix::sys::{stat, stat::SFlag}; +use tokio::sync::RwLock; -use crate::share_fs::{do_get_guest_path, do_get_host_path}; - -use super::{share_fs_volume::generate_mount_path, Volume}; -use agent::Storage; -use anyhow::{anyhow, Context}; +use super::Volume; +use crate::volume::utils::{ + generate_shared_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE, KATA_DIRECT_VOLUME_TYPE, + KATA_MOUNT_BIND_TYPE, +}; use hypervisor::{ - device::{device_manager::DeviceManager, DeviceConfig}, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, DeviceType, + }, BlockConfig, }; -use nix::sys::stat::{self, SFlag}; -use tokio::sync::RwLock; -#[derive(Debug)] + +#[derive(Clone)] pub(crate) struct BlockVolume { storage: Option, mount: oci::Mount, device_id: String, } -/// BlockVolume: block device volume +/// BlockVolume for bind-mount block volume and direct block volume impl BlockVolume { pub(crate) async fn new( d: &RwLock, @@ -35,54 +38,71 @@ impl BlockVolume { cid: &str, sid: &str, ) -> Result { - let fstat = stat::stat(m.source.as_str()).context(format!("stat {}", m.source))?; - info!(sl!(), "device stat: {:?}", fstat); - let mut options = HashMap::new(); - if read_only { - options.insert("read_only".to_string(), "true".to_string()); - } + let mnt_src: &str = &m.source; + // default block device fs type: ext4. + let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string(); - let block_device_config = &mut BlockConfig { - major: stat::major(fstat.st_rdev) as i64, - minor: stat::minor(fstat.st_rdev) as i64, - ..Default::default() + let block_device_config = match m.r#type.as_str() { + KATA_MOUNT_BIND_TYPE => { + let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?; + + BlockConfig { + major: stat::major(fstat.st_rdev) as i64, + minor: stat::minor(fstat.st_rdev) as i64, + ..Default::default() + } + } + KATA_DIRECT_VOLUME_TYPE => { + // get volume mountinfo from mountinfo.json + let v = volume_mount_info(mnt_src) + .context("deserde information from mountinfo.json")?; + // check volume type + if v.volume_type != KATA_DIRECT_VOLUME_TYPE { + return Err(anyhow!("volume type {:?} is invalid", v.volume_type)); + } + + let fstat = stat::stat(v.device.as_str()) + .with_context(|| format!("stat volume device file: {}", v.device.clone()))?; + if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG + && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK + { + return Err(anyhow!( + "invalid volume device {:?} for volume type {:?}", + v.device, + v.volume_type + )); + } + + blk_dev_fstype = v.fs_type.clone(); + + BlockConfig { + path_on_host: v.device, + ..Default::default() + } + } + _ => { + return Err(anyhow!( + "unsupport direct block volume r#type: {:?}", + m.r#type.as_str() + )) + } }; - let device_id = d - .write() + // create and insert block device into Kata VM + let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone())) .await - .new_device(&DeviceConfig::BlockCfg(block_device_config.clone())) - .await - .context("failed to create deviec")?; + .context("do handle device failed.")?; - d.write() + // generate host guest shared path + let guest_path = generate_shared_path(m.destination.clone(), read_only, cid, sid) .await - .try_add_device(device_id.as_str()) - .await - .context("failed to add deivce")?; - - let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); - let file_name = generate_mount_path(cid, file_name); - let guest_path = do_get_guest_path(&file_name, cid, true, false); - let host_path = do_get_host_path(&file_name, sid, cid, true, read_only); - fs::create_dir_all(&host_path) - .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?; - - // get complete device information - let dev_info = d - .read() - .await - .get_device_info(&device_id) - .await - .context("failed to get device info")?; + .context("generate host-guest shared path failed")?; // storage - let mut storage = Storage::default(); - - if let DeviceConfig::BlockCfg(config) = dev_info { - storage.driver = config.driver_option; - storage.source = config.virt_path; - } + let mut storage = agent::Storage { + mount_point: guest_path.clone(), + ..Default::default() + }; storage.options = if read_only { vec!["ro".to_string()] @@ -90,21 +110,32 @@ impl BlockVolume { Vec::new() }; - storage.mount_point = guest_path.clone(); - - // If the volume had specified the filesystem type, use it. Otherwise, set it - // to ext4 since but right now we only support it. - if m.r#type != "bind" { - storage.fs_type = m.r#type.clone(); - } else { - storage.fs_type = "ext4".to_string(); + // As the true Block Device wrapped in DeviceType, we need to + // get it out from the wrapper, and the device_id will be for + // BlockVolume. + // safe here, device_info is correct and only unwrap it. + let mut device_id = String::new(); + if let DeviceType::Block(device) = device_info { + // blk, mmioblk + storage.driver = device.config.driver_option; + // /dev/vdX + storage.source = device.config.virt_path; + device_id = device.device_id; + } + + // In some case, dest is device /dev/xxx + if m.destination.clone().starts_with("/dev") { + storage.fs_type = "bind".to_string(); + storage.options.append(&mut m.options.clone()); + } else { + // usually, the dest is directory. + storage.fs_type = blk_dev_fstype; } - // mount let mount = oci::Mount { destination: m.destination.clone(), - r#type: m.r#type.clone(), - source: guest_path.clone(), + r#type: storage.fs_type.clone(), + source: guest_path, options: m.options.clone(), }; @@ -128,6 +159,7 @@ impl Volume for BlockVolume { } else { vec![] }; + Ok(s) } @@ -144,13 +176,22 @@ impl Volume for BlockVolume { } } -pub(crate) fn is_block_volume(m: &oci::Mount) -> bool { - if m.r#type != "bind" { - return false; +pub(crate) fn is_block_volume(m: &oci::Mount) -> Result { + let vol_types = vec![KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE]; + if !vol_types.contains(&m.r#type.as_str()) { + return Ok(false); } - if let Ok(fstat) = stat::stat(m.source.as_str()).context(format!("stat {}", m.source)) { - info!(sl!(), "device stat: {:?}", fstat); - return SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK; + + let fstat = + stat::stat(m.source.as_str()).context(format!("stat mount source {} failed.", m.source))?; + let s_flag = SFlag::from_bits_truncate(fstat.st_mode); + + match m.r#type.as_str() { + // case: mount bind and block device + KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true), + // case: directvol and directory + KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true), + // else: unsupported or todo for other volume type. + _ => Ok(false), } - false } diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index ea7d4f5d6..6fc6e3bc8 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -9,16 +9,18 @@ mod default_volume; pub mod hugepage; mod share_fs_volume; mod shm_volume; -use async_trait::async_trait; +pub mod utils; + +use std::{sync::Arc, vec::Vec}; use anyhow::{Context, Result}; -use hypervisor::device::device_manager::DeviceManager; -use std::{sync::Arc, vec::Vec}; +use async_trait::async_trait; use tokio::sync::RwLock; -use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; - use self::hugepage::{get_huge_page_limits_map, get_huge_page_option}; +use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; +use agent::Agent; +use hypervisor::device::device_manager::DeviceManager; const BIND: &str = "bind"; @@ -52,6 +54,7 @@ impl VolumeResource { spec: &oci::Spec, d: &RwLock, sid: &str, + agent: Arc, ) -> Result>> { let mut volumes: Vec> = vec![]; let oci_mounts = &spec.mounts; @@ -65,7 +68,7 @@ impl VolumeResource { shm_volume::ShmVolume::new(m, shm_size) .with_context(|| format!("new shm volume {:?}", m))?, ) - } else if is_block_volume(m) { + } else if is_block_volume(m).context("block volume type")? { // handle block volume Arc::new( block_volume::BlockVolume::new(d, m, read_only, cid, sid) @@ -85,7 +88,7 @@ impl VolumeResource { ) } else if share_fs_volume::is_share_fs_volume(m) { Arc::new( - share_fs_volume::ShareFsVolume::new(share_fs, m, cid, read_only) + share_fs_volume::ShareFsVolume::new(share_fs, m, cid, read_only, agent.clone()) .await .with_context(|| format!("new share fs volume {:?}", m))?, ) diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs index 0748e374d..098dc399f 100644 --- a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs @@ -5,11 +5,15 @@ // use std::{ + fs::File, + io::Read, + os::unix::fs::MetadataExt, path::{Path, PathBuf}, str::FromStr, sync::Arc, }; +use agent::Agent; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use hypervisor::device::device_manager::DeviceManager; @@ -19,6 +23,9 @@ use super::Volume; use crate::share_fs::{MountedInfo, ShareFs, ShareFsVolumeConfig}; use kata_types::mount; +use crate::share_fs::DEFAULT_KATA_GUEST_SANDBOX_DIR; +use crate::share_fs::PASSTHROUGH_FS_DIR; + const SYS_MOUNT_PREFIX: [&str; 2] = ["/proc", "/sys"]; // copy file to container's rootfs if filesystem sharing is not supported, otherwise @@ -39,6 +46,7 @@ impl ShareFsVolume { m: &oci::Mount, cid: &str, readonly: bool, + agent: Arc, ) -> Result { // The file_name is in the format of "sandbox-{uuid}-{file_name}" let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); @@ -61,11 +69,67 @@ impl ShareFsVolume { Ok(src) => src, }; + // If the mount source is a file, we can copy it to the sandbox if src.is_file() { - // TODO: copy file - debug!(sl!(), "FIXME: copy file {}", &m.source); - } else { + // This is where we set the value for the guest path + let dest = [ + DEFAULT_KATA_GUEST_SANDBOX_DIR, + PASSTHROUGH_FS_DIR, + file_name.clone().as_str(), + ] + .join("/"); + debug!( + sl!(), + "copy local file {:?} to guest {:?}", + &m.source, + dest.clone() + ); + + // Read file metadata + let file_metadata = std::fs::metadata(src.clone()) + .with_context(|| format!("Failed to read metadata from file: {:?}", src))?; + + // Open file + let mut file = File::open(&src) + .with_context(|| format!("Failed to open file: {:?}", src))?; + + // Open read file contents to buffer + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer) + .with_context(|| format!("Failed to read file: {:?}", src))?; + + // Create gRPC request + let r = agent::CopyFileRequest { + path: dest.clone(), + file_size: file_metadata.len() as i64, + uid: file_metadata.uid() as i32, + gid: file_metadata.gid() as i32, + file_mode: file_metadata.mode(), + data: buffer, + ..Default::default() + }; + + debug!(sl!(), "copy_file: {:?} to sandbox {:?}", &src, dest.clone()); + + // Issue gRPC request to agent + agent.copy_file(r).await.with_context(|| { + format!( + "copy file request failed: src: {:?}, dest: {:?}", + file_name, dest + ) + })?; + + // append oci::Mount structure to volume mounts + volume.mounts.push(oci::Mount { + destination: m.destination.clone(), + r#type: "bind".to_string(), + source: dest.clone(), + options: m.options.clone(), + }) + } else { + // If not, we can ignore it. Let's issue a warning so that the user knows. + warn!( sl!(), "Ignoring non-regular file as FS sharing not supported. mount: {:?}", m ); diff --git a/src/runtime-rs/crates/resource/src/volume/utils.rs b/src/runtime-rs/crates/resource/src/volume/utils.rs new file mode 100644 index 000000000..892724e1e --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/utils.rs @@ -0,0 +1,65 @@ +// Copyright (c) 2022-2023 Alibaba Cloud +// Copyright (c) 2022-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{fs, path::Path}; + +use anyhow::{anyhow, Context, Result}; + +use crate::{ + share_fs::{do_get_guest_path, do_get_host_path}, + volume::share_fs_volume::generate_mount_path, +}; +use kata_sys_util::eother; +use kata_types::mount::{get_volume_mount_info, DirectVolumeMountInfo}; + +pub const DEFAULT_VOLUME_FS_TYPE: &str = "ext4"; +pub const KATA_MOUNT_BIND_TYPE: &str = "bind"; +pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol"; +pub const KATA_VFIO_VOLUME_TYPE: &str = "vfiovol"; +pub const KATA_SPDK_VOLUME_TYPE: &str = "spdkvol"; + +// volume mount info load infomation from mountinfo.json +pub fn volume_mount_info(volume_path: &str) -> Result { + get_volume_mount_info(volume_path) +} + +pub fn get_file_name>(src: P) -> Result { + let file_name = src + .as_ref() + .file_name() + .map(|v| v.to_os_string()) + .ok_or_else(|| { + eother!( + "failed to get file name of path {}", + src.as_ref().to_string_lossy() + ) + })? + .into_string() + .map_err(|e| anyhow!("failed to convert to string {:?}", e))?; + + Ok(file_name) +} + +pub(crate) async fn generate_shared_path( + dest: String, + read_only: bool, + cid: &str, + sid: &str, +) -> Result { + let file_name = get_file_name(&dest).context("failed to get file name.")?; + let mount_name = generate_mount_path(cid, file_name.as_str()); + let guest_path = do_get_guest_path(&mount_name, cid, true, false); + let host_path = do_get_host_path(&mount_name, sid, cid, true, read_only); + + if dest.starts_with("/dev") { + fs::File::create(&host_path).context(format!("failed to create file {:?}", &host_path))?; + } else { + std::fs::create_dir_all(&host_path) + .map_err(|e| anyhow!("failed to create dir {}: {:?}", host_path, e))?; + } + + Ok(guest_path) +} diff --git a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs index c12df38b1..80e4149c3 100644 --- a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs +++ b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: Apache-2.0 // + use std::sync::Arc; use anyhow::Result; diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index efe06fa43..1a79f23d6 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -26,12 +26,10 @@ pub trait Sandbox: Send + Sync { async fn cleanup(&self) -> Result<()>; async fn shutdown(&self) -> Result<()>; - // agent function - async fn agent_sock(&self) -> Result; - // utils async fn set_iptables(&self, is_ipv6: bool, data: Vec) -> Result>; async fn get_iptables(&self, is_ipv6: bool) -> Result>; async fn direct_volume_stats(&self, volume_path: &str) -> Result; async fn direct_volume_resize(&self, resize_req: agent::ResizeVolumeRequest) -> Result<()>; + async fn agent_sock(&self) -> Result; } diff --git a/src/runtime-rs/crates/runtimes/src/lib.rs b/src/runtime-rs/crates/runtimes/src/lib.rs index dffa69697..867c8ef9e 100644 --- a/src/runtime-rs/crates/runtimes/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/src/lib.rs @@ -13,4 +13,3 @@ pub mod manager; pub use manager::RuntimeHandlerManager; pub use shim_interface; mod shim_mgmt; -mod static_resource; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 1fa7ce4a0..d05e1961f 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -6,7 +6,6 @@ use std::{path::PathBuf, str::from_utf8, sync::Arc}; -use crate::{shim_mgmt::server::MgmtServer, static_resource::StaticResourceManager}; use anyhow::{anyhow, Context, Result}; use common::{ message::Message, @@ -18,12 +17,11 @@ use kata_sys_util::spec::load_oci_spec; use kata_types::{ annotations::Annotation, config::default::DEFAULT_GUEST_DNS_FILE, config::TomlConfig, }; -use netns_rs::NetNs; -use resource::network::generate_netns_name; - #[cfg(feature = "linux")] use linux_container::LinuxContainer; +use netns_rs::NetNs; use persist::sandbox_persist::Persist; +use resource::{cpu_mem::initial_size::InitialSizeManager, network::generate_netns_name}; use shim_interface::shim_mgmt::ERR_NO_SHIM_SERVER; use tokio::fs; use tokio::sync::{mpsc::Sender, RwLock}; @@ -36,6 +34,8 @@ use virt_container::{ #[cfg(feature = "wasm")] use wasm_container::WasmContainer; +use crate::shim_mgmt::server::MgmtServer; + struct RuntimeHandlerManagerInner { id: String, msg_sender: Sender, @@ -422,14 +422,11 @@ fn load_config(spec: &oci::Spec, option: &Option>) -> Result // 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."), // then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is // a single container. - if toml_config.runtime.static_sandbox_resource_mgmt { - info!(sl!(), "static resource management enabled"); - let static_resource_manager = StaticResourceManager::new(spec) - .context("failed to construct static resource manager")?; - static_resource_manager - .setup_config(&mut toml_config) - .context("failed to setup static resource mgmt config")?; - } + let initial_size_manager = + InitialSizeManager::new(spec).context("failed to construct static resource manager")?; + initial_size_manager + .setup_config(&mut toml_config) + .context("failed to setup static resource mgmt config")?; info!(sl!(), "get config content {:?}", &toml_config); Ok(toml_config) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 6dfea9fd0..fba45a784 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -19,7 +19,7 @@ use common::{ use kata_sys_util::k8s::update_ephemeral_storage_type; use oci::{LinuxResources, Process as OCIProcess}; -use resource::ResourceManager; +use resource::{ResourceManager, ResourceUpdateOp}; use tokio::sync::RwLock; use super::{ @@ -64,6 +64,10 @@ impl Container { config.stderr.clone(), config.terminal, ); + let linux_resources = spec + .linux + .as_ref() + .and_then(|linux| linux.resources.clone()); Ok(Self { pid, @@ -74,6 +78,7 @@ impl Container { agent.clone(), init_process, logger.clone(), + linux_resources, ))), agent, resource_manager, @@ -150,15 +155,18 @@ impl Container { .handler_devices(&config.container_id, linux) .await?; - // update cgroups - self.resource_manager - .update_cgroups( + // update vcpus, mems and host cgroups + let resources = self + .resource_manager + .update_linux_resource( &config.container_id, - spec.linux - .as_ref() - .and_then(|linux| linux.resources.as_ref()), + inner.linux_resources.as_ref(), + ResourceUpdateOp::Add, ) .await?; + if let Some(linux) = &mut spec.linux { + linux.resources = resources; + } // create container let r = agent::CreateContainerRequest { @@ -323,7 +331,20 @@ impl Container { inner .stop_process(container_process, true, &device_manager) .await - .context("stop process") + .context("stop process")?; + + // update vcpus, mems and host cgroups + if container_process.process_type == ProcessType::Container { + self.resource_manager + .update_linux_resource( + &self.config.container_id, + inner.linux_resources.as_ref(), + ResourceUpdateOp::Del, + ) + .await?; + } + + Ok(()) } pub async fn pause(&self) -> Result<()> { @@ -398,13 +419,21 @@ impl Container { } pub async fn update(&self, resources: &LinuxResources) -> Result<()> { - self.resource_manager - .update_cgroups(&self.config.container_id, Some(resources)) + let mut inner = self.inner.write().await; + inner.linux_resources = Some(resources.clone()); + // update vcpus, mems and host cgroups + let agent_resources = self + .resource_manager + .update_linux_resource( + &self.config.container_id, + Some(resources), + ResourceUpdateOp::Update, + ) .await?; let req = agent::UpdateContainerRequest { container_id: self.container_id.container_id.clone(), - resources: resources.clone(), + resources: agent_resources, mounts: Vec::new(), }; self.agent diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index 12d4810fb..bc478cbcd 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -14,6 +14,7 @@ use common::{ }; use hypervisor::device::device_manager::DeviceManager; use nix::sys::signal::Signal; +use oci::LinuxResources; use resource::{rootfs::Rootfs, volume::Volume}; use tokio::sync::RwLock; @@ -32,10 +33,16 @@ pub struct ContainerInner { pub(crate) exec_processes: HashMap, pub(crate) rootfs: Vec>, pub(crate) volumes: Vec>, + pub(crate) linux_resources: Option, } impl ContainerInner { - pub(crate) fn new(agent: Arc, init_process: Process, logger: slog::Logger) -> Self { + pub(crate) fn new( + agent: Arc, + init_process: Process, + logger: slog::Logger, + linux_resources: Option, + ) -> Self { Self { agent, logger, @@ -43,6 +50,7 @@ impl ContainerInner { exec_processes: HashMap::new(), rootfs: vec![], volumes: vec![], + linux_resources, } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index f85e7901a..f5aa05e6c 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -6,6 +6,7 @@ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; + use std::{collections::HashMap, sync::Arc}; use agent::Agent; @@ -96,7 +97,6 @@ impl ContainerManager for VirtContainerManager { let mut containers = self.containers.write().await; container.create(spec).await.context("create")?; containers.insert(container.container_id.to_string(), container); - Ok(PID { pid: self.pid }) } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 8202d854d..3cbdba493 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -361,7 +361,7 @@ impl Sandbox for VirtSandbox { .await .context("resource clean up")?; - // TODO: cleanup other snadbox resource + // TODO: cleanup other sandbox resource Ok(()) } diff --git a/src/runtime/Makefile b/src/runtime/Makefile index c15fb146e..17e36ed09 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -254,7 +254,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024 # # see `virtiofsd -h` for possible options. # Make sure you quote args. -DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"-o\", \"announce_submounts\"] +DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"--announce-submounts\"] DEFENABLEIOTHREADS := false DEFENABLEVHOSTUSERSTORE := false DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user @@ -387,31 +387,6 @@ ifneq (,$(QEMUCMD)) CONFIGS += $(CONFIG_REMOTE) - - CONFIG_FILE_QEMU_GPU = configuration-qemu-gpu.toml - CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU) - CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in - - CONFIG_PATH_QEMU_SEV = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SEV)) - CONFIG_PATHS += $(CONFIG_PATH_QEMU_SEV) - - SYSCONFIG_QEMU_SEV = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SEV)) - SYSCONFIG_PATHS_SEV += $(SYSCONFIG_QEMU_SEV) - - CONFIGS += $(CONFIG_QEMU_SEV) - - CONFIG_FILE_QEMU_SNP = configuration-qemu-snp.toml - CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP) - CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in - - CONFIG_PATH_QEMU_SNP = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SNP)) - CONFIG_PATHS += $(CONFIG_PATH_QEMU_SNP) - - SYSCONFIG_QEMU_SNP = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SNP)) - SYSCONFIG_PATHS_SNP += $(SYSCONFIG_QEMU_SNP) - - CONFIGS += $(CONFIG_QEMU_SNP) - CONFIG_FILE_QEMU_NVIDIA_GPU = configuration-qemu-nvidia-gpu.toml CONFIG_QEMU_NVIDIA_GPU = config/$(CONFIG_FILE_QEMU_NVIDIA_GPU) CONFIG_QEMU_NVIDIA_GPU_IN = $(CONFIG_QEMU_NVIDIA_GPU).in diff --git a/src/runtime/README.md b/src/runtime/README.md index c8aeec0ce..d2290e200 100644 --- a/src/runtime/README.md +++ b/src/runtime/README.md @@ -32,8 +32,6 @@ to work seamlessly with both Docker and Kubernetes respectively. ## Download and install -[![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers) - See the [installation guides](../../docs/install/README.md) available for various operating systems. diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index bcaccd717..e7c115f92 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -150,9 +150,9 @@ virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ # Extra args for virtiofsd daemon # # Format example: -# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# ["--arg1=xxx", "--arg2=yyy"] # Examples: -# Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"] +# Set virtiofsd log level to debug : ["--log-level=debug"] # see `virtiofsd -h` for possible options. virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 7ef2cb610..bc0dcb736 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -225,9 +225,9 @@ virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ # Extra args for virtiofsd daemon # # Format example: -# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# ["--arg1=xxx", "--arg2=yyy"] # Examples: -# Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"] +# Set virtiofsd log level to debug : ["--log-level=debug"] # # see `virtiofsd -h` for possible options. virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index 43ea782c2..aeb79acfd 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -10,6 +10,7 @@ import ( "time" "context" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" @@ -120,7 +121,7 @@ type agent interface { // onlineCPUMem will online CPUs and Memory inside the Sandbox. // This function should be called after hot adding vCPUs or Memory. - // cpus specifies the number of CPUs that were added and the agent should online + // cpus specifies the number of CPUs that should be onlined in the guest, and special value 0 means agent will skip this check. // cpuOnly specifies that we should online cpu or online memory or both onlineCPUMem(ctx context.Context, cpus uint32, cpuOnly bool) error diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index a8dd81104..3a1b7c607 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1924,7 +1924,8 @@ type OnlineCPUMemRequest struct { // If true the agent returns once all resources have been connected, otherwise all // resources are connected asynchronously and the agent returns immediately. Wait bool `protobuf:"varint,1,opt,name=wait,proto3" json:"wait,omitempty"` - // NbCpus specifies the number of CPUs that were added and the agent has to online. + // NbCpus specifies the number of CPUs that should be onlined in the guest. + // Special value 0 means agent will skip this check. NbCpus uint32 `protobuf:"varint,2,opt,name=nb_cpus,json=nbCpus,proto3" json:"nb_cpus,omitempty"` // CpuOnly specifies whether only online CPU or not. CpuOnly bool `protobuf:"varint,3,opt,name=cpu_only,json=cpuOnly,proto3" json:"cpu_only,omitempty"` diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 302effc5d..6ded5cbe0 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1207,22 +1207,21 @@ func (q *qemu) StopVM(ctx context.Context, waitOnly bool) (err error) { return err } + pids := q.GetPids() + if len(pids) == 0 { + return errors.New("cannot determine QEMU PID") + } + pid := pids[0] + if waitOnly { - pids := q.GetPids() - if len(pids) == 0 { - return errors.New("cannot determine QEMU PID") - } - - pid := pids[0] - err := utils.WaitLocalProcess(pid, qemuStopSandboxTimeoutSecs, syscall.Signal(0), q.Logger()) if err != nil { return err } } else { - err := q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) + err = syscall.Kill(pid, syscall.SIGKILL) if err != nil { - q.Logger().WithError(err).Error("Fail to execute qmp QUIT") + q.Logger().WithError(err).Error("Fail to send SIGKILL to qemu") return err } } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 1d3b729ff..530713475 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -2131,9 +2131,8 @@ func (s *Sandbox) updateResources(ctx context.Context) error { s.Logger().Debugf("Request to hypervisor to update oldCPUs/newCPUs: %d/%d", oldCPUs, newCPUs) // If the CPUs were increased, ask agent to online them if oldCPUs < newCPUs { - vcpusAdded := newCPUs - oldCPUs - s.Logger().Debugf("Request to onlineCPUMem with %d CPUs", vcpusAdded) - if err := s.agent.onlineCPUMem(ctx, vcpusAdded, true); err != nil { + s.Logger().Debugf("Request to onlineCPUMem with %d CPUs", newCPUs) + if err := s.agent.onlineCPUMem(ctx, newCPUs, true); err != nil { return err } } diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index 6df62adf9..3e02756eb 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -186,16 +186,11 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) { // Send logs to syslog "--syslog", // cache mode for virtiofsd - "-o", "cache=" + v.cache, - // disable posix locking in daemon: bunch of basic posix locks properties are broken - // apt-get update is broken if enabled - "-o", "no_posix_lock", + "--cache=" + v.cache, // shared directory tree - "-o", "source=" + v.sourcePath, + "--shared-dir=" + v.sourcePath, // fd number of vhost-user socket fmt.Sprintf("--fd=%v", FdSocketNumber), - // foreground operation - "-f", } if len(v.extraArgs) != 0 { diff --git a/src/runtime/virtcontainers/virtiofsd_test.go b/src/runtime/virtcontainers/virtiofsd_test.go index a4252a2ba..c7d1e1e78 100644 --- a/src/runtime/virtcontainers/virtiofsd_test.go +++ b/src/runtime/virtcontainers/virtiofsd_test.go @@ -79,12 +79,12 @@ func TestVirtiofsdArgs(t *testing.T) { cache: "none", } - expected := "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=123 -f" + expected := "--syslog --cache=none --shared-dir=/run/kata-shared/foo --fd=123" args, err := v.args(123) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) - expected = "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=456 -f" + expected = "--syslog --cache=none --shared-dir=/run/kata-shared/foo --fd=456" args, err = v.args(456) assert.NoError(err) assert.Equal(expected, strings.Join(args, " ")) diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go index b5dec9912..72afa9581 100644 --- a/src/runtime/virtcontainers/vm.go +++ b/src/runtime/virtcontainers/vm.go @@ -293,7 +293,7 @@ func (v *VM) AddMemory(ctx context.Context, numMB uint32) error { // OnlineCPUMemory puts the hotplugged CPU and memory online. func (v *VM) OnlineCPUMemory(ctx context.Context) error { v.logger().Infof("online CPU %d and memory", v.cpuDelta) - err := v.agent.onlineCPUMem(ctx, v.cpuDelta, false) + err := v.agent.onlineCPUMem(ctx, v.cpu, false) if err == nil { v.cpuDelta = 0 } diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index 5b41886e7..cf54d4473 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -4,13 +4,19 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.18" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -31,15 +37,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arc-swap" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-broadcast" @@ -54,9 +60,9 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ "concurrent-queue", "event-listener", @@ -65,43 +71,43 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" dependencies = [ + "async-lock", "async-task", "concurrent-queue", "fastrand", "futures-lite", - "once_cell", "slab", ] [[package]] name = "async-io" -version = "1.9.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ + "async-lock", "autocfg", + "cfg-if 1.0.0", "concurrent-queue", "futures-lite", - "libc", "log", - "once_cell", "parking", "polling", + "rustix", "slab", "socket2", "waker-fn", - "winapi", ] [[package]] name = "async-lock" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", ] @@ -114,24 +120,24 @@ checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] name = "async-task" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" +checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] @@ -165,9 +171,9 @@ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitmask-enum" @@ -176,14 +182,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2" dependencies = [ "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byte-unit" @@ -199,42 +205,36 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" - -[[package]] -name = "cache-padded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "capctl" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eea0d91a34c56f0a0779e1cc2ec7040fa7f672819c4d3fe7d9dd4af3d2e78aca" +checksum = "fbdc32a78afc325d71a48d13084f1c3ddf67cc5dc06c6e5439a8630b14612cad" dependencies = [ "bitflags", + "cfg-if 1.0.0", "libc", ] [[package]] name = "caps" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61bf7211aad104ce2769ec05efcdfabf85ee84ac92461d142f22cf8badd0e54c" +checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" dependencies = [ - "errno 0.2.8", "libc", "thiserror", ] [[package]] name = "cc" -version = "1.0.72" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -263,15 +263,15 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", - "time", + "time 0.1.45", "wasm-bindgen", "winapi", ] @@ -299,24 +299,24 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" [[package]] name = "concurrent-queue" -version = "1.2.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" dependencies = [ - "cache-padded", + "crossbeam-utils", ] [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "crossbeam-channel" -version = "0.5.1" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -324,12 +324,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if 1.0.0", - "lazy_static", ] [[package]] @@ -340,7 +339,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] @@ -351,7 +350,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] @@ -363,6 +362,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.7" @@ -375,16 +384,27 @@ dependencies = [ ] [[package]] -name = "either" -version = "1.6.1" +name = "dirs-sys-next" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "enumflags2" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75d4cd21b95383444831539909fbb14b9dc3fdceb2a6f5d36577329a1f55ccb" +checksum = "c041f5090df68b32bcd905365fd51769c8b9d553fe87fde0b683534f10c01bd2" dependencies = [ "enumflags2_derive", "serde", @@ -392,24 +412,13 @@ dependencies = [ [[package]] name = "enumflags2_derive" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae" +checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", -] - -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", + "syn 2.0.18", ] [[package]] @@ -467,9 +476,9 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "futures" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -482,9 +491,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -492,15 +501,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -509,15 +518,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-lite" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ "fastrand", "futures-core", @@ -530,36 +539,33 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "autocfg", - "proc-macro-hack", "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "futures-sink" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.17" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ - "autocfg", "futures-channel", "futures-core", "futures-io", @@ -569,8 +575,6 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] @@ -582,7 +586,7 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -644,15 +648,25 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.46" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" dependencies = [ "android_system_properties", "core-foundation-sys", + "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", ] [[package]] @@ -709,24 +723,24 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.58" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -743,7 +757,7 @@ dependencies = [ "lazy_static", "libc", "logging", - "nix 0.23.1", + "nix 0.23.2", "oci", "protobuf 3.2.0", "protocols", @@ -768,7 +782,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix 0.24.2", + "nix 0.24.3", "oci", "once_cell", "rand", @@ -792,6 +806,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -820,9 +835,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -830,12 +845,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "logging" @@ -846,13 +858,14 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -870,7 +883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -882,9 +895,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "nix" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ "bitflags", "cc", @@ -895,9 +908,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -917,21 +930,11 @@ dependencies = [ "libc", ] -[[package]] -name = "num-integer" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -946,6 +949,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "oci" version = "0.1.0" @@ -958,9 +970,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.14.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-stream" @@ -974,9 +986,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" [[package]] name = "parking_lot" @@ -990,15 +1002,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.2.16", "smallvec", - "windows-sys 0.36.1", + "windows-sys 0.45.0", ] [[package]] @@ -1044,52 +1056,41 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "polling" -version = "2.3.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", + "bitflags", "cfg-if 1.0.0", + "concurrent-queue", "libc", "log", - "wepoll-ffi", - "winapi", + "pin-project-lite", + "windows-sys 0.48.0", ] [[package]] name = "ppv-lite86" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-crate" -version = "1.2.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "thiserror", - "toml", + "toml_edit", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -1132,7 +1133,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] @@ -1225,9 +1226,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -1293,9 +1294,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.6" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -1304,9 +1305,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.26" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "rlimit" @@ -1324,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ "bitflags", - "errno 0.3.1", + "errno", "io-lifetimes", "libc", "linux-raw-sys", @@ -1347,7 +1348,7 @@ dependencies = [ "kata-sys-util", "lazy_static", "libc", - "nix 0.24.2", + "nix 0.24.3", "oci", "path-absolutize", "protobuf 3.2.0", @@ -1367,10 +1368,23 @@ dependencies = [ ] [[package]] -name = "ryu" -version = "1.0.9" +name = "rustversion" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] [[package]] name = "scan_fmt" @@ -1389,29 +1403,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "serde_json" -version = "1.0.73" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1420,13 +1434,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] @@ -1446,18 +1460,21 @@ checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.5" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] [[package]] name = "slash-formatter" @@ -1485,14 +1502,14 @@ dependencies = [ [[package]] name = "slog-json" -version = "2.4.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9b96fb6b5e80e371423b4aca6656eb537661ce8f82c2697e619f8ca85d043" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" dependencies = [ - "chrono", "serde", "serde_json", "slog", + "time 0.3.21", ] [[package]] @@ -1507,10 +1524,23 @@ dependencies = [ ] [[package]] -name = "smallvec" -version = "1.9.0" +name = "slog-term" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.21", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" @@ -1546,20 +1576,20 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.82" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] name = "syn" -version = "2.0.14" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -1585,6 +1615,17 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1596,43 +1637,74 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if 1.0.0", "once_cell", ] [[package]] name = "time" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] +[[package]] +name = "time" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +dependencies = [ + "itoa", + "libc", + "num_threads", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", +] + [[package]] name = "tokio" version = "1.28.2" @@ -1658,23 +1730,40 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "toml" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] [[package]] -name = "tracing" -version = "0.1.36" +name = "toml_datetime" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" + +[[package]] +name = "toml_edit" +version = "0.19.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", @@ -1684,20 +1773,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", ] [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -1711,7 +1800,7 @@ dependencies = [ "byteorder", "libc", "log", - "nix 0.23.1", + "nix 0.23.2", "protobuf 3.2.0", "protobuf-codegen 3.2.0", "thiserror", @@ -1756,27 +1845,21 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-segmentation" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" - -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "vec_map" @@ -1790,6 +1873,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1798,9 +1887,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -1808,24 +1897,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1833,41 +1922,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.82", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" - -[[package]] -name = "wepoll-ffi" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" -dependencies = [ - "cc", -] +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "which" -version = "4.2.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", - "lazy_static", "libc", + "once_cell", ] [[package]] @@ -1893,16 +1973,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.36.1" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", + "windows-targets 0.48.0", ] [[package]] @@ -1965,12 +2041,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -1983,12 +2053,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -2001,12 +2065,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -2019,12 +2077,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -2049,12 +2101,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -2067,6 +2113,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winnow" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +dependencies = [ + "memchr", +] + [[package]] name = "xattr" version = "0.2.3" @@ -2100,7 +2155,7 @@ dependencies = [ "futures-util", "hex", "lazy_static", - "nix 0.23.1", + "nix 0.23.2", "once_cell", "ordered-stream", "rand", @@ -2126,14 +2181,14 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 1.0.82", + "syn 1.0.109", ] [[package]] name = "zbus_names" -version = "2.2.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41a408fd8a352695690f53906dc7fd036be924ec51ea5e05666ff42685ed0af5" +checksum = "82441e6033be0a741157a72951a3e4957d519698f3a824439cc131c5ba77ac2a" dependencies = [ "serde", "static_assertions", @@ -2142,9 +2197,9 @@ dependencies = [ [[package]] name = "zvariant" -version = "3.6.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bd68e4e6432ef19df47d7e90e2e72b5e7e3d778e0ae3baddf12b951265cc758" +checksum = "622cc473f10cef1b0d73b7b34a266be30ebdcfaea40ec297dd8cbda088f9f93c" dependencies = [ "byteorder", "enumflags2", @@ -2156,12 +2211,24 @@ dependencies = [ [[package]] name = "zvariant_derive" -version = "3.6.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08e977eaa3af652f63d479ce50d924254ad76722a6289ec1a1eac3231ca30430" +checksum = "5d9c1b57352c25b778257c661f3c4744b7cefb7fc09dd46909a153cce7773da2" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.82", + "syn 1.0.109", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7234f0d811589db492d16893e3f21e8e2fd282e6d01b0cddee310322062cc200" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index ed90e1117..c7586b6d4 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -96,6 +96,17 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -280,6 +291,27 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "either" version = "1.8.1" @@ -460,6 +492,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glob" version = "0.3.1" @@ -506,6 +549,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -728,6 +780,7 @@ dependencies = [ "hyper", "kata-types", "libc", + "logging", "micro_http", "nix 0.25.1", "privdrop", @@ -765,6 +818,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -819,6 +873,7 @@ dependencies = [ "slog-async", "slog-json", "slog-scope", + "slog-term", ] [[package]] @@ -959,6 +1014,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "oci" version = "0.1.0" @@ -1264,6 +1328,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.7.1" @@ -1580,6 +1655,19 @@ dependencies = [ "slog", ] +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -1685,6 +1773,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "test-utils" version = "0.1.0" @@ -1729,6 +1828,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml index 141a90a0c..a0ba95adb 100644 --- a/src/tools/kata-ctl/Cargo.toml +++ b/src/tools/kata-ctl/Cargo.toml @@ -36,6 +36,10 @@ serial_test = "0.5.1" vmm-sys-util = "0.11.0" epoll = "4.0.1" libc = "0.2.138" + +# Note: this crate sets the slog 'max_*' features which allows the log level +# to be modified at runtime. +logging = { path = "../../libs/logging" } slog = "2.7.0" slog-scope = "4.4.0" hyper = "0.14.20" diff --git a/src/tools/kata-ctl/src/arch/aarch64/mod.rs b/src/tools/kata-ctl/src/arch/aarch64/mod.rs index 7eaf3ccb4..0f4012a36 100644 --- a/src/tools/kata-ctl/src/arch/aarch64/mod.rs +++ b/src/tools/kata-ctl/src/arch/aarch64/mod.rs @@ -11,6 +11,7 @@ mod arch_specific { use crate::types::*; use crate::utils; use anyhow::Result; + use slog::{info, o, warn}; use std::path::Path; const KVM_DEV: &str = "/dev/kvm"; @@ -19,6 +20,12 @@ mod arch_specific { #[allow(dead_code)] pub const ARCH_CPU_MODEL_FIELD: &str = "CPU architecture"; + macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "aarch64")) + }; + } + // List of check functions static CHECK_LIST: &[CheckItem] = &[CheckItem { name: CheckType::Cpu, @@ -28,11 +35,14 @@ mod arch_specific { }]; pub fn check(_args: &str) -> Result<()> { - println!("INFO: check: aarch64"); + info!(sl!(), "check: aarch64"); if Path::new(KVM_DEV).exists() { - println!("Kata Containers can run on this host\n"); + info!(sl!(), "Kata Containers can run on this host\n"); } else { - eprintln!("WARNING: Kata Containers can't run on this host as lack of virtulization support\n"); + warn!( + sl!(), + "Kata Containers can't run on this host as lack of virtulization support\n" + ); } Ok(()) diff --git a/src/tools/kata-ctl/src/arch/s390x/mod.rs b/src/tools/kata-ctl/src/arch/s390x/mod.rs index 929e79c92..d3efe6f3c 100644 --- a/src/tools/kata-ctl/src/arch/s390x/mod.rs +++ b/src/tools/kata-ctl/src/arch/s390x/mod.rs @@ -13,6 +13,7 @@ mod arch_specific { use crate::utils; use anyhow::{anyhow, Result}; use nix::unistd::Uid; + use slog::{info, o, warn}; use std::collections::HashMap; use std::io::BufRead; use std::io::BufReader; @@ -21,6 +22,12 @@ mod arch_specific { const CPUINFO_FEATURES_TAG: &str = "features"; const CPU_FEATURES_REQ: &[&str] = &["sie"]; + macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "s390x")) + }; + } + #[allow(dead_code)] pub const ARCH_CPU_VENDOR_FIELD: &str = check::GENERIC_CPU_VENDOR_FIELD; #[allow(dead_code)] @@ -28,7 +35,7 @@ mod arch_specific { // check cpu fn check_cpu() -> Result<()> { - println!("INFO: check CPU: s390x"); + info!(sl!(), "check CPU: s390x"); let cpu_info = check::get_single_cpu_info(check::PROC_CPUINFO, CPUINFO_DELIMITER)?; @@ -42,14 +49,14 @@ mod arch_specific { let missing_cpu_features = check::check_cpu_flags(&cpu_features, CPU_FEATURES_REQ)?; if !missing_cpu_features.is_empty() { - eprintln!("WARNING: Missing CPU flags {:?}", missing_cpu_features); + warn!(sl!(), "Missing CPU flags {:?}", missing_cpu_features); } Ok(()) } pub fn check(_args: &str) -> Result<()> { - println!("INFO: check: s390x"); + info!(sl!(), "check: s390x"); let _cpu_result = check_cpu(); diff --git a/src/tools/kata-ctl/src/arch/x86_64/mod.rs b/src/tools/kata-ctl/src/arch/x86_64/mod.rs index 206c447a9..53779104d 100644 --- a/src/tools/kata-ctl/src/arch/x86_64/mod.rs +++ b/src/tools/kata-ctl/src/arch/x86_64/mod.rs @@ -15,6 +15,7 @@ mod arch_specific { use crate::utils; use anyhow::{anyhow, Context, Result}; use nix::unistd::Uid; + use slog::{info, o, warn}; use std::fs; use std::path::Path; @@ -27,6 +28,12 @@ mod arch_specific { pub const ARCH_CPU_VENDOR_FIELD: &str = check::GENERIC_CPU_VENDOR_FIELD; pub const ARCH_CPU_MODEL_FIELD: &str = check::GENERIC_CPU_MODEL_FIELD; + macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "x86_64")) + }; + } + // List of check functions static CHECK_LIST: &[CheckItem] = &[ CheckItem { @@ -72,7 +79,7 @@ mod arch_specific { // check cpu fn check_cpu(_args: &str) -> Result<()> { - println!("INFO: check CPU: x86_64"); + info!(sl!(), "check CPU: x86_64"); let cpu_info = check::get_single_cpu_info(check::PROC_CPUINFO, CPUINFO_DELIMITER)?; @@ -89,14 +96,11 @@ mod arch_specific { // TODO: Add more information to output (see kata-check in go tool); adjust formatting let missing_cpu_attributes = check::check_cpu_attribs(&cpu_info, CPU_ATTRIBS_INTEL)?; if !missing_cpu_attributes.is_empty() { - eprintln!( - "WARNING: Missing CPU attributes {:?}", - missing_cpu_attributes - ); + warn!(sl!(), "Missing CPU attributes {:?}", missing_cpu_attributes); } let missing_cpu_flags = check::check_cpu_flags(&cpu_flags, CPU_FLAGS_INTEL)?; if !missing_cpu_flags.is_empty() { - eprintln!("WARNING: Missing CPU flags {:?}", missing_cpu_flags); + warn!(sl!(), "Missing CPU flags {:?}", missing_cpu_flags); } Ok(()) @@ -122,7 +126,7 @@ mod arch_specific { // check if kvm is usable fn check_kvm_is_usable(_args: &str) -> Result<()> { - println!("INFO: check if kvm is usable: x86_64"); + info!(sl!(), "check if kvm is usable: x86_64"); let result = check::check_kvm_is_usable_generic(); @@ -275,7 +279,7 @@ mod arch_specific { } fn check_kernel_modules(_args: &str) -> Result<()> { - println!("INFO: check kernel modules for: x86_64"); + info!(sl!(), "check kernel modules for: x86_64"); for module in MODULE_LIST { let module_loaded = @@ -291,12 +295,12 @@ mod arch_specific { ); match parameter_check { - Ok(_v) => println!("{} Ok", module.name), + Ok(_v) => info!(sl!(), "{} Ok", module.name), Err(e) => return Err(e), } } Err(err) => { - eprintln!("WARNING {:}", err.replace('\n', "")) + warn!(sl!(), "{:}", err.replace('\n', "")) } } } diff --git a/src/tools/kata-ctl/src/args.rs b/src/tools/kata-ctl/src/args.rs index cc2dc0513..8049ae31b 100644 --- a/src/tools/kata-ctl/src/args.rs +++ b/src/tools/kata-ctl/src/args.rs @@ -12,6 +12,25 @@ use thiserror::Error; pub struct KataCtlCli { #[clap(subcommand)] pub command: Commands, + #[clap(short, long, value_enum, value_parser = parse_log_level)] + /// Sets the minimum log level required for log messages to be displayed. Default is 'info'. + /// Valid values are: trace, debug, info, warning, error, critical + pub log_level: Option, + #[clap(short, long, action)] + /// If enabled, log messages will be JSON formatted for easier machine parsing + pub json_logging: bool, +} + +fn parse_log_level(arg: &str) -> Result { + match arg { + "trace" => Ok(slog::Level::Trace), + "debug" => Ok(slog::Level::Debug), + "info" => Ok(slog::Level::Info), + "warning" => Ok(slog::Level::Warning), + "error" => Ok(slog::Level::Error), + "critical" => Ok(slog::Level::Critical), + _ => Err("Must be one of [trace, debug, info, warning, error, critical]".to_string()), + } } #[derive(Debug, Subcommand)] diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs index ef1007f86..d531425c2 100644 --- a/src/tools/kata-ctl/src/check.rs +++ b/src/tools/kata-ctl/src/check.rs @@ -12,6 +12,7 @@ use nix::unistd::close; use nix::{ioctl_write_int_bad, request_code_none}; use reqwest::header::{CONTENT_TYPE, USER_AGENT}; use serde::{Deserialize, Serialize}; +use slog::{info, o}; use std::fmt; use thiserror::Error; @@ -51,6 +52,12 @@ pub const GENERIC_CPU_MODEL_FIELD: &str = "model name"; #[allow(dead_code)] pub const PROC_CPUINFO: &str = "/proc/cpuinfo"; +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "check")) + }; +} + fn read_file_contents(file_path: &str) -> Result { let contents = std::fs::read_to_string(file_path)?; Ok(contents) @@ -276,14 +283,20 @@ pub fn check_all_releases() -> Result<()> { for release in releases { if !release.prerelease { - println!( + info!( + sl!(), "Official : Release {:15}; created {} ; {}", - release.tag_name, release.created_at, release.tarball_url + release.tag_name, + release.created_at, + release.tarball_url ); } else { - println!( + info!( + sl!(), "PreRelease: Release {:15}; created {} ; {}", - release.tag_name, release.created_at, release.tarball_url + release.tag_name, + release.created_at, + release.tarball_url ); } } @@ -294,12 +307,15 @@ pub fn check_official_releases() -> Result<()> { let releases: Vec = get_kata_all_releases_by_url(KATA_GITHUB_RELEASE_URL).map_err(handle_reqwest_error)?; - println!("Official Releases..."); + info!(sl!(), "Official Releases..."); for release in releases { if !release.prerelease { - println!( + info!( + sl!(), "Release {:15}; created {} ; {}", - release.tag_name, release.created_at, release.tarball_url + release.tag_name, + release.created_at, + release.tarball_url ); } } @@ -392,6 +408,7 @@ pub fn check_kernel_module_loaded(module: &str, parameter: &str) -> Result { + slog_scope::logger().new(o!("subsystem" => "kata-ctl_main")) + }; +} fn real_main() -> Result<()> { let args = KataCtlCli::parse(); - match args.command { + let log_level = args.log_level.unwrap_or(slog::Level::Info); + + let (logger, _guard) = if args.json_logging { + logging::create_logger(crate_name!(), crate_name!(), log_level, io::stdout()) + } else { + logging::create_term_logger(log_level) + }; + + let _guard = slog_scope::set_global_logger(logger); + + let res = match args.command { Commands::Check(args) => handle_check(args), Commands::DirectVolume(args) => handle_direct_volume(args), Commands::Exec(args) => handle_exec(args), @@ -35,12 +53,19 @@ fn real_main() -> Result<()> { Commands::Iptables(args) => handle_iptables(args), Commands::Metrics(args) => handle_metrics(args), Commands::Version => handle_version(), + }; + + // Log errors here, then let the logger go out of scope in main() to ensure + // the asynchronous drain flushes all messages before exit() + if let Err(e) = &res { + error!(sl!(), "{:#?}", e); } + + res } fn main() { - if let Err(e) = real_main() { - eprintln!("ERROR: {:#?}", e); + if let Err(_e) = real_main() { exit(1); } } diff --git a/src/tools/kata-ctl/src/ops/check_ops.rs b/src/tools/kata-ctl/src/ops/check_ops.rs index ed418169d..8b19e0d8f 100644 --- a/src/tools/kata-ctl/src/ops/check_ops.rs +++ b/src/tools/kata-ctl/src/ops/check_ops.rs @@ -15,8 +15,16 @@ use crate::types::*; use anyhow::{anyhow, Result}; +use slog::{info, o, warn}; + const NAME: &str = "kata-ctl"; +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "check_ops")) + }; +} + // This function retrieves the cmd function passes as argument fn get_builtin_check_func(name: CheckType) -> Result { if let Some(check_list) = get_checks() { @@ -55,14 +63,12 @@ fn print_check_list() -> Result<()> { let cmds = get_client_cmd_details(); if cmds.is_empty() { - println!("Checks not found!\n"); + warn!(sl!(), "Checks not found!\n"); return Ok(()); } - cmds.iter().for_each(|n| println!(" - {}", n)); - - println!(); + cmds.iter().for_each(|n| info!(sl!(), " - {}", n)); Ok(()) } @@ -125,6 +131,6 @@ pub fn handle_metrics(_args: MetricsCommand) -> Result<()> { pub fn handle_version() -> Result<()> { let version = version::get().unwrap(); - println!("{} version {:?} (type: rust)", NAME, version); + info!(sl!(), "{} version {:?} (type: rust)", NAME, version); Ok(()) } diff --git a/src/tools/kata-ctl/src/ops/exec_ops.rs b/src/tools/kata-ctl/src/ops/exec_ops.rs index 9e1d4549a..51651188b 100644 --- a/src/tools/kata-ctl/src/ops/exec_ops.rs +++ b/src/tools/kata-ctl/src/ops/exec_ops.rs @@ -19,7 +19,7 @@ use std::{ use anyhow::{anyhow, Context}; use nix::sys::socket::{connect, socket, AddressFamily, SockFlag, SockType, VsockAddr}; use reqwest::StatusCode; -use slog::debug; +use slog::{debug, error, o}; use vmm_sys_util::terminal::Terminal; use crate::args::ExecArguments; @@ -40,7 +40,7 @@ type Result = std::result::Result; #[macro_export] macro_rules! sl { () => { - slog_scope::logger() + slog_scope::logger().new(o!("subsystem" => "exec_ops")) }; } @@ -142,7 +142,7 @@ impl EpollContext { return Ok(()); } Err(e) => { - println!("error with errno {:?} while reading stdin", e); + error!(sl!(), "errno {:?} while reading stdin", e); return Ok(()); } Ok(count) => { @@ -159,7 +159,7 @@ impl EpollContext { return Ok(()); } Err(e) => { - println!("error with errno {:?} while reading server", e); + error!(sl!(), "errno {:?} while reading server", e); return Ok(()); } Ok(count) => { diff --git a/src/tools/kata-ctl/src/ops/volume_ops.rs b/src/tools/kata-ctl/src/ops/volume_ops.rs index 1027dcb72..94243743c 100644 --- a/src/tools/kata-ctl/src/ops/volume_ops.rs +++ b/src/tools/kata-ctl/src/ops/volume_ops.rs @@ -8,12 +8,13 @@ use crate::args::{DirectVolSubcommand, DirectVolumeCommand}; use anyhow::{anyhow, Ok, Result}; use futures::executor; use kata_types::mount::{ - DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, KATA_MOUNT_INFO_FILE_NAME, + get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH, + KATA_MOUNT_INFO_FILE_NAME, }; use nix; use reqwest::StatusCode; -use safe_path; -use std::{fs, path::PathBuf, time::Duration}; +use slog::{info, o}; +use std::{fs, time::Duration}; use url; use agent::ResizeVolumeRequest; @@ -25,6 +26,12 @@ use shim_interface::shim_mgmt::{ const TIMEOUT: Duration = Duration::from_millis(2000); const CONTENT_TYPE_JSON: &str = "application/json"; +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "volume_ops")) + }; +} + pub fn handle_direct_volume(vol_cmd: DirectVolumeCommand) -> Result<()> { if !nix::unistd::Uid::effective().is_root() { return Err(anyhow!( @@ -41,7 +48,7 @@ pub fn handle_direct_volume(vol_cmd: DirectVolumeCommand) -> Result<()> { } }; if let Some(cmd_result) = cmd_result { - println!("{:?}", cmd_result); + info!(sl!(), "{:?}", cmd_result); } Ok(()) @@ -90,17 +97,6 @@ async fn stats(volume_path: &str) -> Result> { Ok(Some(body)) } -// join_path joins user provided volumepath with kata direct-volume root path -// the volume_path is base64-encoded and then safely joined to the end of path prefix -fn join_path(prefix: &str, volume_path: &str) -> Result { - if volume_path.is_empty() { - return Err(anyhow!("volume path must not be empty")); - } - let b64_encoded_path = base64::encode(volume_path.as_bytes()); - - Ok(safe_path::scoped_join(prefix, b64_encoded_path)?) -} - // add writes the mount info (json string) of a direct volume into a filesystem path known to Kata Containers. pub fn add(volume_path: &str, mount_info: &str) -> Result> { let mount_info_dir_path = join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?; @@ -129,15 +125,6 @@ pub fn remove(volume_path: &str) -> Result> { Ok(None) } -pub fn get_volume_mount_info(volume_path: &str) -> Result { - let mount_info_file_path = - join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path)?.join(KATA_MOUNT_INFO_FILE_NAME); - let mount_info_file = fs::read_to_string(mount_info_file_path)?; - let mount_info: DirectVolumeMountInfo = serde_json::from_str(&mount_info_file)?; - - Ok(mount_info) -} - // get_sandbox_id_for_volume finds the id of the first sandbox found in the dir. // We expect a direct-assigned volume is associated with only a sandbox at a time. pub fn get_sandbox_id_for_volume(volume_path: &str) -> Result { @@ -170,7 +157,7 @@ mod tests { use super::*; use kata_types::mount::DirectVolumeMountInfo; use serial_test::serial; - use std::{collections::HashMap, fs}; + use std::{collections::HashMap, fs, path::PathBuf}; use tempfile::tempdir; use test_utils::skip_if_not_root; diff --git a/src/tools/log-parser-rs/README.md b/src/tools/log-parser-rs/README.md index a641b0030..18df4feeb 100644 --- a/src/tools/log-parser-rs/README.md +++ b/src/tools/log-parser-rs/README.md @@ -48,7 +48,7 @@ For a comprehensive (and guaranteed up to date) list, please run `log-parser-rs ## Usage -1. Make sure containerd is in [debug mode](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#enabling-full-containerd-debug) +1. Make sure containerd is in [debug mode](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enabling-full-containerd-debug) 1. Make sure you are running runtime-rs: ``` $ containerd-shim-kata-v2 --version|grep -qi rust && echo rust || echo golang diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock index 08cb8c261..ed239100a 100644 --- a/src/tools/runk/Cargo.lock +++ b/src/tools/runk/Cargo.lock @@ -806,6 +806,7 @@ dependencies = [ "num_cpus", "oci", "regex", + "safe-path", "serde", "serde_json", "slog", @@ -1517,6 +1518,13 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "safe-path" +version = "0.1.0" +dependencies = [ + "libc", +] + [[package]] name = "scan_fmt" version = "0.2.6" diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh new file mode 100755 index 000000000..103ce2cda --- /dev/null +++ b/tests/integration/gha-run.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +integration_dir="$(dirname "$(readlink -f "$0")")" +tools_dir="${integration_dir}/../../tools" + +function _print_cluster_name() { + short_sha="$(git rev-parse --short=12 HEAD)" + echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" +} + +function install_azure_cli() { + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # The aks-preview extension is required while the Mariner Kata host is in preview. + az extension add --name aks-preview +} + +function login_azure() { + az login \ + --service-principal \ + -u "${AZ_APPID}" \ + -p "${AZ_PASSWORD}" \ + --tenant "${AZ_TENANT_ID}" +} + +function create_cluster() { + az aks create \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + -s "Standard_D4s_v5" \ + --node-count 1 \ + --generate-ssh-keys \ + $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") +} + +function install_bats() { + sudo apt-get update + sudo apt-get -y install bats +} + +function install_kubectl() { + sudo az aks install-cli +} + +function get_cluster_credentials() { + az aks get-credentials \ + -g "kataCI" \ + -n "$(_print_cluster_name)" +} + +function run_tests() { + platform="${1}" + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" + + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + if [ "${platform}" = "tdx" ]; then + kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" + else + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + fi + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" + + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + if [ "${platform}" = "aks" ]; then + sleep 240s + else + sleep 60s + fi + + pushd "${integration_dir}/kubernetes" + bash setup.sh + bash run_kubernetes_tests.sh + popd +} + +function cleanup() { + platform="${1}" + + if [ "${platform}" = "tdx" ]; then + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + else + deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + fi + + kubectl delete ${deploy_spec} + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" + kubectl apply ${cleanup_spec} + sleep 180s + + kubectl delete ${cleanup_spec} + kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" +} + +function delete_cluster() { + az aks delete \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + --yes \ + --no-wait +} + +function main() { + action="${1:-}" + + case "${action}" in + install-azure-cli) install_azure_cli ;; + login-azure) login_azure ;; + create-cluster) create_cluster ;; + install-bats) install_bats ;; + install-kubectl) install_kubectl ;; + get-cluster-credentials) get_cluster_credentials ;; + run-tests-aks) run_tests "aks" ;; + run-tests-sev) run_tests "sev" ;; + run-tests-snp) run_tests "snp" ;; + run-tests-tdx) run_tests "tdx" ;; + cleanup-sev) cleanup "sev" ;; + cleanup-snp) cleanup "snp" ;; + cleanup-tdx) cleanup "tdx" ;; + delete-cluster) delete_cluster ;; + *) >&2 echo "Invalid argument"; exit 2 ;; + esac +} + +main "$@" diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index db1e16633..0975ec0d5 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -54,6 +54,10 @@ else ) fi +if [ ${KATA_HOST_OS} == "cbl-mariner" ]; then + exit 0 +fi + # we may need to skip a few test cases when running on non-x86_64 arch arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml" if [ -f "${arch_config_file}" ]; then diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml index e887cc92c..83b0ee0f7 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml @@ -16,6 +16,8 @@ spec: containers: - name: mounttest-container image: ${agnhost_image} + command: + - /agnhost args: - mounttest - --fs_type=/test-volume @@ -27,6 +29,8 @@ spec: mountPath: /test-volume - name: mounttest-container-2 image: ${agnhost_image} + command: + - /agnhost args: - mounttest - --fs_type=/test-volume-2 diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml index 3d336761f..081b7941f 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-http-liveness.yaml @@ -15,6 +15,8 @@ spec: containers: - name: liveness image: ${agnhost_image} + command: + - /agnhost args: - liveness livenessProbe: diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml index 6d5343cfe..4fbe3b6d6 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-tcp-liveness.yaml @@ -15,6 +15,8 @@ spec: containers: - name: tcp-liveness image: ${agnhost_image} + command: + - /agnhost args: - liveness ports: diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh new file mode 100755 index 000000000..0c3baf2dc --- /dev/null +++ b/tests/integration/kubernetes/setup.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +kubernetes_dir=$(dirname "$(readlink -f "$0")") + +set_runtime_class() { + sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml +} + +main() { + set_runtime_class +} + +main "$@" diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh new file mode 100755 index 000000000..e3656b2cd --- /dev/null +++ b/tests/metrics/gha-run.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +metrics_dir="$(dirname "$(readlink -f "$0")")" + +function run_test_launchtimes() { + hypervisor="${1}" + + echo "Running launchtimes tests: " + + if [ "${hypervisor}" = 'qemu' ]; then + echo "qemu" + elif [ "${hypervisor}" = 'clh' ]; then + echo "clh" + fi +} + +function main() { + action="${1:-}" + case "${action}" in + run-test-launchtimes-qemu) run_test_launchtimes "qemu" ;; + run-test-launchtimes-clh) run_test_launchtimes "clh" ;; + *) >&2 echo "Invalid argument"; exit 2 ;; + esac +} + +main "$@" diff --git a/tests/metrics/lib/common.bash b/tests/metrics/lib/common.bash new file mode 100755 index 000000000..30c5c1321 --- /dev/null +++ b/tests/metrics/lib/common.bash @@ -0,0 +1,353 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +THIS_FILE=$(readlink -f ${BASH_SOURCE[0]}) +LIB_DIR=${THIS_FILE%/*} +RESULT_DIR="${LIB_DIR}/../results" + +source ${LIB_DIR}/../../lib/common.bash +source ${LIB_DIR}/json.bash +source /etc/os-release || source /usr/lib/os-release + +# Set variables to reasonable defaults if unset or empty +CTR_EXE="${CTR_EXE:-ctr}" +DOCKER_EXE="${DOCKER_EXE:-docker}" +CTR_RUNTIME="${CTR_RUNTIME:-io.containerd.kata.v2}" +RUNTIME="${RUNTIME:-containerd-shim-kata-v2}" +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +TEST_REPO="${TEST_REPO:-github.com/kata-containers/tests}" +JSON_HOST="${JSON_HOST:-}" + +KSM_BASE="/sys/kernel/mm/ksm" +KSM_ENABLE_FILE="${KSM_BASE}/run" +KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan" +KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs" +KSM_PAGES_SHARED="${KSM_BASE}/pages_shared" + +http_proxy="${http_proxy:-}" +https_proxy="${https_proxy:-}" + +# The settings we use for an 'aggresive' KSM setup +# Scan 1000 pages every 50ms - 20,000 pages/s +KSM_AGGRESIVE_PAGES=1000 +KSM_AGGRESIVE_SLEEP=50 + +declare -A registries +registries[ubuntu]=\ +"docker.io/library +public.ecr.aws/lts +mirror.gcr.io/library +quay.io/libpod" + +# This function checks existence of commands. +# They can be received standalone or as an array, e.g. +# +# cmds=(“cmd1” “cmd2”) +# check_cmds "${cmds[@]}" +check_cmds() +{ + local cmd req_cmds=( "$@" ) + for cmd in "${req_cmds[@]}"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + die "command $cmd not available" + fi + echo "command: $cmd: yes" + done +} + +# This function performs a pull on the image names +# passed in (notionally as 'about to be used'), to ensure +# - that we have the most upto date images +# - that any pull/refresh time (for a first pull) does not +# happen during the test itself. +# +# The image list can be received standalone or as an array, e.g. +# +# images=(“img1” “img2”) +# check_imgs "${images[@]}" +check_images() +{ + local img req_images=( "$@" ) + for img in "${req_images[@]}"; do + echo "ctr pull'ing: $img" + if ! sudo "${CTR_EXE}" image pull "$img"; then + die "Failed to pull image $img" + fi + echo "ctr pull'd: $img" + done +} + +generate_build_dockerfile() { + local dockerfile="$1" + local image="$2" + local map_key="$3" + local text_to_replace="$4" + local regs=(${registries["${map_key}"]}) + for r in ${regs[@]}; do + sed 's|'${text_to_replace}'|'${r}'|g' \ + "${dockerfile}.in" > "${dockerfile}" + if sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile" "$dockerfile_dir"; then + return 0 + fi + done + return 1 +} + +# This function performs a build on the image names +# passed in, to ensure that we have the latest changes from +# the dockerfiles +build_dockerfile_image() +{ + local image="$1" + local dockerfile_path="$2" + local dockerfile_dir=${2%/*} + + if [ -f "$dockerfile_path" ]; then + echo "docker building $image" + if ! sudo "${DOCKER_EXE}" build --build-arg http_proxy="${http_proxy}" --build-arg https_proxy="${https_proxy}" --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then + die "Failed to docker build image $image" + fi + return 0 + fi + + generate_build_dockerfile "${dockerfile_path}" "${image}" "ubuntu" "@UBUNTU_REGISTRY@" \ + || die "Failed to docker build image $image" +} + +# This function removes the ctr image, builds a new one using a dockerfile +# and imports the image from docker to ctr +check_ctr_images() +{ + local ctr_image="$1" + local dockerfile_path="$2" + local docker_image="$(echo ${ctr_image} | cut -d/ -f3 | cut -d: -f1)" + + if [ -z "$ctr_image" ] || [ -z "$dockerfile_path" ]; then + die "Missing image or dockerfile path variable" + fi + + sudo "${CTR_EXE}" i rm "${ctr_image}" + build_dockerfile_image "${docker_image}" "${dockerfile_path}" + sudo "${DOCKER_EXE}" save -o "${docker_image}.tar" "${docker_image}" + sudo "${CTR_EXE}" i import "${docker_image}.tar" + rm -rf "${docker_image}".tar +} + +# A one time (per uber test cycle) init that tries to get the +# system to a 'known state' as much as possible +metrics_onetime_init() +{ + # The onetime init must be called once, and only once + if [ ! -z "$onetime_init_done" ]; then + die "onetime_init() called more than once" + fi + + # Restart services + restart_containerd_service + + # We want this to be seen in sub shells as well... + # otherwise init_env() cannot check us + export onetime_init_done=1 +} + +# Print a banner to the logs noting clearly which test +# we are about to run +test_banner() +{ + echo -e "\n===== starting test [$1] =====" +} + +# Initialization/verification environment. This function makes +# minimal steps for metrics/tests execution. +init_env() +{ + test_banner "${TEST_NAME}" + + cmd=("docker" "ctr") + + # check dependencies + check_cmds "${cmd[@]}" + + # Remove all stopped containers + clean_env + clean_env_ctr + + # This clean up is more aggressive, this is in order to + # decrease the factors that could affect the metrics results. + kill_processes_before_start +} + +# This function checks if there are containers or +# shim/proxy/hypervisor processes up, if found, they are +# killed to start test with clean environment. +kill_processes_before_start() { + DOCKER_PROCS=$(sudo "${DOCKER_EXE}" ps -q) + [[ -n "${DOCKER_PROCS}" ]] && clean_env + + CTR_PROCS=$(sudo "${CTR_EXE}" t list -q) + [[ -n "${CTR_PROCS}" ]] && clean_env_ctr + + check_processes +} + +# Generate a random name - generally used when creating containers, but can +# be used for any other appropriate purpose +random_name() { + mktemp -u kata-XXXXXX +} + +show_system_ctr_state() { + echo "Showing system state:" + echo " --Check containers--" + sudo "${CTR_EXE}" c list + echo " --Check tasks--" + sudo "${CTR_EXE}" task list + + local processes="containerd-shim-kata-v2" + + for p in ${processes}; do + echo " --pgrep ${p}--" + pgrep -a ${p} + done +} + +common_init(){ + if [ "$CTR_RUNTIME" == "io.containerd.kata.v2" ] || [ "$RUNTIME" == "containerd-shim-kata-v2" ]; then + extract_kata_env + else + # We know we have nothing to do for runc or shimv2 + if [ "$CTR_RUNTIME" != "io.containerd.runc.v2" ] || [ "$RUNTIME" != "runc" ]; then + warn "Unrecognised runtime" + fi + fi +} + + +# Save the current KSM settings so we can restore them later +save_ksm_settings(){ + echo "saving KSM settings" + ksm_stored_run=$(cat ${KSM_ENABLE_FILE}) + ksm_stored_pages=$(cat ${KSM_ENABLE_FILE}) + ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE}) +} + +set_ksm_aggressive(){ + echo "setting KSM to aggressive mode" + # Flip the run off/on to ensure a restart/rescan + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}" + sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}" + sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}" + + if [ "${KATA_HYPERVISOR}" == "qemu" ]; then + # Disable virtio-fs and save whether it was enabled previously + set_virtio_out=$(sudo -E PATH="$PATH" "${LIB_DIR}/../../.ci/set_kata_config.sh" shared_fs virtio-9p) + echo "${set_virtio_out}" + grep -q "already" <<< "${set_virtio_out}" || was_virtio_fs=true; + fi +} + +restore_virtio_fs(){ + # Re-enable virtio-fs if it was enabled previously + [ -n "${was_virtio_fs}" ] && sudo -E PATH="$PATH" "${LIB_DIR}/../../.ci/set_kata_config.sh" shared_fs virtio-fs || \ + info "Not restoring virtio-fs since it wasn't enabled previously" +} + +restore_ksm_settings(){ + echo "restoring KSM settings" + # First turn off the run to ensure if we are then re-enabling + # that any changes take effect + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}" + sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}" + sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}" + [ "${KATA_HYPERVISOR}" == "qemu" ] && restore_virtio_fs +} + +disable_ksm(){ + echo "disabling KSM" + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + [ "${KATA_HYPERVISOR}" == "qemu" ] && restore_virtio_fs +} + +# See if KSM is enabled. +# If so, amend the test name to reflect that +check_for_ksm(){ + if [ ! -f ${KSM_ENABLE_FILE} ]; then + return + fi + + ksm_on=$(< ${KSM_ENABLE_FILE}) + + if [ $ksm_on == "1" ]; then + TEST_NAME="${TEST_NAME} ksm" + fi +} + +# Wait for KSM to settle down, or timeout waiting +# The basic algorithm is to look at the pages_shared value +# at the end of every 'full scan', and if the value +# has changed very little, then we are done (because we presume +# a full scan has managed to do few new merges) +# +# arg1 - timeout in seconds +wait_ksm_settle(){ + [[ "$RUNTIME" == "runc" ]] || [[ "$CTR_RUNTIME" == "io.containerd.runc.v2" ]] && return + local t pcnt + local oldscan=-1 newscan + local oldpages=-1 newpages + + oldscan=$(cat /sys/kernel/mm/ksm/full_scans) + + # Wait some time for KSM to kick in to avoid early dismissal + for ((t=0; t<5; t++)); do + pages=$(cat "${KSM_PAGES_SHARED}") + [[ "$pages" -ne 0 ]] && echo "Discovered KSM activity" && break + sleep 1 + done + + # Go around the loop until either we see a small % change + # between two full_scans, or we timeout + for ((t=0; t<$1; t++)); do + + newscan=$(cat /sys/kernel/mm/ksm/full_scans) + newpages=$(cat "${KSM_PAGES_SHARED}") + [[ "$newpages" -eq 0 ]] && echo "No need to wait for KSM to settle" && return + + if (( newscan != oldscan )); then + echo -e "\nnew full_scan ($oldscan to $newscan)" + + # Do we have a previous scan to compare with + echo "check pages $oldpages to $newpages" + + if (( oldpages != -1 )); then + # avoid divide by zero problems + if (( $oldpages > 0 )); then + pcnt=$(( 100 - ((newpages * 100) / oldpages) )) + # abs() + pcnt=$(( $pcnt * -1 )) + + echo "$oldpages to $newpages is ${pcnt}%" + + if (( $pcnt <= 5 )); then + echo "KSM stabilised at ${t}s" + return + fi + else + echo "$oldpages KSM pages... waiting" + fi + fi + oldscan=$newscan + oldpages=$newpages + else + echo -n "." + fi + sleep 1 + done + echo "Timed out after ${1}s waiting for KSM to settle" +} + +common_init diff --git a/tools/osbuilder/image-builder/Dockerfile b/tools/osbuilder/image-builder/Dockerfile index 5961ee2ed..fcfa5ac20 100644 --- a/tools/osbuilder/image-builder/Dockerfile +++ b/tools/osbuilder/image-builder/Dockerfile @@ -14,6 +14,6 @@ RUN ([ -n "$http_proxy" ] && \ gdisk \ parted \ qemu-img \ - veritysetup \ + veritysetup \ xfsprogs && \ dnf clean all diff --git a/tools/osbuilder/image-builder/image_builder.sh b/tools/osbuilder/image-builder/image_builder.sh index a22d8d12a..3e7f0babc 100755 --- a/tools/osbuilder/image-builder/image_builder.sh +++ b/tools/osbuilder/image-builder/image_builder.sh @@ -11,7 +11,7 @@ set -o errexit set -o pipefail DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc} -KATA_BUILD_CC=${KATA_BUILD_CC:-no} +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} readonly script_name="${0##*/}" readonly script_dir=$(dirname "$(readlink -f "$0")") @@ -186,7 +186,7 @@ build_with_container() { --env BLOCK_SIZE="${block_size}" \ --env ROOT_FREE_SPACE="${root_free_space}" \ --env NSDAX_BIN="${nsdax_bin}" \ - --env KATA_BUILD_CC="${KATA_BUILD_CC}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ --env SELINUX="${SELINUX}" \ --env DEBUG="${DEBUG}" \ -v /dev:/dev \ @@ -394,7 +394,7 @@ create_disk() { # The partition is the rootfs content info "Creating partitions" - if [ "${KATA_BUILD_CC}" == "yes" ]; then + if [ "${MEASURED_ROOTFS}" == "yes" ]; then info "Creating partitions with hash device" # The hash data will take less than one percent disk space to store hash_start=$(echo $img_size | awk '{print $1 * 0.99}' |cut -d $(locale decimal_point) -f 1) @@ -504,7 +504,7 @@ create_rootfs_image() { fsck.ext4 -D -y "${device}p1" fi - if [ "${KATA_BUILD_CC}" == "yes" ] && [ -b "${device}p2" ]; then + if [ "${MEASURED_ROOTFS}" == "yes" ] && [ -b "${device}p2" ]; then info "veritysetup format rootfs device: ${device}p1, hash device: ${device}p2" local image_dir=$(dirname "${image}") veritysetup format "${device}p1" "${device}p2" > "${image_dir}"/root_hash.txt 2>&1 diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index e0c900034..82af8ec66 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -17,6 +17,7 @@ RUST_VERSION="null" AGENT_BIN=${AGENT_BIN:-kata-agent} AGENT_INIT=${AGENT_INIT:-no} KATA_BUILD_CC=${KATA_BUILD_CC:-no} +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""} OSBUILDER_VERSION="unknown" DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc} @@ -447,6 +448,7 @@ build_rootfs_distro() --env KATA_BUILD_CC="${KATA_BUILD_CC}" \ --env ARCH="${ARCH}" \ --env CI="${CI}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \ --env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \ --env LIBC="${LIBC}" \ --env EXTRA_PKGS="${EXTRA_PKGS}" \ diff --git a/tools/osbuilder/rootfs-builder/ubuntu/config.sh b/tools/osbuilder/rootfs-builder/ubuntu/config.sh index 5356dfdca..e86893931 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/config.sh +++ b/tools/osbuilder/rootfs-builder/ubuntu/config.sh @@ -7,7 +7,7 @@ OS_NAME=ubuntu OS_VERSION=${OS_VERSION:-focal} PACKAGES="chrony iptables dbus kmod" [ "$AGENT_INIT" = no ] && PACKAGES+=" init" -[ "$KATA_BUILD_CC" = yes ] && PACKAGES+=" cryptsetup-bin e2fsprogs" +[ "$MEASURED_ROOTFS" = yes ] && PACKAGES+=" cryptsetup-bin e2fsprogs" [ "$SECCOMP" = yes ] && PACKAGES+=" libseccomp2" REPO_URL=http://ports.ubuntu.com diff --git a/tools/packaging/.gitignore b/tools/packaging/.gitignore index 82abc0533..2a056e4ba 100644 --- a/tools/packaging/.gitignore +++ b/tools/packaging/.gitignore @@ -1,7 +1,6 @@ *.img *.initrd *.sha256 -*.snap *.tar.gz *.tar.xz *.gz @@ -13,7 +12,6 @@ debian.series parts/ prime/ sha256sums.asc -snap/.snapcraft/ stage/ typescript kata-linux-* diff --git a/tools/packaging/Makefile b/tools/packaging/Makefile index f8444e8f1..4bd1e0334 100644 --- a/tools/packaging/Makefile +++ b/tools/packaging/Makefile @@ -8,13 +8,11 @@ MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) SED := sed YQ := $(MK_DIR)/yq -SNAPCRAFT_FILE := ../../snap/snapcraft.yaml VERSIONS_YAML_FILE := ../../versions.yaml VERSION_FILE := ../../VERSION export MK_DIR export YQ -export SNAPCRAFT_FILE export VERSION_FILE export VERSIONS_YAML_FILE @@ -25,10 +23,4 @@ test-static-build: $(YQ): @bash -c "source scripts/lib.sh; install_yq $${MK_DIR}" -snap: $(YQ) - @if [ "$$(cat $(VERSION_FILE))" != "$$($(YQ) r $(SNAPCRAFT_FILE) version)" ]; then \ - >&2 echo "Warning: $(SNAPCRAFT_FILE) version is different to upstream $(VERSION_FILE) file"; \ - fi - snapcraft -d --destructive-mode - -.PHONY: test-static-build snap +.PHONY: test-static-build diff --git a/tools/packaging/README.md b/tools/packaging/README.md index 8a31c0a80..b1c7ae594 100644 --- a/tools/packaging/README.md +++ b/tools/packaging/README.md @@ -12,10 +12,6 @@ Kata build artifacts are available within a container image, created by a [`kata-deploy`](kata-deploy), which make installation of Kata Containers in a running Kubernetes Cluster very straightforward. -## Build a snap package - -See [the snap documentation](../../snap/local). - ## Build static binaries See [the static build documentation](static-build). diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index 26c32178d..14cc17008 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -26,6 +26,7 @@ final_initrd_name="kata-containers-initrd" image_initrd_extension=".img" arch_target="$(uname -m)" +final_image_name="kata-containers" final_initrd_name="kata-containers-initrd" image_initrd_extension=".img" @@ -105,6 +106,7 @@ main() { image_type=image destdir="$PWD" prefix="/opt/kata" + image_suffix="" image_initrd_suffix="" root_hash_suffix="" builddir="${PWD}" @@ -133,16 +135,11 @@ main() { initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version") initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}" final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" - elif [ -n "${image_initrd_suffix}" ]; then + elif [ "${image_initrd_suffix}" == "tdx" ]; then img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name") img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version") image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}" final_image_name="${final_image_name}-${image_initrd_suffix}" - - initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name") - initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version") - initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}" - final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" fi ;; root_hash_suffix=*) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index bb579631b..f8b9edf4c 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -16,9 +16,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN \ apt-get update && \ -apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \ +apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl gpg xz-utils systemd && \ mkdir -p /etc/apt/keyrings/ && \ -curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && \ +curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ apt-get update && \ apt-get install -y --no-install-recommends kubectl && \ diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 7d63560d5..328bb3dd9 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -56,9 +56,12 @@ all: serial-targets \ serial-targets: ${MAKE} -f $(MK_PATH) -j 1 V= \ rootfs-image-tarball \ + rootfs-image-tdx-tarball \ + rootfs-initrd-mariner-tarball \ rootfs-initrd-sev-tarball \ rootfs-initrd-tarball \ - cloud-hypervisor-tarball + cloud-hypervisor-tarball \ + cloud-hypervisor-glibc-tarball %-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh $(call BUILD,$*) @@ -66,6 +69,9 @@ serial-targets: cloud-hypervisor-tarball: ${MAKE} $@-build +cloud-hypervisor-glibc-tarball: + ${MAKE} $@-build + firecracker-tarball: ${MAKE} $@-build @@ -114,6 +120,12 @@ qemu-tdx-experimental-tarball: rootfs-image-tarball: ${MAKE} $@-build +rootfs-image-tdx-tarball: kernel-tdx-experimental-tarball + ${MAKE} $@-build + +rootfs-initrd-mariner-tarball: + ${MAKE} $@-build + rootfs-initrd-sev-tarball: kernel-sev-tarball ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index d9f1401a1..606ce8435 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -61,6 +61,7 @@ docker run \ --env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \ --env INCLUDE_ROOTFS="$(realpath "${INCLUDE_ROOTFS:-}" 2> /dev/null || true)" \ --env HKD_PATH="$(realpath "${HKD_PATH:-}" 2> /dev/null || true)" \ + --env CC_BUILDER_REGISTRY="${CC_BUILDER_REGISTRY:-}" \ --env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" \ --env INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER:-}" \ --env KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER:-}" \ @@ -69,6 +70,7 @@ docker run \ --env SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER:-}" \ --env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \ --env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ --rm \ -w ${script_dir} \ build-kata-deploy "${kata_deploy_create}" $@ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 3a4dc4c83..73ce73d05 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -47,6 +47,7 @@ readonly jenkins_url="http://jenkins.katacontainers.io" readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts" ARCH=$(uname -m) +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} workdir="${WORKDIR:-$PWD}" @@ -102,6 +103,7 @@ options: qemu-snp-experimental qemu-tdx-experimental rootfs-image + rootfs-image-tdx rootfs-initrd rootfs-initrd-sev shim-v2 @@ -387,21 +389,25 @@ install_cc_shimv2() { export RUST_VERSION export REMOVE_VMM_CONFIGS="acrn fc" - extra_opts="DEFSERVICEOFFLOAD=true" - if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then - root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt") - root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" - extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" - fi + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + extra_opts="DEFSERVICEOFFLOAD=true" + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt") + root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" + fi - if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" ]; then - root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt") - root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" - extra_opts+=" ROOTMEASURECONFIGTDX=\"${root_measure_config}\"" + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt") + root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIGTDX=\"${root_measure_config}\"" + fi + + info "extra_opts: ${extra_opts}" + DESTDIR="${destdir}" PREFIX="${cc_prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" + else + DESTDIR="${destdir}" PREFIX="${cc_prefix}" "${shimv2_builder}" fi - - info "extra_opts: ${extra_opts}" - DESTDIR="${destdir}" PREFIX="${cc_prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" } # Install static CC virtiofsd asset @@ -564,8 +570,10 @@ install_cc_x86_64_ovmf(){ #Install guest image install_image() { - local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-image-$(uname -m)/${cached_artifacts_path}" - local component="rootfs-image" + local image_type="${1:-"image"}" + local initrd_suffix="${2:-""}" + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-${image_type}" local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" @@ -585,7 +593,12 @@ install_image() { && return 0 info "Create image" - "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" + "${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}" +} + +#Install guest image for tdx +install_image_tdx() { + install_image "image-tdx" "tdx" } #Install guest initrd @@ -678,6 +691,11 @@ install_kernel_helper() { install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0 + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + info "build initramfs for cc kernel" + "${initramfs_builder}" + fi + info "build ${kernel_name}" info "Kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -v "${kernel_version}" ${extra_cmd} @@ -740,6 +758,8 @@ install_kernel_experimental() { install_kernel_tdx_experimental() { local kernel_url="$(get_from_kata_deps assets.kernel-tdx-experimental.url)" + export MEASURED_ROOTFS=yes + install_kernel_helper \ "assets.kernel-tdx-experimental.version" \ "kernel-tdx-experimental" \ @@ -916,7 +936,19 @@ install_shimv2() { export GO_VERSION export RUST_VERSION - DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" + + if [ "${MEASURED_ROOTFS}" == "yes" ]; then + extra_opts="DEFSERVICEOFFLOAD=true" + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder//root_hash.txt") + root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" + fi + + DESTDIR="${destdir}" PREFIX="${prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" + else + DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" + fi } install_ovmf() { @@ -1033,6 +1065,8 @@ handle_build() { cloud-hypervisor) install_clh ;; + cloud-hypervisor-glibc) ;; + firecracker) install_firecracker ;; kernel) install_kernel ;; @@ -1065,8 +1099,12 @@ handle_build() { rootfs-image) install_image ;; + rootfs-image-tdx) install_image_tdx ;; + rootfs-initrd) install_initrd ;; + rootfs-initrd-mariner) ;; + rootfs-initrd-sev) install_initrd_sev ;; shim-v2) install_shimv2 ;; diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md index 5c8580a68..32aa7a9e2 100644 --- a/tools/packaging/kernel/README.md +++ b/tools/packaging/kernel/README.md @@ -9,8 +9,7 @@ automates the process to build a kernel for Kata Containers. The `build-kernel.sh` script requires an installed Golang version matching the [component build requirements](../../../docs/Developer-Guide.md#requirements-to-build-individual-components). It also requires [yq](https://github.com/mikefarah/yq) version 3.4.1 -> **Hint**: `snap install yq --channel=v3/stable` \ -> **or** `go install github.com/mikefarah/yq/v3@latest` +> **Hint**: `go install github.com/mikefarah/yq/v3@latest` The Linux kernel scripts further require a few packages (flex, bison, and libelf-dev) diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 9b58de355..31af469a3 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -63,7 +63,8 @@ kernel_url="" #Linux headers for GPU guest fs module building linux_headers="" -KATA_BUILD_CC=${KATA_BUILD_CC:-no} +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} + packaging_scripts_dir="${script_dir}/../scripts" source "${packaging_scripts_dir}/lib.sh" @@ -129,7 +130,13 @@ get_tee_kernel() { mkdir -p ${kernel_path} - [ -z "${kernel_url}" ] && kernel_url=$(get_from_kata_deps "assets.kernel.${tee}.url") + if [ -z "${kernel_url}" ]; then + if [[ "${conf_guest}" == "tdx" ]]; then + kernel_url=$(get_from_kata_deps "assets.kernel-tdx-experimental.url") + else + kernel_url=$(get_from_kata_deps "assets.kernel.${tee}.url") + fi + fi local kernel_tarball="${version}.tar.gz" @@ -263,7 +270,7 @@ get_kernel_frag_path() { fi fi - if [ "${KATA_BUILD_CC}" == "yes" ]; then + if [ "${MEASURED_ROOTFS}" == "yes" ]; then info "Enabling config for confidential guest trust storage protection" local cryptsetup_configs="$(ls ${common_path}/confidential_containers/cryptsetup.conf)" all_configs="${all_configs} ${cryptsetup_configs}" @@ -408,7 +415,7 @@ setup_kernel() { [ -n "${hypervisor_target}" ] || hypervisor_target="kvm" [ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}" "${kernel_path}") - if [ "${KATA_BUILD_CC}" == "yes" ] && [ -f "${default_initramfs}" ]; then + if [ "${MEASURED_ROOTFS}" == "yes" ] && [ -f "${default_initramfs}" ]; then info "Copying initramfs from: ${default_initramfs}" cp "${default_initramfs}" ./ fi @@ -612,8 +619,12 @@ main() { kernel_version=$(get_from_kata_deps "assets.kernel-dragonball-experimental.version") elif [[ "${conf_guest}" != "" ]]; then #If specifying a tag for kernel_version, must be formatted version-like to avoid unintended parsing issues - kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) - [ -n "${kernel_version}" ] || kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.tag") + if [[ "${conf_guest}" == "tdx" ]]; then + kernel_version=$(get_from_kata_deps "assets.kernel-tdx-experimental.version" 2>/dev/null || true) + else + kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) + [ -n "${kernel_version}" ] || kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.tag") + fi else kernel_version=$(get_from_kata_deps "assets.kernel.version") fi diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index b6e2c490e..626ea112c 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -106cc +108cc diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch new file mode 100644 index 000000000..74dcc732e --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch @@ -0,0 +1,163 @@ +From 16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72 Mon Sep 17 00:00:00 2001 +Message-Id: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Wed, 10 May 2023 13:55:43 +0800 +Subject: [PATCH 1/3] upcall: dragonball-devmgr suppots cpu hotplug on arm64 + +Enable vcpuhotplug feature on aarch64 in guest kernel. It communicates +with dragonball by using upcall. This commit does these changes: + +1. Wraps x86 related fields with CONFIG_X86_64. +2. Add "cpu_event_notification" for arm64. +3. Add "add_cpu_dev" and "del_cpu_dev" for arm64. + +Signed-off-by: xuejun-xj +Reviewed-by : Chao Wu +Reviewed-by: Zizheng Bian +Reviewed-by: Baolin Wang +--- + .../upcall_srv/dragonball_device_manager.c | 84 ++++++++++++++++++- + 1 file changed, 81 insertions(+), 3 deletions(-) + +diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c +index 5a95b2ba63e8..088d38623b8d 100644 +--- a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c ++++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c +@@ -85,15 +85,21 @@ struct devmgr_req { + #if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) + struct { + uint8_t count; ++#ifdef CONFIG_X86_64 + uint8_t apic_ver; + uint8_t apic_ids[256]; ++#endif + } cpu_dev_info; + #endif + } msg_load; + }; + + struct cpu_dev_reply_info { ++#if defined(CONFIG_X86_64) + uint32_t apic_index; ++#elif defined(CONFIG_ARM64) ++ uint32_t cpu_id; ++#endif + }; + + struct devmgr_reply { +@@ -190,7 +196,8 @@ static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size, + msg->msg_flags = msg_flags; + } + +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) ++#if defined(CONFIG_X86_64) + static int get_cpu_id(int apic_id) + { + int i; +@@ -219,6 +226,24 @@ static void cpu_event_notification( + _fill_msg_header(&rep->msg_header, + sizeof(struct cpu_dev_reply_info), action_type, 0); + } ++#elif defined(CONFIG_ARM64) ++/** ++ * Return the first failed hotplug index of the cpu_id to dragonball. ++ * If hotplug/hotunplug succeeds, it will equals to the expected cpu count. ++ */ ++static void cpu_event_notification( ++ uint8_t cpu_id, ++ int ret, ++ uint32_t action_type, ++ struct devmgr_reply *rep) ++{ ++ pr_info("cpu event notification: cpu_id %d\n", cpu_id); ++ rep->msg_load.cpu_dev_info.cpu_id = cpu_id; ++ rep->ret = ret; ++ _fill_msg_header(&rep->msg_header, ++ sizeof(struct cpu_dev_reply_info), action_type, 0); ++} ++#endif + #endif + + #if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO) +@@ -262,7 +287,8 @@ static int del_mmio_dev(struct devmgr_req *req, + #endif + + +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) ++#if defined(CONFIG_X86_64) + static int add_cpu_upcall(int apic_id, uint8_t apic_ver) + { + int cpu_id, node_id; +@@ -430,6 +456,58 @@ static int del_cpu_dev(struct devmgr_req *req, + cpu_event_notification(i, ret, DEL_CPU, rep); + return ret; + } ++#elif defined(CONFIG_ARM64) ++static int add_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep) ++{ ++ int i, ret = 0; ++ unsigned int cpu_id, nr_online_cpus; ++ uint8_t count = req->msg_load.cpu_dev_info.count; ++ ++ nr_online_cpus = num_online_cpus(); ++ ++ pr_info("Current vcpu number: %d, Add vcpu number: %d\n", ++ nr_online_cpus, count); ++ ++ for (i = 0; i < count; ++i) { ++ cpu_id = nr_online_cpus + i; ++ ret = add_cpu(cpu_id); ++ if (ret != 0) ++ break; ++ } ++ ++ cpu_event_notification(nr_online_cpus + i, ret, ADD_CPU, rep); ++ return ret; ++} ++ ++static int del_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep) ++{ ++ int i, ret = 0; ++ unsigned int cpu_id, nr_online_cpus; ++ uint8_t count = req->msg_load.cpu_dev_info.count; ++ ++ nr_online_cpus = num_online_cpus(); ++ ++ pr_info("Current vcpu number: %d, Delete vcpu number: %d\n", ++ nr_online_cpus, count); ++ ++ if (count >= nr_online_cpus) { ++ pr_err("cpu del parameter check error: cannot remove all vcpus\n"); ++ ret = -EINVAL; ++ cpu_event_notification(0, ret, DEL_CPU, rep); ++ return ret; ++ } ++ ++ for (i = 0; i < count; ++i) { ++ cpu_id = nr_online_cpus - i - 1; ++ ret = remove_cpu(cpu_id); ++ if (ret != 0) ++ break; ++ } ++ ++ cpu_event_notification(nr_online_cpus - i, ret, DEL_CPU, rep); ++ return ret; ++} ++#endif + #endif + + static struct { +@@ -440,7 +518,7 @@ static struct { + {ADD_MMIO, add_mmio_dev}, + {DEL_MMIO, del_mmio_dev}, + #endif +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) + {ADD_CPU, add_cpu_dev}, + {DEL_CPU, del_cpu_dev}, + #endif +-- +2.28.0 + diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch new file mode 100644 index 000000000..b40cf6666 --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch @@ -0,0 +1,67 @@ +From 6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a Mon Sep 17 00:00:00 2001 +Message-Id: <6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a.1685428663.git.jiyunxue@linux.alibaba.com> +In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Wed, 10 May 2023 14:51:40 +0800 +Subject: [PATCH 2/3] msi: control msi irq number activated + +When passthroughing pci device, kernel will initialize and activate +(max_cpu_count+1) msi irq. However, in vcpu hotplugging situation, +because of vgic, max_cpu_count may be greater than online_cpu_count. +Those offline cpus will also be activated by kernel, which cause failure +of passthroughing pci device. + +To solve this problem, this patch add a function +"check_affinity_mask_online" to check if msi_desc->affinity contains +online cpus. If current cpu is offline, it will continue the for loop to +skip activating related irq. + +Signed-off-by: xuejun-xj +Reviewed-by: Shuo Tan +Reviewed-by: Baolin Wang +--- + kernel/irq/msi.c | 20 ++++++++++++++++++++ + 1 file changed, 20 insertions(+) + +diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c +index d924676c8781..d60a3fc654e6 100644 +--- a/kernel/irq/msi.c ++++ b/kernel/irq/msi.c +@@ -395,6 +395,23 @@ static bool msi_check_reservation_mode(struct irq_domain *domain, + return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit; + } + ++/* This function is used for check whether the cpu affinity belongs to the ++ * online cpus. When we passthrough the nvme devices, the kernel will allocate ++ * maxcpus+1 MSI irqs and then activate them. In vcpu hotplug situations, it ++ * may happen that kernel activates the offline cpus when bootcpus < maxcpus. ++ * To avoid this conflict, this function check the affinities. ++ */ ++static inline bool check_affinity_mask_online(struct irq_affinity_desc *affinity) ++{ ++ int cpu; ++ ++ for_each_cpu(cpu, &affinity->mask) ++ if (cpu_online(cpu)) ++ return true; ++ ++ return false; ++} ++ + int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + int nvec) + { +@@ -445,6 +462,9 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + goto skip_activate; + + for_each_msi_vector(desc, i, dev) { ++ if (desc->affinity ++ && !check_affinity_mask_online(desc->affinity)) ++ continue; + if (desc->irq == i) { + virq = desc->irq; + dev_dbg(dev, "irq [%d-%d] for MSI\n", +-- +2.28.0 + diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch new file mode 100644 index 000000000..71d4fb977 --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch @@ -0,0 +1,139 @@ +From a05086142be13d43c7fc92500bcb870a2f37e485 Mon Sep 17 00:00:00 2001 +Message-Id: +In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Tue, 23 May 2023 09:43:02 +0800 +Subject: [PATCH 3/3] smp: update bringup_nonboot_cpus parameters + +On aarch64, kvm doesn't allow vmm to call KVM_CREATE_VCPU ioctls after +vm has already started, which is caused by vgic_initialized check in +kvm_arch_vcpu_precreate() function. Therefore, to support vcpu hotplug +feature on aarch64, all the vcpus should be created and configured ready +for start at booting procedure. + +To solve the problem, dragonball will add a property in each cpu node, +called "boot-onlined". This property indicates whether this cpu should +be onlined at first boot. It has two values: 0 and 1. 0 means offline, +while 1 means online. + +This commit also add a helper function called "of_get_cpu_boot_onlined", +which parse the cpu node and get the value of boot-onlined property. +Then update the global variable "boot_onlined_cpu". + +When kernel calling smp_init(), bringup_nonboot_cpus will start all the +other cpus except cpu0. The activated cpu number equals setup_max_cpus. +In vcpu hotplug scenario, vmm will create all the vcpufd before vm is +initialized, while activating only a few vcpus at first boot. The +setup_max_cpus variable will be initialized as all vcpu count. This +cause that the other cpus cannot find enough cpu threads, and they will +wait for 5 seconds each cpu. + +Therefore, we use boot_onlined_cpu instead of setup_max_cpus to give +"bringup_nonboot_cpus" correct cpu number it needs. + +Signed-off-by: xuejun-xj +--- + .../devicetree/bindings/arm/cpus.yaml | 11 +++++++++ + arch/arm64/kernel/smp.c | 24 +++++++++++++++++++ + kernel/smp.c | 10 +++++++- + 3 files changed, 44 insertions(+), 1 deletion(-) + +diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml +index 14cd727d3c4b..691bb352d842 100644 +--- a/Documentation/devicetree/bindings/arm/cpus.yaml ++++ b/Documentation/devicetree/bindings/arm/cpus.yaml +@@ -316,6 +316,17 @@ properties: + formed by encoding the target CPU id into the low bits of the + physical start address it should jump to. + ++ boot-onlined: ++ $ref: '/schemas/types.yaml#/definitions/uint32' ++ description: | ++ The boot-onlined property is an optional u32 value that indicates ++ whether the cpu device should be activated at first boot. This is ++ useful in vcpu hotplug scenario to pass correct value of activated ++ cpu number. ++ ++ This property has two values: 0 and 1. 1 means the cpu should be ++ activated while 0 means it shouldn't. ++ + if: + # If the enable-method property contains one of those values + properties: +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 18e9727d3f64..5db8041929a6 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -464,6 +464,27 @@ void __init smp_prepare_boot_cpu(void) + init_gic_priority_masking(); + } + ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++extern unsigned int boot_onlined_cpu; ++static void __init of_get_cpu_boot_onlined(struct device_node *dn) ++{ ++ unsigned int boot_onlined; ++ int r; ++ ++ r = of_property_read_u32(dn, "boot-onlined", &boot_onlined); ++ if (r) { ++ pr_err("%pOF: missing boot-onlined property\n", dn); ++ return; ++ } ++ /* ++ * Property boot-onlined has two values: 0 and 1. ++ * 0 means offline, and 1 means online. ++ * Here just count the number of boot_onlined_cpu. ++ */ ++ boot_onlined_cpu += boot_onlined; ++} ++#endif ++ + static u64 __init of_get_cpu_mpidr(struct device_node *dn) + { + const __be32 *cell; +@@ -654,6 +675,9 @@ static void __init of_parse_and_init_cpus(void) + struct device_node *dn; + + for_each_of_cpu_node(dn) { ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++ of_get_cpu_boot_onlined(dn); ++#endif + u64 hwid = of_get_cpu_mpidr(dn); + + if (hwid == INVALID_HWID) +diff --git a/kernel/smp.c b/kernel/smp.c +index 25240fb2df94..567615b9a008 100644 +--- a/kernel/smp.c ++++ b/kernel/smp.c +@@ -801,17 +801,25 @@ void __init setup_nr_cpu_ids(void) + nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; + } + ++/* Setup number of CPUs to activate */ ++unsigned int boot_onlined_cpu = 0; ++ + /* Called by boot processor to activate the rest. */ + void __init smp_init(void) + { + int num_nodes, num_cpus; ++ int num_onlined_cpu = setup_max_cpus; + + idle_threads_init(); + cpuhp_threads_init(); + + pr_info("Bringing up secondary CPUs ...\n"); + +- bringup_nonboot_cpus(setup_max_cpus); ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++ if (boot_onlined_cpu != 0) ++ num_onlined_cpu = boot_onlined_cpu; ++#endif ++ bringup_nonboot_cpus(num_onlined_cpu); + + num_nodes = num_online_nodes(); + num_cpus = num_online_cpus(); +-- +2.28.0 + diff --git a/tools/packaging/scripts/lib.sh b/tools/packaging/scripts/lib.sh index 64c6b650e..fbc70d6e8 100755 --- a/tools/packaging/scripts/lib.sh +++ b/tools/packaging/scripts/lib.sh @@ -8,7 +8,7 @@ export GOPATH=${GOPATH:-${HOME}/go} export tests_repo="${tests_repo:-github.com/kata-containers/tests}" export tests_repo_dir="$GOPATH/src/$tests_repo" -export CC_BUILDER_REGISTRY="quay.io/kata-containers/cc-builders" +export CC_BUILDER_REGISTRY="${CC_BUILDER_REGISTRY:-quay.io/kata-containers/cc-builders}" export PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}" this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/tools/packaging/static-build/initramfs/Dockerfile b/tools/packaging/static-build/initramfs/Dockerfile index ec92124c8..1af2e26b7 100644 --- a/tools/packaging/static-build/initramfs/Dockerfile +++ b/tools/packaging/static-build/initramfs/Dockerfile @@ -14,12 +14,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV TZ=UTC RUN apt-get update &&\ apt-get --no-install-recommends install -y software-properties-common &&\ + add-apt-repository ppa:git-core/ppa -y &&\ apt-get update && apt-get upgrade -y && \ apt-get --no-install-recommends install -y \ apt-utils \ asciidoctor \ autoconf \ - autopoint \ + autopoint \ automake \ busybox-static \ ca-certificates \ @@ -31,8 +32,8 @@ RUN apt-get update &&\ libblkid-dev \ libselinux1-dev \ libtool \ - libpopt-dev \ - libjson-c-dev \ + libpopt-dev \ + libjson-c-dev \ libssl-dev \ make \ ninja-build \ diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index 5f3562419..c6ac27727 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -30,6 +30,7 @@ package_output_dir="${package_output_dir:-}" [ -n "${lvm2_repo}" ] || die "Failed to get lvm2 repo" [ -n "${lvm2_version}" ] || die "Failed to get lvm2 version" + container_image="${INITRAMFS_CONTAINER_BUILDER:-$(get_initramfs_image_name)}" sudo docker pull ${container_image} || (sudo docker build \ diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index a9dd223f9..b36f06cff 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -25,7 +25,7 @@ sudo docker pull ${container_image} || \ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ - --env KATA_BUILD_CC="${KATA_BUILD_CC:-}" \ + --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ "${container_image}" \ bash -c "${kernel_builder} $* setup" diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index 857a4d8e0..925d15acb 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -60,7 +60,7 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \ "${container_image}" \ - bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" ${EXTRA_OPTS} install" + bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" ${EXTRA_OPTS} install" for vmm in ${VMM_CONFIGS}; do config_file="${DESTDIR}/${PREFIX}/share/defaults/kata-containers/configuration-${vmm}.toml" diff --git a/versions.yaml b/versions.yaml index 199627610..1c2ddc790 100644 --- a/versions.yaml +++ b/versions.yaml @@ -83,7 +83,7 @@ assets: uscan-url: >- https://github.com/firecracker-microvm/firecracker/tags .*/v?(\d\S+)\.tar\.gz - version: "v1.3.1" + version: "v1.3.3" qemu: description: "VMM that uses KVM" @@ -344,12 +344,12 @@ languages: rust: description: "Rust language" notes: "'version' is the default minimum version used by this project." - version: "1.68.0" + version: "1.69.0" meta: description: | 'newest-version' is the latest version known to work when building Kata - newest-version: "1.68.0" + newest-version: "1.69.0" golangci-lint: description: "golangci-lint"