diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml index 5942a5d79..ade4eb9e7 100644 --- a/.github/workflows/build-kata-static-tarball-amd64.yaml +++ b/.github/workflows/build-kata-static-tarball-amd64.yaml @@ -19,24 +19,38 @@ jobs: - cloud-hypervisor - firecracker - kernel + - kernel-sev - kernel-dragonball-experimental - kernel-tdx-experimental - - kernel-gpu - - kernel-gpu-snp - - kernel-gpu-tdx-experimental + - kernel-nvidia-gpu + - kernel-nvidia-gpu-snp + - kernel-nvidia-gpu-tdx-experimental - nydus + - ovmf + - ovmf-sev - qemu + - qemu-snp-experimental - qemu-tdx-experimental - rootfs-image - rootfs-initrd + - rootfs-initrd-sev - shim-v2 - tdvf - virtiofsd steps: + - name: Login to Kata Containers quay.io + if: ${{ inputs.push-to-registry == 'yes' }} + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 # This is needed in order to keep the commit ids history + - name: Build ${{ matrix.asset }} run: | make "${KATA_ASSET}-tarball" diff --git a/.github/workflows/build-kata-static-tarball-arm64.yaml b/.github/workflows/build-kata-static-tarball-arm64.yaml index 753bcf13a..1fc981733 100644 --- a/.github/workflows/build-kata-static-tarball-arm64.yaml +++ b/.github/workflows/build-kata-static-tarball-arm64.yaml @@ -31,6 +31,14 @@ jobs: run: | sudo chown -R $USER:$USER $GITHUB_WORKSPACE + - name: Login to Kata Containers quay.io + if: ${{ inputs.push-to-registry == 'yes' }} + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/build-kata-static-tarball-s390x.yaml b/.github/workflows/build-kata-static-tarball-s390x.yaml index 95e4a5ff5..58186ab8c 100644 --- a/.github/workflows/build-kata-static-tarball-s390x.yaml +++ b/.github/workflows/build-kata-static-tarball-s390x.yaml @@ -27,6 +27,14 @@ jobs: run: | sudo chown -R $USER:$USER $GITHUB_WORKSPACE + - name: Login to Kata Containers quay.io + if: ${{ inputs.push-to-registry == 'yes' }} + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/ci-on-push.yaml b/.github/workflows/ci-on-push.yaml index 6db1cda72..3951bba08 100644 --- a/.github/workflows/ci-on-push.yaml +++ b/.github/workflows/ci-on-push.yaml @@ -29,6 +29,22 @@ jobs: tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64 secrets: inherit + run-k8s-tests-on-sev: + needs: publish-kata-deploy-payload-amd64 + uses: ./.github/workflows/run-k8s-tests-on-sev.yaml + with: + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64 + + run-k8s-tests-on-snp: + needs: publish-kata-deploy-payload-amd64 + uses: ./.github/workflows/run-k8s-tests-on-snp.yaml + with: + registry: ghcr.io + repo: ${{ github.repository_owner }}/kata-deploy-ci + tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-amd64 + run-k8s-tests-on-tdx: needs: publish-kata-deploy-payload-amd64 uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml diff --git a/.github/workflows/commit-message-check.yaml b/.github/workflows/commit-message-check.yaml index 191e94b0d..20be9f688 100644 --- a/.github/workflows/commit-message-check.yaml +++ b/.github/workflows/commit-message-check.yaml @@ -62,6 +62,9 @@ jobs: # to be specified at the start of the regex as the action is passed # the entire commit message. # + # - This check will pass if the commit message only contains a subject + # line, as other body message properties are enforced elsewhere. + # # - Body lines *can* be longer than the maximum if they start # with a non-alphabetic character or if there is no whitespace in # the line. @@ -75,7 +78,7 @@ jobs: # # - A SoB comment can be any length (as it is unreasonable to penalise # people with long names/email addresses :) - pattern: '^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$' + pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)' error: 'Body line too long (max 150)' post_error: ${{ env.error_msg }} diff --git a/.github/workflows/payload-after-push.yaml b/.github/workflows/payload-after-push.yaml index 25a7a18c2..97bb309b1 100644 --- a/.github/workflows/payload-after-push.yaml +++ b/.github/workflows/payload-after-push.yaml @@ -10,16 +10,19 @@ jobs: uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml with: push-to-registry: yes + secrets: inherit build-assets-arm64: uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml with: push-to-registry: yes + secrets: inherit build-assets-s390x: uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml with: push-to-registry: yes + secrets: inherit publish-kata-deploy-payload-amd64: needs: build-assets-amd64 diff --git a/.github/workflows/release-amd64.yaml b/.github/workflows/release-amd64.yaml new file mode 100644 index 000000000..8d48b956b --- /dev/null +++ b/.github/workflows/release-amd64.yaml @@ -0,0 +1,51 @@ +name: Publish Kata release artifacts for amd64 +on: + workflow_call: + inputs: + target-arch: + required: true + type: string + +jobs: + build-kata-static-tarball-amd64: + uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml + + kata-deploy: + needs: build-kata-static-tarball-amd64 + runs-on: ubuntu-latest + steps: + - name: Login to Kata Containers docker.io + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to Kata Containers quay.io + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + + - uses: actions/checkout@v3 + - name: get-kata-tarball + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-amd64 + + - name: build-and-push-kata-deploy-ci-amd64 + id: build-and-push-kata-deploy-ci-amd64 + run: | + # We need to do such trick here as the format of the $GITHUB_REF + # is "refs/tags/" + tag=$(echo $GITHUB_REF | cut -d/ -f3-) + tags=($tag) + tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable")) + for tag in ${tags[@]}; do + ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \ + $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \ + "${tag}-${{ inputs.target-arch }}" + ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \ + $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \ + "${tag}-${{ inputs.target-arch }}" + done diff --git a/.github/workflows/release-arm64.yaml b/.github/workflows/release-arm64.yaml new file mode 100644 index 000000000..2b5e810a3 --- /dev/null +++ b/.github/workflows/release-arm64.yaml @@ -0,0 +1,51 @@ +name: Publish Kata release artifacts for arm64 +on: + workflow_call: + inputs: + target-arch: + required: true + type: string + +jobs: + build-kata-static-tarball-arm64: + uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml + + kata-deploy: + needs: build-kata-static-tarball-arm64 + runs-on: arm64 + steps: + - name: Login to Kata Containers docker.io + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to Kata Containers quay.io + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + + - uses: actions/checkout@v3 + - name: get-kata-tarball + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-arm64 + + - name: build-and-push-kata-deploy-ci-arm64 + id: build-and-push-kata-deploy-ci-arm64 + run: | + # We need to do such trick here as the format of the $GITHUB_REF + # is "refs/tags/" + tag=$(echo $GITHUB_REF | cut -d/ -f3-) + tags=($tag) + tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable")) + for tag in ${tags[@]}; do + ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \ + $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \ + "${tag}-${{ inputs.target-arch }}" + ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \ + $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \ + "${tag}-${{ inputs.target-arch }}" + done diff --git a/.github/workflows/release-s390x.yaml b/.github/workflows/release-s390x.yaml new file mode 100644 index 000000000..ef436b7b8 --- /dev/null +++ b/.github/workflows/release-s390x.yaml @@ -0,0 +1,51 @@ +name: Publish Kata release artifacts for s390x +on: + workflow_call: + inputs: + target-arch: + required: true + type: string + +jobs: + build-kata-static-tarball-s390x: + uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml + + kata-deploy: + needs: build-kata-static-tarball-s390x + runs-on: s390x + steps: + - name: Login to Kata Containers docker.io + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to Kata Containers quay.io + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + + - uses: actions/checkout@v3 + - name: get-kata-tarball + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-s390x + + - name: build-and-push-kata-deploy-ci-s390x + id: build-and-push-kata-deploy-ci-s390x + run: | + # We need to do such trick here as the format of the $GITHUB_REF + # is "refs/tags/" + tag=$(echo $GITHUB_REF | cut -d/ -f3-) + tags=($tag) + tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable")) + for tag in ${tags[@]}; do + ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \ + $(pwd)/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \ + "${tag}-${{ inputs.target-arch }}" + ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \ + $(pwd)/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \ + "${tag}-${{ inputs.target-arch }}" + done diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a642fa36f..af9b93132 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,73 +5,82 @@ on: - '[0-9]+.[0-9]+.[0-9]+*' jobs: - build-kata-static-tarball-amd64: - uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml + build-and-push-assets-amd64: + uses: ./.github/workflows/release-amd64.yaml + with: + target-arch: amd64 + secrets: inherit - kata-deploy: - needs: build-kata-static-tarball-amd64 + build-and-push-assets-arm64: + uses: ./.github/workflows/release-arm64.yaml + with: + target-arch: arm64 + secrets: inherit + + build-and-push-assets-s390x: + uses: ./.github/workflows/release-s390x.yaml + with: + target-arch: s390x + secrets: inherit + + publish-multi-arch-images: runs-on: ubuntu-latest + needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x] steps: - - uses: actions/checkout@v3 - - name: get-kata-tarball - uses: actions/download-artifact@v3 + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Login to Kata Containers docker.io + uses: docker/login-action@v2 with: - name: kata-static-tarball-amd64 - - name: build-and-push-kata-deploy-ci - id: build-and-push-kata-deploy-ci - run: | - tag=$(echo $GITHUB_REF | cut -d/ -f3-) - pushd $GITHUB_WORKSPACE - git checkout $tag - pkg_sha=$(git rev-parse HEAD) - popd - mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz - docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy - docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} - docker push katadocker/kata-deploy-ci:$pkg_sha - docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io - docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha - mkdir -p packaging/kata-deploy - ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action - echo "PKG_SHA=${pkg_sha}" >> $GITHUB_OUTPUT - - name: test-kata-deploy-ci-in-aks - uses: ./packaging/kata-deploy/action + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to Kata Containers quay.io + uses: docker/login-action@v2 with: - packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} - env: - PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} - AZ_APPID: ${{ secrets.AZ_APPID }} - AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} - AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} - AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - - name: push-tarball + registry: quay.io + username: ${{ secrets.QUAY_DEPLOYER_USERNAME }} + password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} + + - name: Push multi-arch manifest run: | # tag the container image we created and push to DockerHub tag=$(echo $GITHUB_REF | cut -d/ -f3-) tags=($tag) tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable")) - for tag in ${tags[@]}; do \ - docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \ - docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \ - docker push katadocker/kata-deploy:${tag} && \ - docker push quay.io/kata-containers/kata-deploy:${tag}; \ + # push to quay.io and docker.io + for tag in ${tags[@]}; do + docker manifest create quay.io/kata-containers/kata-deploy:${tag} \ + --amend quay.io/kata-containers/kata-deploy:${tag}-amd64 \ + --amend quay.io/kata-containers/kata-deploy:${tag}-arm64 \ + --amend quay.io/kata-containers/kata-deploy:${tag}-s390x + + docker manifest create docker.io/katadocker/kata-deploy:${tag} \ + --amend docker.io/katadocker/kata-deploy:${tag}-amd64 \ + --amend docker.io/katadocker/kata-deploy:${tag}-arm64 \ + --amend docker.io/katadocker/kata-deploy:${tag}-s390x + + docker manifest push quay.io/kata-containers/kata-deploy:${tag} + docker manifest push docker.io/katadocker/kata-deploy:${tag} done - upload-static-tarball: - needs: kata-deploy + upload-multi-arch-static-tarball: + needs: publish-multi-arch-images runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: download-artifacts - uses: actions/download-artifact@v3 - with: - name: kata-static-tarball - name: install hub run: | HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//') wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \ tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub - - name: push static tarball to github + + - name: download-artifacts-amd64 + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-amd64 + - name: push amd64 static tarball to github run: | tag=$(echo $GITHUB_REF | cut -d/ -f3-) tarball="kata-static-$tag-x86_64.tar.xz" @@ -81,8 +90,36 @@ jobs: GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" popd + - name: download-artifacts-arm64 + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-arm64 + - name: push arm64 static tarball to github + run: | + tag=$(echo $GITHUB_REF | cut -d/ -f3-) + tarball="kata-static-$tag-aarch64.tar.xz" + mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}" + pushd $GITHUB_WORKSPACE + echo "uploading asset '${tarball}' for tag: ${tag}" + GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" + popd + + - name: download-artifacts-s390x + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-s390x + - name: push s390x static tarball to github + run: | + tag=$(echo $GITHUB_REF | cut -d/ -f3-) + tarball="kata-static-$tag-s390x.tar.xz" + mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}" + pushd $GITHUB_WORKSPACE + echo "uploading asset '${tarball}' for tag: ${tag}" + GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" + popd + upload-cargo-vendored-tarball: - needs: upload-static-tarball + needs: upload-multi-arch-static-tarball runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index f9a26debb..4ed5a2d30 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -61,7 +61,8 @@ jobs: run: | az aks get-credentials -g "kataCI" -n ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.vmm }}-amd64 - - name: Deploy kata-deploy + - name: Run tests + timeout-minutes: 35 run: | sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -72,9 +73,11 @@ jobs: kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - name: Run tests - timeout-minutes: 30 - run: | + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + sleep 150s + pushd tests/integration/kubernetes sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml bash run_kubernetes_tests.sh diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml new file mode 100644 index 000000000..98a6db610 --- /dev/null +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -0,0 +1,68 @@ +name: CI | Run kubernetes tests on SEV +on: + workflow_call: + inputs: + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + +jobs: + run-k8s-tests: + strategy: + fail-fast: false + matrix: + vmm: + - qemu-sev + runs-on: sev + env: + KUBECONFIG: /home/kata/.kube/config + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Run tests + timeout-minutes: 30 + run: | + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" + + kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml + kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + sleep 60s + + pushd tests/integration/kubernetes + sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml + bash run_kubernetes_tests.sh + popd + env: + KATA_HYPERVISOR: ${{ matrix.vmm }} + + - name: Delete kata-deploy + if: always() + run: | + kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" + kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + sleep 180s + + kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml + kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml new file mode 100644 index 000000000..541695e0f --- /dev/null +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -0,0 +1,68 @@ +name: CI | Run kubernetes tests on SEV-SNP +on: + workflow_call: + inputs: + registry: + required: true + type: string + repo: + required: true + type: string + tag: + required: true + type: string + +jobs: + run-k8s-tests: + strategy: + fail-fast: false + matrix: + vmm: + - qemu-snp + runs-on: sev-snp + env: + KUBECONFIG: /home/kata/.kube/config + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Run tests + timeout-minutes: 30 + run: | + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" + + kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml + kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + sleep 60s + + pushd tests/integration/kubernetes + sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml + bash run_kubernetes_tests.sh + popd + env: + KATA_HYPERVISOR: ${{ matrix.vmm }} + + - name: Delete kata-deploy + if: always() + run: | + kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" + kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + sleep 180s + + kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml + kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 1777a16c8..6c5ba0dc1 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -27,7 +27,8 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - - name: Deploy kata-deploy + - name: Run tests + timeout-minutes: 30 run: | sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -38,9 +39,11 @@ jobs: kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - name: Run tests - timeout-minutes: 30 - run: | + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + sleep 60s + pushd tests/integration/kubernetes sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml bash run_kubernetes_tests.sh @@ -57,9 +60,9 @@ jobs: sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s sleep 180s - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml diff --git a/.github/workflows/snap-release.yaml b/.github/workflows/snap-release.yaml index 0d41197a0..4a12ce8d7 100644 --- a/.github/workflows/snap-release.yaml +++ b/.github/workflows/snap-release.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Check out Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -32,6 +32,7 @@ jobs: run: | # Removing man-db, workflow kept failing, fixes: #4480 sudo apt -y remove --purge man-db + sudo apt-get update sudo apt-get install -y git git-extras kata_url="https://github.com/kata-containers/kata-containers" latest_version=$(git ls-remote --tags ${kata_url} | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1) diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml index 2f1495824..ac163fa2e 100644 --- a/.github/workflows/snap.yaml +++ b/.github/workflows/snap.yaml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.gitignore b/.gitignore index fca150940..29d21ac6d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ **/.vscode **/.idea **/.fleet +**/*.swp +**/*.swo pkg/logging/Cargo.lock src/agent/src/version.rs src/agent/kata-agent.service diff --git a/Makefile b/Makefile index 8e856138c..e70af93e4 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ TOOLS = TOOLS += agent-ctl TOOLS += kata-ctl TOOLS += log-parser +TOOLS += log-parser-rs TOOLS += runk TOOLS += trace-forwarder diff --git a/README.md b/README.md index 6972ed278..662899ea6 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ The table below lists the remaining parts of the project: | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. | | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. | | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. | +| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. | | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. | | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. | | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. | diff --git a/VERSION b/VERSION index 2f81ab203..bb48c8b0a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-alpha0 +3.2.0-alpha3 diff --git a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md index a87c9be62..3a56cbba2 100644 --- a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md +++ b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md @@ -44,12 +44,11 @@ $ popd - Build a custom QEMU ```bash $ source kata-containers/tools/packaging/scripts/lib.sh -$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu.snp.url")" -$ qemu_branch="$(get_from_kata_deps "assets.hypervisor.qemu.snp.branch")" -$ qemu_commit="$(get_from_kata_deps "assets.hypervisor.qemu.snp.commit")" -$ git clone -b "${qemu_branch}" "${qemu_url}" +$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")" +$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")" +$ git clone "${qemu_url}" $ pushd qemu -$ git checkout "${qemu_commit}" +$ git checkout "${qemu_tag}" $ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug $ make -j "$(nproc)" $ popd diff --git a/docs/install/README.md b/docs/install/README.md index 0ed42d87f..8a86bc7c8 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -19,7 +19,7 @@ Packaged installation methods uses your distribution's native package format (su |------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------| | [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | | [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | -| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | +| ~~[Using snap](#snap-installation)~~ | ~~Easy to install~~ | ~~yes~~ | **Snap is unmaintained!** ~~Good alternative to official distro packages.~~ | | [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | | [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | | [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | @@ -44,9 +44,24 @@ Kata packages are provided by official distribution repositories for: ### Snap Installation -The snap installation is available for all distributions which support `snapd`. +> **WARNING:** +> +> The Snap package method is **unmaintained** and only provides an old +> version of Kata Containers: +> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) +> provides Kata Containers +> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) +> but the latest stable Kata Containers release at the time of writing is +> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). +> +> We recommend strongly that you switch to an alternative Kata Containers installation method. +> +> See: https://github.com/kata-containers/kata-containers/issues/6769 +> for further details. -[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. +~~The snap installation is available for all distributions which support `snapd`.~~ + +~~[Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. ~~ ### Automatic Installation diff --git a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md index f83e4ea02..4cfcb392d 100644 --- a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md +++ b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md @@ -49,14 +49,14 @@ Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md). * Download `Rustup` and install `Rust` > **Notes:** - > For Rust version, please see [`versions.yaml`](../../versions.yaml) file's rust section. + > For Rust version, please set `RUST_VERSION` to the value of `languages.rust.meta.newest-version key` in [`versions.yaml`](../../versions.yaml) or, if `yq` is available on your system, run `export RUST_VERSION=$(yq read versions.yaml languages.rust.meta.newest-version)`. Example for `x86_64` ``` $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh $ source $HOME/.cargo/env - $ rustup install 1.62.0 - $ rustup default 1.62.0-x86_64-unknown-linux-gnu + $ rustup install ${RUST_VERSION} + $ rustup default ${RUST_VERSION}-x86_64-unknown-linux-gnu ``` * Musl support for fully static binary diff --git a/docs/install/snap-installation-guide.md b/docs/install/snap-installation-guide.md index 0f22c2211..acf2e637e 100644 --- a/docs/install/snap-installation-guide.md +++ b/docs/install/snap-installation-guide.md @@ -1,5 +1,20 @@ # Kata Containers snap package +> **WARNING:** +> +> The Snap package method is **unmaintained** and only provides an old +> version of Kata Containers: +> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) +> provides Kata Containers +> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) +> but the latest stable Kata Containers release at the time of writing is +> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). +> +> We recommend strongly that you switch to an alternative Kata Containers installation method. +> +> See: https://github.com/kata-containers/kata-containers/issues/6769 +> for further details. + ## Install Kata Containers Kata Containers can be installed in any Linux distribution that supports @@ -7,6 +22,21 @@ Kata Containers can be installed in any Linux distribution that supports Run the following command to install **Kata Containers**: +> **WARNING:** +> +> The Snap package method is **unmaintained** and only provides an old +> version of Kata Containers: +> The [latest Kata Containers snap](https://snapcraft.io/kata-containers) +> provides Kata Containers +> [version 2.4.2](https://github.com/kata-containers/kata-containers/releases/tag/2.4.2) +> but the latest stable Kata Containers release at the time of writing is +> [version 3.1.0](https://github.com/kata-containers/kata-containers/releases/tag/3.1.0). +> +> We recommend strongly that you switch to an alternative Kata Containers installation method. +> +> See: https://github.com/kata-containers/kata-containers/issues/6769 +> for further details. + ```sh $ sudo snap install kata-containers --stable --classic ``` diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index eaec36ffb..fefec4a2d 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -4831,9 +4831,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 25627e46e..a90f355a5 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -31,7 +31,7 @@ async-recursion = "0.3.2" futures = "0.3.17" # Async runtime -tokio = { version = "1.21.2", features = ["full"] } +tokio = { version = "1.28.1", features = ["full"] } tokio-vsock = "0.3.1" netlink-sys = { version = "0.7.0", features = ["tokio_socket",]} diff --git a/src/agent/Makefile b/src/agent/Makefile index 51da58d3a..69423edda 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -33,6 +33,12 @@ ifeq ($(SECCOMP),yes) override EXTRA_RUSTFEATURES += seccomp endif +include ../../utils.mk + +ifeq ($(ARCH), ppc64le) + override ARCH = powerpc64le +endif + ##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature STANDARD_OCI_RUNTIME := no @@ -45,8 +51,6 @@ ifneq ($(EXTRA_RUSTFEATURES),) override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)" endif -include ../../utils.mk - TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) ##VAR DESTDIR= is a directory prepended to each installed target file diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index 19602bee2..3113ac643 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -29,7 +29,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" } rlimit = "0.5.3" cfg-if = "0.1.0" -tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] } +tokio = { version = "1.28.1", features = ["sync", "io-util", "process", "time", "macros", "rt"] } futures = "0.3.17" async-trait = "0.1.31" inotify = "0.9.2" diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 76922024d..d9ba15041 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -35,7 +35,7 @@ use crate::log_child; // struct is populated from the content in the /proc//mountinfo file. #[derive(std::fmt::Debug, PartialEq)] pub struct Info { - mount_point: String, + pub mount_point: String, optional: String, fstype: String, } @@ -553,7 +553,7 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> { // Parse /proc/self/mountinfo because comparing Dev and ino does not work from // bind mounts -fn parse_mount_table(mountinfo_path: &str) -> Result> { +pub fn parse_mount_table(mountinfo_path: &str) -> Result> { let file = File::open(mountinfo_path)?; let reader = BufReader::new(file); let mut infos = Vec::new(); diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index 72e1d7dfb..8a299a2ae 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -234,7 +234,7 @@ impl AgentConfig { let config_position = args.iter().position(|a| a == "--config" || a == "-c"); if let Some(config_position) = config_position { if let Some(config_file) = args.get(config_position + 1) { - return AgentConfig::from_config_file(config_file); + return AgentConfig::from_config_file(config_file).context("AgentConfig from args"); } else { panic!("The config argument wasn't formed properly: {:?}", args); } @@ -254,7 +254,9 @@ impl AgentConfig { // or if it can't be parsed properly. if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) { let config_file = get_string_value(param)?; - config = AgentConfig::from_config_file(&config_file)?; + config = AgentConfig::from_config_file(&config_file) + .context("AgentConfig from kernel cmdline") + .unwrap(); using_config_file = true; break; } @@ -365,7 +367,8 @@ impl AgentConfig { #[instrument] pub fn from_config_file(file: &str) -> Result { - let config = fs::read_to_string(file)?; + let config = fs::read_to_string(file) + .with_context(|| format!("Failed to read config file {}", file))?; AgentConfig::from_str(&config) } diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 8331ecb53..f292299dc 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -34,7 +34,7 @@ macro_rules! sl { } const VM_ROOTFS: &str = "/"; - +const BLOCK: &str = "block"; pub const DRIVER_9P_TYPE: &str = "9p"; pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs"; pub const DRIVER_BLK_TYPE: &str = "blk"; @@ -204,7 +204,7 @@ impl ScsiBlockMatcher { impl UeventMatcher for ScsiBlockMatcher { fn is_match(&self, uev: &Uevent) -> bool { - uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty() + uev.subsystem == BLOCK && uev.devpath.contains(&self.search) && !uev.devname.is_empty() } } @@ -238,7 +238,7 @@ impl VirtioBlkPciMatcher { impl UeventMatcher for VirtioBlkPciMatcher { fn is_match(&self, uev: &Uevent) -> bool { - uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty() + uev.subsystem == BLOCK && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty() } } @@ -311,7 +311,7 @@ impl PmemBlockMatcher { impl UeventMatcher for PmemBlockMatcher { fn is_match(&self, uev: &Uevent) -> bool { - uev.subsystem == "block" + uev.subsystem == BLOCK && uev.devpath.starts_with(ACPI_DEV_PATH) && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty() @@ -441,6 +441,48 @@ async fn wait_for_ap_device(sandbox: &Arc>, address: ap::Address) Ok(()) } +#[derive(Debug)] +struct MmioBlockMatcher { + suffix: String, +} + +impl MmioBlockMatcher { + fn new(devname: &str) -> MmioBlockMatcher { + MmioBlockMatcher { + suffix: format!(r"/block/{}", devname), + } + } +} + +impl UeventMatcher for MmioBlockMatcher { + fn is_match(&self, uev: &Uevent) -> bool { + uev.subsystem == BLOCK && uev.devpath.ends_with(&self.suffix) && !uev.devname.is_empty() + } +} + +#[instrument] +pub async fn get_virtio_mmio_device_name( + sandbox: &Arc>, + devpath: &str, +) -> Result<()> { + let devname = devpath + .strip_prefix("/dev/") + .ok_or_else(|| anyhow!("Storage source '{}' must start with /dev/", devpath))?; + + let matcher = MmioBlockMatcher::new(devname); + let uev = wait_for_uevent(sandbox, matcher) + .await + .context("failed to wait for uevent")?; + if uev.devname != devname { + return Err(anyhow!( + "Unexpected device name {} for mmio device (expected {})", + uev.devname, + devname + )); + } + Ok(()) +} + /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN) #[instrument] fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { @@ -678,12 +720,18 @@ pub fn update_env_pci( #[instrument] async fn virtiommio_blk_device_handler( device: &Device, - _sandbox: &Arc>, + sandbox: &Arc>, ) -> Result { if device.vm_path.is_empty() { return Err(anyhow!("Invalid path for virtio mmio blk device")); } + if !Path::new(&device.vm_path).exists() { + get_virtio_mmio_device_name(sandbox, &device.vm_path.to_string()) + .await + .context("failed to get mmio device name")?; + } + Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into()) } @@ -1396,7 +1444,7 @@ mod tests { let mut uev = crate::uevent::Uevent::default(); uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string(); - uev.subsystem = "block".to_string(); + uev.subsystem = BLOCK.to_string(); uev.devpath = devpath.clone(); uev.devname = devname.to_string(); @@ -1430,7 +1478,7 @@ mod tests { let mut uev_a = crate::uevent::Uevent::default(); let relpath_a = "/0000:00:0a.0"; uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string(); - uev_a.subsystem = "block".to_string(); + uev_a.subsystem = BLOCK.to_string(); uev_a.devname = devname.to_string(); uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname); let matcher_a = VirtioBlkPciMatcher::new(relpath_a); @@ -1514,7 +1562,7 @@ mod tests { let mut uev_a = crate::uevent::Uevent::default(); let addr_a = "0:0"; uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string(); - uev_a.subsystem = "block".to_string(); + uev_a.subsystem = BLOCK.to_string(); uev_a.devname = devname.to_string(); uev_a.devpath = format!( "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda", @@ -1557,6 +1605,33 @@ mod tests { assert!(!matcher_a.is_match(&uev_b)); } + #[tokio::test] + async fn test_mmio_block_matcher() { + let devname_a = "vda"; + let devname_b = "vdb"; + let mut uev_a = crate::uevent::Uevent::default(); + uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string(); + uev_a.subsystem = BLOCK.to_string(); + uev_a.devname = devname_a.to_string(); + uev_a.devpath = format!( + "/sys/devices/virtio-mmio-cmdline/virtio-mmio.0/virtio0/block/{}", + devname_a + ); + let matcher_a = MmioBlockMatcher::new(devname_a); + + let mut uev_b = uev_a.clone(); + uev_b.devpath = format!( + "/sys/devices/virtio-mmio-cmdline/virtio-mmio.4/virtio4/block/{}", + devname_b + ); + let matcher_b = MmioBlockMatcher::new(devname_b); + + assert!(matcher_a.is_match(&uev_a)); + assert!(matcher_b.is_match(&uev_b)); + assert!(!matcher_b.is_match(&uev_a)); + assert!(!matcher_a.is_match(&uev_b)); + } + #[test] fn test_split_vfio_pci_option() { assert_eq!( diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 0eff266f2..5b0d95c19 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -21,10 +21,11 @@ use nix::unistd::{Gid, Uid}; use regex::Regex; use crate::device::{ - get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device, - DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, - DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, - DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, FS_TYPE_HUGETLB, + get_scsi_device_name, get_virtio_blk_pci_device_name, get_virtio_mmio_device_name, + online_device, wait_for_pmem_device, DRIVER_9P_TYPE, DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, + DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, + DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, DRIVER_VIRTIOFS_TYPE, DRIVER_WATCHABLE_BIND_TYPE, + FS_TYPE_HUGETLB, }; use crate::linux_abi::*; use crate::pci; @@ -473,8 +474,14 @@ async fn virtiommio_blk_storage_handler( storage: &Storage, sandbox: Arc>, ) -> Result { + let storage = storage.clone(); + if !Path::new(&storage.source).exists() { + get_virtio_mmio_device_name(&sandbox, &storage.source) + .await + .context("failed to get mmio device name")?; + } //The source path is VmPath - common_storage_handler(logger, storage) + common_storage_handler(logger, &storage) } // virtiofs_storage_handler handles the storage for virtio-fs. diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index b99ac8950..46c282bb9 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -43,6 +43,7 @@ use protocols::{ }; use rustjail::cgroups::notifier; use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT}; +use rustjail::mount::parse_mount_table; use rustjail::process::Process; use rustjail::specconv::CreateOpts; @@ -104,6 +105,7 @@ const USR_IP6TABLES_SAVE: &str = "/usr/sbin/ip6tables-save"; const IP6TABLES_SAVE: &str = "/sbin/ip6tables-save"; const USR_IP6TABLES_RESTORE: &str = "/usr/sbin/ip6tables-save"; const IP6TABLES_RESTORE: &str = "/sbin/ip6tables-restore"; +const KATA_GUEST_SHARE_DIR: &str = "/run/kata-containers/shared/containers/"; const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer"; const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size"; @@ -947,6 +949,29 @@ impl agent_ttrpc::AgentService for AgentService { Ok(Empty::new()) } + async fn remove_stale_virtiofs_share_mounts( + &self, + ctx: &TtrpcContext, + req: protocols::agent::RemoveStaleVirtiofsShareMountsRequest, + ) -> ttrpc::Result { + trace_rpc_call!(ctx, "remove_stale_virtiofs_share_mounts", req); + is_allowed!(req); + let mount_infos = parse_mount_table("/proc/self/mountinfo") + .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?; + for m in &mount_infos { + if m.mount_point.starts_with(KATA_GUEST_SHARE_DIR) { + // stat the mount point, virtiofs daemon will remove the stale cache and release the fds if the mount point doesn't exist any more. + // More details in https://github.com/kata-containers/kata-containers/issues/6455#issuecomment-1477137277 + match stat::stat(Path::new(&m.mount_point)) { + Ok(_) => info!(sl!(), "stat {} success", m.mount_point), + Err(e) => info!(sl!(), "stat {} failed: {}", m.mount_point, e), + } + } + } + + Ok(Empty::new()) + } + async fn write_stdin( &self, _ctx: &TtrpcContext, diff --git a/src/agent/src/signal.rs b/src/agent/src/signal.rs index a22892db3..d67000b80 100644 --- a/src/agent/src/signal.rs +++ b/src/agent/src/signal.rs @@ -24,7 +24,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc>) -> Result loop { // Avoid reaping the undesirable child's signal, e.g., execute_hook's // The lock should be released immediately. - let _ = rustjail::container::WAIT_PID_LOCKER.lock().await; + let _locker = rustjail::container::WAIT_PID_LOCKER.lock().await; let result = wait::waitpid( Some(Pid::from_raw(-1)), Some(WaitPidFlag::WNOHANG | WaitPidFlag::__WALL), diff --git a/src/agent/vsock-exporter/Cargo.toml b/src/agent/vsock-exporter/Cargo.toml index 0cdf0b91d..7bec1d87a 100644 --- a/src/agent/vsock-exporter/Cargo.toml +++ b/src/agent/vsock-exporter/Cargo.toml @@ -18,4 +18,4 @@ bincode = "1.3.3" byteorder = "1.4.3" slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] } async-trait = "0.1.50" -tokio = "1.2.0" +tokio = "1.28.1" diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 56affcacf..a271d04cb 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -486,7 +486,9 @@ impl VmmService { VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot) })?; - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + vm.device_manager_mut() + .block_manager + .insert_device(ctx, config) .map(|_| VmmData::Empty) .map_err(VmmActionError::Block) } @@ -500,7 +502,9 @@ impl VmmService { ) -> VmmRequestResult { let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; - BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + vm.device_manager_mut() + .block_manager + .update_device_ratelimiters(config) .map(|_| VmmData::Empty) .map_err(VmmActionError::Block) } @@ -518,7 +522,9 @@ impl VmmService { .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|_| VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot))?; - BlockDeviceMgr::remove_device(vm.device_manager_mut(), ctx, drive_id) + vm.device_manager_mut() + .block_manager + .remove_device(ctx, drive_id) .map(|_| VmmData::Empty) .map_err(VmmActionError::Block) } @@ -543,7 +549,9 @@ impl VmmService { } })?; - VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + vm.device_manager_mut() + .virtio_net_manager + .insert_device(ctx, config) .map(|_| VmmData::Empty) .map_err(VmmActionError::VirtioNet) } @@ -556,7 +564,9 @@ impl VmmService { ) -> VmmRequestResult { let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; - VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + vm.device_manager_mut() + .virtio_net_manager + .update_device_ratelimiters(config) .map(|_| VmmData::Empty) .map_err(VmmActionError::VirtioNet) } diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs index e1c89d2fa..ff74fb925 100644 --- a/src/dragonball/src/config_manager.rs +++ b/src/dragonball/src/config_manager.rs @@ -278,6 +278,11 @@ where self.info_list.iter_mut() } + /// Remove the last device config info from the `info_list`. + pub fn pop(&mut self) -> Option> { + self.info_list.pop() + } + fn get_index_by_id(&self, config: &T) -> Option { self.info_list .iter() diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 66890d7f9..f7cdfa9b0 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -340,7 +340,7 @@ impl BlockDeviceMgr { /// the existing entry. /// Inserting a secondary root block device will fail. pub fn insert_device( - device_mgr: &mut DeviceManager, + &mut self, mut ctx: DeviceOpContext, config: BlockDeviceConfigInfo, ) -> std::result::Result<(), BlockDeviceError> { @@ -348,10 +348,8 @@ impl BlockDeviceMgr { return Err(BlockDeviceError::UpdateNotAllowedPostBoot); } - let mgr = &mut device_mgr.block_manager; - // If the id of the drive already exists in the list, the operation is update. - match mgr.get_index_of_drive_id(config.id()) { + match self.get_index_of_drive_id(config.id()) { Some(index) => { // No support for runtime update yet. if ctx.is_hotplug { @@ -359,19 +357,19 @@ impl BlockDeviceMgr { config.path_on_host.clone(), )) } else { - for (idx, info) in mgr.info_list.iter().enumerate() { + for (idx, info) in self.info_list.iter().enumerate() { if idx != index { info.config.check_conflicts(&config)?; } } - mgr.update(index, config) + self.update(index, config) } } None => { - for info in mgr.info_list.iter() { + for info in self.info_list.iter() { info.config.check_conflicts(&config)?; } - let index = mgr.create(config.clone())?; + let index = self.create(config.clone())?; if !ctx.is_hotplug { return Ok(()); } @@ -383,17 +381,16 @@ impl BlockDeviceMgr { let dev = DeviceManager::create_mmio_virtio_device( device, &mut ctx, - config.use_shared_irq.unwrap_or(mgr.use_shared_irq), + config.use_shared_irq.unwrap_or(self.use_shared_irq), config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), ) .map_err(BlockDeviceError::DeviceManager)?; - mgr.update_device_by_index(index, Arc::clone(&dev))?; + self.update_device_by_index(index, Arc::clone(&dev))?; // live-upgrade need save/restore device from info.device. - mgr.info_list[index].set_device(dev.clone()); + self.info_list[index].set_device(dev.clone()); ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| { let logger = ctx.logger().new(slog::o!()); - BlockDeviceMgr::remove_device(device_mgr, ctx, &config.drive_id) - .unwrap(); + self.remove_device(ctx, &config.drive_id).unwrap(); error!( logger, "failed to hot-add virtio block device {}, {:?}", @@ -466,7 +463,7 @@ impl BlockDeviceMgr { /// remove a block device, it basically is the inverse operation of `insert_device`` pub fn remove_device( - dev_mgr: &mut DeviceManager, + &mut self, mut ctx: DeviceOpContext, drive_id: &str, ) -> std::result::Result<(), BlockDeviceError> { @@ -474,8 +471,7 @@ impl BlockDeviceMgr { return Err(BlockDeviceError::UpdateNotAllowedPostBoot); } - let mgr = &mut dev_mgr.block_manager; - match mgr.remove(drive_id) { + match self.remove(drive_id) { Some(mut info) => { info!(ctx.logger(), "remove drive {}", info.config.drive_id); if let Some(device) = info.device.take() { @@ -731,15 +727,14 @@ impl BlockDeviceMgr { /// Update the ratelimiter settings of a virtio blk device. pub fn update_device_ratelimiters( - device_mgr: &mut DeviceManager, + &mut self, new_cfg: BlockDeviceConfigUpdateInfo, ) -> std::result::Result<(), BlockDeviceError> { - let mgr = &mut device_mgr.block_manager; - match mgr.get_index_of_drive_id(&new_cfg.drive_id) { + match self.get_index_of_drive_id(&new_cfg.drive_id) { Some(index) => { - let config = &mut mgr.info_list[index].config; + let config = &mut self.info_list[index].config; config.rate_limiter = new_cfg.rate_limiter.clone(); - let device = mgr.info_list[index] + let device = self.info_list[index] .device .as_mut() .ok_or_else(|| BlockDeviceError::InvalidDeviceId("".to_owned()))?; @@ -827,12 +822,11 @@ mod tests { let mut vm = crate::vm::tests::create_vm_instance(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - assert!(BlockDeviceMgr::insert_device( - vm.device_manager_mut(), - ctx, - dummy_block_device.clone(), - ) - .is_ok()); + assert!(vm + .device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device.clone(),) + .is_ok()); assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); assert!(!vm.device_manager().block_manager.has_root_block_device()); @@ -897,7 +891,9 @@ mod tests { use_shared_irq: None, use_generic_irq: None, }; - BlockDeviceMgr::insert_device(vm.device_manager_mut(), device_op_ctx, dummy_block_device) + vm.device_manager_mut() + .block_manager + .insert_device(device_op_ctx, dummy_block_device) .unwrap(); let cfg = BlockDeviceConfigUpdateInfo { @@ -923,7 +919,9 @@ mod tests { let expected_error = "could not send patch message to the block epoll handler".to_string(); assert_eq!( - BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg) + vm.device_manager_mut() + .block_manager + .update_device_ratelimiters(cfg) .unwrap_err() .to_string(), expected_error @@ -938,7 +936,9 @@ mod tests { let expected_error = format!("invalid block device id '{0}'", cfg2.drive_id); assert_eq!( - BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), cfg2) + vm.device_manager_mut() + .block_manager + .update_device_ratelimiters(cfg2) .unwrap_err() .to_string(), expected_error @@ -968,12 +968,11 @@ mod tests { let mut vm = crate::vm::tests::create_vm_instance(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - assert!(BlockDeviceMgr::insert_device( - vm.device_manager_mut(), - ctx, - dummy_block_device.clone(), - ) - .is_ok()); + assert!(vm + .device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device.clone(),) + .is_ok()); assert_eq!(vm.device_manager().block_manager.info_list.len(), 1); assert!(vm.device_manager().block_manager.has_root_block); @@ -1027,12 +1026,16 @@ mod tests { let mut vm = crate::vm::tests::create_vm_instance(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_1).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device_1) + .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - assert!( - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_2) - .is_err() - ); + assert!(vm + .device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device_2) + .is_err()); } #[test] @@ -1112,13 +1115,22 @@ mod tests { assert_eq!(vm.device_manager().block_manager.info_list.len(), 3); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device) + .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_2) + .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_3) + .unwrap(); } #[test] @@ -1182,13 +1194,19 @@ mod tests { let mut vm = crate::vm::tests::create_vm_instance(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + vm.device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_2.clone()) .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_3.clone()) + vm.device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_3.clone()) .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device.clone()) + vm.device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device.clone()) .unwrap(); assert!(vm.device_manager().block_manager.has_root_block_device(),); @@ -1255,9 +1273,14 @@ mod tests { // Add 2 block devices. let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device) + .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + vm.device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_2.clone()) .unwrap(); // Get index zero. @@ -1286,7 +1309,9 @@ mod tests { // Update OK. dummy_block_device_2.is_read_only = true; let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2.clone()) + vm.device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_2.clone()) .unwrap(); let index = vm @@ -1306,21 +1331,21 @@ mod tests { let dummy_path_3 = PathBuf::from(dummy_filename_3); dummy_block_device_2.path_on_host = dummy_path_3; let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - assert!(BlockDeviceMgr::insert_device( - vm.device_manager_mut(), - ctx, - dummy_block_device_2.clone(), - ) - .is_err()); + assert!(vm + .device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_2.clone(),) + .is_err()); // Update with 2 root block devices. dummy_block_device_2.path_on_host = dummy_path_2.clone(); dummy_block_device_2.is_root_device = true; let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - assert!( - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, dummy_block_device_2,) - .is_err(), - ); + assert!(vm + .device_manager_mut() + .block_manager + .insert_device(ctx, dummy_block_device_2,) + .is_err(),); // Switch roots and add a PARTUUID for the new one. let root_block_device_old = BlockDeviceConfigInfo { @@ -1354,9 +1379,15 @@ mod tests { use_generic_irq: None, }; let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_old).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device_old) + .unwrap(); let ctx = DeviceOpContext::create_boot_ctx(&vm, None); - BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, root_block_device_new).unwrap(); + vm.device_manager_mut() + .block_manager + .insert_device(ctx, root_block_device_new) + .unwrap(); assert!(vm.device_manager().block_manager.has_part_uuid_root); } } diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index ed651f577..766c5eef9 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -714,6 +714,14 @@ impl DeviceManager { #[cfg(feature = "virtio-blk")] self.block_manager.remove_devices(&mut ctx)?; + // FIXME: To acquire the full abilities for gracefully removing + // virtio-net and virtio-vsock devices, updating dragonball-sandbox + // is required. + #[cfg(feature = "virtio-net")] + self.virtio_net_manager.remove_devices(&mut ctx)?; + #[cfg(feature = "virtio-vsock")] + self.vsock_manager.remove_devices(&mut ctx)?; + Ok(()) } } diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs index c0b0f62da..f57c4ed67 100644 --- a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs +++ b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs @@ -223,7 +223,7 @@ impl VirtioNetDeviceMgr { /// Insert or update a virtio net device into the manager. pub fn insert_device( - device_mgr: &mut DeviceManager, + &mut self, mut ctx: DeviceOpContext, config: VirtioNetDeviceConfigInfo, ) -> std::result::Result<(), VirtioNetDeviceError> { @@ -234,8 +234,6 @@ impl VirtioNetDeviceMgr { return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot); } - let mgr = &mut device_mgr.virtio_net_manager; - slog::info!( ctx.logger(), "add virtio-net device configuration"; @@ -244,7 +242,7 @@ impl VirtioNetDeviceMgr { "host_dev_name" => &config.host_dev_name, ); - let device_index = mgr.info_list.insert_or_update(&config)?; + let device_index = self.info_list.insert_or_update(&config)?; if ctx.is_hotplug { slog::info!( @@ -260,17 +258,17 @@ impl VirtioNetDeviceMgr { let dev = DeviceManager::create_mmio_virtio_device( device, &mut ctx, - config.use_shared_irq.unwrap_or(mgr.use_shared_irq), + config.use_shared_irq.unwrap_or(self.use_shared_irq), config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), ) .map_err(VirtioNetDeviceError::DeviceManager)?; ctx.insert_hotplug_mmio_device(&dev, None) .map_err(VirtioNetDeviceError::DeviceManager)?; // live-upgrade need save/restore device from info.device. - mgr.info_list[device_index].set_device(dev); + self.info_list[device_index].set_device(dev); } Err(e) => { - mgr.info_list.remove(device_index); + self.info_list.remove(device_index); return Err(VirtioNetDeviceError::Virtio(e)); } } @@ -281,16 +279,15 @@ impl VirtioNetDeviceMgr { /// Update the ratelimiter settings of a virtio net device. pub fn update_device_ratelimiters( - device_mgr: &mut DeviceManager, + &mut self, new_cfg: VirtioNetDeviceConfigUpdateInfo, ) -> std::result::Result<(), VirtioNetDeviceError> { - let mgr = &mut device_mgr.virtio_net_manager; - match mgr.get_index_of_iface_id(&new_cfg.iface_id) { + match self.get_index_of_iface_id(&new_cfg.iface_id) { Some(index) => { - let config = &mut mgr.info_list[index].config; + let config = &mut self.info_list[index].config; config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone(); config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone(); - let device = mgr.info_list[index].device.as_mut().ok_or_else(|| { + let device = self.info_list[index].device.as_mut().ok_or_else(|| { VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone()) })?; @@ -374,6 +371,21 @@ impl VirtioNetDeviceMgr { Ok(Box::new(net_device)) } + + /// Remove all virtio-net devices. + pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + while let Some(mut info) = self.info_list.pop() { + slog::info!( + ctx.logger(), + "remove virtio-net device: {}", + info.config.iface_id + ); + if let Some(device) = info.device.take() { + DeviceManager::destroy_mmio_virtio_device(device, ctx)?; + } + } + Ok(()) + } } impl Default for VirtioNetDeviceMgr { diff --git a/src/dragonball/src/device_manager/vsock_dev_mgr.rs b/src/dragonball/src/device_manager/vsock_dev_mgr.rs index 8588471b7..791d9ded6 100644 --- a/src/dragonball/src/device_manager/vsock_dev_mgr.rs +++ b/src/dragonball/src/device_manager/vsock_dev_mgr.rs @@ -17,7 +17,7 @@ use dbs_virtio_devices::vsock::Vsock; use dbs_virtio_devices::Error as VirtioError; use serde_derive::{Deserialize, Serialize}; -use super::StartMicroVmError; +use super::{DeviceMgrError, StartMicroVmError}; use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; use crate::device_manager::{DeviceManager, DeviceOpContext}; @@ -284,6 +284,21 @@ impl VsockDeviceMgr { // safe to unwrap, because we created the inner connector before Ok(self.default_inner_connector.clone().unwrap()) } + + /// Remove all virtio-vsock devices + pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + while let Some(mut info) = self.info_list.pop() { + slog::info!( + ctx.logger(), + "remove virtio-vsock device: {}", + info.config.id + ); + if let Some(device) = info.device.take() { + DeviceManager::destroy_mmio_virtio_device(device, ctx)?; + } + } + Ok(()) + } } impl Default for VsockDeviceMgr { diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index 04cf4605c..9593e8276 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -7,7 +7,6 @@ // found in the THIRD-PARTY file. use std::convert::TryInto; -use std::mem; use std::ops::Deref; use dbs_address_space::AddressSpace; @@ -16,8 +15,9 @@ use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY}; use linux_loader::cmdline::Cmdline; +use linux_loader::configurator::{linux::LinuxBootConfigurator, BootConfigurator, BootParams}; use slog::info; -use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; +use vm_memory::{Address, GuestAddress, GuestAddressSpace, GuestMemory}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, Result, StartMicroVmError}; @@ -110,15 +110,11 @@ fn configure_system( } } - let zero_page_addr = GuestAddress(layout::ZERO_PAGE_START); - guest_mem - .checked_offset(zero_page_addr, mem::size_of::()) - .ok_or(Error::ZeroPagePastRamEnd)?; - guest_mem - .write_obj(params, zero_page_addr) - .map_err(|_| Error::ZeroPageSetup)?; - - Ok(()) + LinuxBootConfigurator::write_bootparams( + &BootParams::new(¶ms, GuestAddress(layout::ZERO_PAGE_START)), + guest_mem, + ) + .map_err(|_| Error::ZeroPageSetup) } impl Vm { diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 16af5ab28..3af0563c1 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -417,10 +417,10 @@ impl Annotation { match self.get_value::(KATA_ANNO_CONTAINER_RES_SWAPPINESS) { Ok(r) => { if r.unwrap_or_default() > 100 { - return Err(io::Error::new( + Err(io::Error::new( io::ErrorKind::InvalidData, format!("{} greater than 100", r.unwrap_or_default()), - )); + )) } else { Ok(r) } diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 98ec142f4..f30ab9a23 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -80,6 +80,7 @@ pub struct Agent { pub kernel_modules: Vec, /// container pipe size + #[serde(default)] pub container_pipe_size: u32, } diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 1d5e2b4d3..6b5495c5a 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -16,6 +16,7 @@ lazy_static! { pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![ "/etc/kata-containers/configuration.toml", "/usr/share/defaults/kata-containers/configuration.toml", + "/opt/kata/share/defaults/kata-containers/configuration.toml", ]; } diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs index bb72944a2..405731945 100644 --- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -12,7 +12,7 @@ use super::{default, register_hypervisor_plugin}; use crate::config::default::MAX_DRAGONBALL_VCPUS; use crate::config::default::MIN_DRAGONBALL_MEMORY_SIZE_MB; use crate::config::hypervisor::{ - VIRTIO_BLK, VIRTIO_BLK_MMIO, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM, + VIRTIO_BLK_MMIO, VIRTIO_BLK_PCI, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM, }; use crate::config::{ConfigPlugin, TomlConfig}; use crate::{eother, resolve_path, validate_path}; @@ -107,7 +107,7 @@ impl ConfigPlugin for DragonballConfig { } if !db.blockdev_info.disable_block_device_use - && db.blockdev_info.block_device_driver != VIRTIO_BLK + && db.blockdev_info.block_device_driver != VIRTIO_BLK_PCI && db.blockdev_info.block_device_driver != VIRTIO_BLK_MMIO && db.blockdev_info.block_device_driver != VIRTIO_PMEM { diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 7818b897c..fedbba14b 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -43,8 +43,8 @@ pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU}; mod ch; pub use self::ch::{CloudHypervisorConfig, HYPERVISOR_NAME_CH}; -const VIRTIO_BLK: &str = "virtio-blk"; -const VIRTIO_BLK_MMIO: &str = "virtio-mmio"; +const VIRTIO_BLK_PCI: &str = "virtio-blk-pci"; +const VIRTIO_BLK_MMIO: &str = "virtio-blk-mmio"; const VIRTIO_BLK_CCW: &str = "virtio-blk-ccw"; const VIRTIO_SCSI: &str = "virtio-scsi"; const VIRTIO_PMEM: &str = "nvdimm"; @@ -172,7 +172,7 @@ impl BlockDeviceInfo { return Ok(()); } let l = [ - VIRTIO_BLK, + VIRTIO_BLK_PCI, VIRTIO_BLK_CCW, VIRTIO_BLK_MMIO, VIRTIO_PMEM, diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 173026921..e73f8b84e 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -127,6 +127,14 @@ impl TomlConfig { result } + /// Load raw Kata configuration information from default configuration file. + /// + /// Configuration file is probed according to the default configuration file list + /// default::DEFAULT_RUNTIME_CONFIGURATIONS. + pub fn load_from_default() -> Result<(TomlConfig, PathBuf)> { + Self::load_raw_from_file("") + } + /// Load raw Kata configuration information from configuration files. /// /// If `config_file` is valid, it will used, otherwise a built-in default path list will be @@ -196,7 +204,7 @@ impl TomlConfig { } /// Probe configuration file according to the default configuration file list. - fn get_default_config_file() -> Result { + pub fn get_default_config_file() -> Result { for f in default::DEFAULT_RUNTIME_CONFIGURATIONS.iter() { if let Ok(path) = fs::canonicalize(f) { return Ok(path); diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index da4377f0d..3ad755256 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -38,6 +38,7 @@ service AgentService { rpc StatsContainer(StatsContainerRequest) returns (StatsContainerResponse); rpc PauseContainer(PauseContainerRequest) returns (google.protobuf.Empty); rpc ResumeContainer(ResumeContainerRequest) returns (google.protobuf.Empty); + rpc RemoveStaleVirtiofsShareMounts(RemoveStaleVirtiofsShareMountsRequest) returns (google.protobuf.Empty); // stdio rpc WriteStdin(WriteStreamRequest) returns (WriteStreamResponse); @@ -301,6 +302,8 @@ message CreateSandboxRequest { message DestroySandboxRequest { } +message RemoveStaleVirtiofsShareMountsRequest {} + message Interfaces { repeated types.Interface Interfaces = 1; } diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index a68b485d1..276facdbb 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -61,6 +61,17 @@ dependencies = [ "url", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.8", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -221,7 +232,7 @@ checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -276,7 +287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -584,7 +595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -611,7 +622,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 1.0.109", ] [[package]] @@ -628,7 +639,7 @@ checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -793,7 +804,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -816,6 +827,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "dragonball" version = "0.1.0" @@ -1073,7 +1090,7 @@ checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1176,6 +1193,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] [[package]] name = "heck" @@ -1281,6 +1301,7 @@ dependencies = [ name = "hypervisor" version = "0.1.0" dependencies = [ + "actix-rt", "anyhow", "async-trait", "ch-config", @@ -1296,6 +1317,7 @@ dependencies = [ "nix 0.24.3", "persist", "rand 0.8.5", + "rust-ini", "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "seccompiler", "serde", @@ -1979,6 +2001,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown", +] + [[package]] name = "parking" version = "2.0.0" @@ -2087,7 +2119,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2130,9 +2162,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] @@ -2175,7 +2207,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2267,9 +2299,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ "proc-macro2", ] @@ -2510,6 +2542,16 @@ dependencies = [ "wasm_container", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -2602,7 +2644,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2635,7 +2677,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2810,9 +2852,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2849,7 +2891,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 1.0.109", ] [[package]] @@ -2879,6 +2921,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "take_mut" version = "0.2.2" @@ -2949,7 +3002,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3035,14 +3088,13 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.26.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ "autocfg", "bytes 1.4.0", "libc", - "memchr", "mio", "num_cpus", "parking_lot 0.12.1", @@ -3050,18 +3102,18 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.16", ] [[package]] @@ -3135,7 +3187,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3454,7 +3506,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -3488,7 +3540,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3577,13 +3629,13 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", ] [[package]] @@ -3592,7 +3644,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -3601,13 +3662,28 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -3616,42 +3692,84 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "xattr" version = "0.2.3" diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 599949bb0..5304290ca 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -17,6 +17,10 @@ CONTAINERD_RUNTIME_NAME = io.containerd.kata.v2 include ../../utils.mk +ifeq ($(ARCH), ppc64le) + override ARCH = powerpc64le +endif + ARCH_DIR = arch ARCH_FILE_SUFFIX = -options.mk ARCH_FILE = $(ARCH_DIR)/$(ARCH)$(ARCH_FILE_SUFFIX) @@ -199,7 +203,7 @@ ifneq (,$(DBCMD)) CONFIGS += $(CONFIG_DB) # dragonball-specific options (all should be suffixed by "_DB") DEFMAXVCPUS_DB := 1 - DEFBLOCKSTORAGEDRIVER_DB := virtio-blk + DEFBLOCKSTORAGEDRIVER_DB := virtio-blk-mmio DEFNETWORKMODEL_DB := tcfilter KERNELPARAMS = console=ttyS1 agent.log_vport=1025 KERNELTYPE_DB = uncompressed diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml index 4475c6d47..7639cb4f7 100644 --- a/src/runtime-rs/crates/agent/Cargo.toml +++ b/src/runtime-rs/crates/agent/Cargo.toml @@ -18,7 +18,7 @@ serde_json = ">=1.0.9" slog = "2.5.2" slog-scope = "4.4.0" ttrpc = { version = "0.7.1" } -tokio = { version = "1.8.0", features = ["fs", "rt"] } +tokio = { version = "1.28.1", features = ["fs", "rt"] } url = "2.2.2" nix = "0.24.2" diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index de8822980..da6e14430 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -20,14 +20,9 @@ impl Empty { } } -impl Default for FSGroupChangePolicy { - fn default() -> Self { - FSGroupChangePolicy::Always - } -} - -#[derive(Debug, Clone, PartialEq)] +#[derive(Default, Debug, Clone, PartialEq)] pub enum FSGroupChangePolicy { + #[default] Always = 0, OnRootMismatch = 1, } @@ -65,18 +60,13 @@ pub struct Storage { pub mount_point: String, } -#[derive(Deserialize, Clone, PartialEq, Eq, Debug, Hash)] +#[derive(Deserialize, Default, Clone, PartialEq, Eq, Debug, Hash)] pub enum IPFamily { + #[default] V4 = 0, V6 = 1, } -impl ::std::default::Default for IPFamily { - fn default() -> Self { - IPFamily::V4 - } -} - #[derive(Deserialize, Debug, PartialEq, Clone, Default)] pub struct IPAddress { pub family: IPFamily, diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index c06b6c404..eb613aad3 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -8,6 +8,7 @@ license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +actix-rt = "2.7.0" anyhow = "^1.0" async-trait = "0.1.48" dbs-utils = "0.2.0" @@ -15,13 +16,14 @@ go-flag = "0.1.0" libc = ">=0.2.39" nix = "0.24.2" persist = { path = "../persist" } +rust-ini = "0.18.0" seccompiler = "0.2.0" serde = { version = "1.0.138", features = ["derive"] } serde_json = ">=1.0.9" slog = "2.5.2" slog-scope = "4.4.0" thiserror = "1.0" -tokio = { version = "1.8.0", features = ["sync", "fs"] } +tokio = { version = "1.28.1", features = ["sync", "fs"] } vmm-sys-util = "0.11.0" rand = "0.8.4" diff --git a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml index 10ed105e3..0e0b45e59 100644 --- a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" anyhow = "1.0.68" serde = { version = "1.0.145", features = ["rc", "derive"] } serde_json = "1.0.91" -tokio = { version = "1.25.0", features = ["sync", "rt"] } +tokio = { version = "1.28.1", features = ["sync", "rt"] } # Cloud Hypervisor public HTTP API functions # Note that the version specified is not necessarily the version of CH diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs index b2bd32176..23b076b16 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs @@ -4,7 +4,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::HypervisorState; -use crate::device::Device; +use crate::device::DeviceType; use crate::VmmState; use anyhow::Result; use async_trait::async_trait; @@ -44,7 +44,7 @@ pub struct CloudHypervisorInner { pub(crate) jailer_root: String, /// List of devices that will be added to the VM once it boots - pub(crate) pending_devices: Option>, + pub(crate) pending_devices: Option>, pub(crate) _capabilities: Capabilities, diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs index f4475f66e..e06832cbd 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs @@ -5,8 +5,9 @@ // SPDX-License-Identifier: Apache-2.0 use super::inner::CloudHypervisorInner; -use crate::device::{Device, ShareFsDeviceConfig}; +use crate::device::DeviceType; use crate::HybridVsockConfig; +use crate::ShareFsDeviceConfig; use crate::VmmState; use anyhow::{anyhow, Context, Result}; use ch_config::ch_api::cloud_hypervisor_vm_fs_add; @@ -18,9 +19,9 @@ use std::path::PathBuf; const VIRTIO_FS: &str = "virtio-fs"; impl CloudHypervisorInner { - pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> { + pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> { if self.state != VmmState::VmRunning { - let mut devices: Vec = if let Some(devices) = self.pending_devices.take() { + let mut devices: Vec = if let Some(devices) = self.pending_devices.take() { devices } else { vec![] @@ -38,11 +39,11 @@ impl CloudHypervisorInner { Ok(()) } - async fn handle_add_device(&mut self, device: Device) -> Result<()> { + async fn handle_add_device(&mut self, device: DeviceType) -> Result<()> { match device { - Device::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await, - Device::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await, - _ => return Err(anyhow!("unhandled device: {:?}", device)), + DeviceType::ShareFs(sharefs) => self.handle_share_fs_device(sharefs.config).await, + DeviceType::HybridVsock(hvsock) => self.handle_hvsock_device(&hvsock.config).await, + _ => Err(anyhow!("unhandled device: {:?}", device)), } } @@ -66,7 +67,7 @@ impl CloudHypervisorInner { Ok(()) } - pub(crate) async fn remove_device(&mut self, _device: Device) -> Result<()> { + pub(crate) async fn remove_device(&mut self, _device: DeviceType) -> Result<()> { Ok(()) } @@ -132,8 +133,8 @@ impl CloudHypervisorInner { if let Some(devices) = pending_root_devices { for dev in devices { match dev { - Device::ShareFsDevice(dev) => { - let settings = ShareFsSettings::new(dev, self.vm_path.clone()); + DeviceType::ShareFs(dev) => { + let settings = ShareFsSettings::new(dev.config, self.vm_path.clone()); let fs_cfg = FsConfig::try_from(settings)?; diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 89747d936..9cf5bcd8c 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -6,9 +6,9 @@ use super::inner::CloudHypervisorInner; use crate::ch::utils::get_api_socket_path; use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path}; +use crate::device::DeviceType; use crate::kernel_param::KernelParams; -use crate::Device; -use crate::VsockConfig; +use crate::VsockDevice; use crate::VM_ROOTFS_DRIVER_PMEM; use crate::{VcpuThreadIds, VmmState}; use anyhow::{anyhow, Context, Result}; @@ -228,11 +228,9 @@ impl CloudHypervisorInner { let join_handle = self.cloud_hypervisor_ping_until_ready(CH_POLL_TIME_MS); - let result = tokio::time::timeout(Duration::new(timeout_secs as u64, 0), join_handle) + tokio::time::timeout(Duration::new(timeout_secs as u64, 0), join_handle) .await - .context(timeout_msg)?; - - result + .context(timeout_msg)? } async fn cloud_hypervisor_ensure_not_launched(&self) -> Result<()> { @@ -419,10 +417,11 @@ impl CloudHypervisorInner { self.netns = netns; - let vsock_cfg = VsockConfig::new(self.id.clone()).await?; + let vsock_dev = VsockDevice::new(self.id.clone()).await?; - let dev = Device::Vsock(vsock_cfg); - self.add_device(dev).await.context("add vsock device")?; + self.add_device(DeviceType::Vsock(vsock_dev)) + .await + .context("add vsock device")?; self.start_hypervisor(self.timeout_secs).await?; diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index f8f44710e..a4b8b05ff 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -4,7 +4,8 @@ // SPDX-License-Identifier: Apache-2.0 use super::HypervisorState; -use crate::{device::Device, Hypervisor, VcpuThreadIds}; +use crate::device::DeviceType; +use crate::{Hypervisor, VcpuThreadIds}; use anyhow::{Context, Result}; use async_trait::async_trait; use kata_types::capabilities::Capabilities; @@ -78,12 +79,12 @@ impl Hypervisor for CloudHypervisor { inner.save_vm().await } - async fn add_device(&self, device: Device) -> Result<()> { + async fn add_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.add_device(device).await } - async fn remove_device(&self, device: Device) -> Result<()> { + async fn remove_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.remove_device(device).await } diff --git a/src/runtime-rs/crates/hypervisor/src/device/block.rs b/src/runtime-rs/crates/hypervisor/src/device/block.rs deleted file mode 100644 index 4f59cc0ea..000000000 --- a/src/runtime-rs/crates/hypervisor/src/device/block.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2019-2022 Alibaba Cloud -// Copyright (c) 2019-2022 Ant Group -// -// SPDX-License-Identifier: Apache-2.0 -// - -#[derive(Debug)] -pub struct BlockConfig { - /// Unique identifier of the drive. - pub id: String, - - /// Path of the drive. - pub path_on_host: String, - - /// If set to true, the drive is opened in read-only mode. Otherwise, the - /// drive is opened as read-write. - pub is_readonly: bool, - - /// Don't close `path_on_host` file when dropping the device. - pub no_drop: bool, - - /// device index - pub index: u64, -} diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs new file mode 100644 index 000000000..c633cf918 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -0,0 +1,239 @@ +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{collections::HashMap, sync::Arc}; + +use anyhow::{anyhow, Context, Result}; +use kata_sys_util::rand::RandomBytes; +use tokio::sync::Mutex; + +use crate::{ + BlockConfig, BlockDevice, Hypervisor, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, + VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, +}; + +use super::{ + util::{get_host_path, get_virt_drive_name}, + Device, DeviceConfig, +}; +pub type ArcMutexDevice = Arc>; + +/// block_index and released_block_index are used to search an available block index +/// in Sandbox. +/// +/// @block_index generally default is 1 for ; +/// @released_block_index for blk devices removed and indexes will released at the same time. +#[derive(Clone, Debug, Default)] +struct SharedInfo { + block_index: u64, + released_block_index: Vec, +} + +impl SharedInfo { + fn new() -> Self { + SharedInfo { + block_index: 1, + released_block_index: vec![], + } + } + + // declare the available block index + fn declare_device_index(&mut self) -> Result { + let current_index = if let Some(index) = self.released_block_index.pop() { + index + } else { + self.block_index + }; + self.block_index += 1; + + Ok(current_index) + } + + fn release_device_index(&mut self, index: u64) { + self.released_block_index.push(index); + self.released_block_index.sort_by(|a, b| b.cmp(a)); + } +} + +// Device manager will manage the lifecycle of sandbox device +pub struct DeviceManager { + devices: HashMap, + hypervisor: Arc, + shared_info: SharedInfo, +} + +impl DeviceManager { + pub fn new(hypervisor: Arc) -> Result { + let devices = HashMap::::new(); + Ok(DeviceManager { + devices, + hypervisor, + shared_info: SharedInfo::new(), + }) + } + + pub async fn new_device(&mut self, device_config: &DeviceConfig) -> Result { + let device_id = if let Some(dev) = self.find_device(device_config).await { + dev + } else { + self.create_device(device_config) + .await + .context("failed to create device")? + }; + Ok(device_id) + } + + pub async fn try_add_device(&mut self, device_id: &str) -> Result<()> { + // find the device + let device = self + .devices + .get(device_id) + .context("failed to find device")?; + let mut device_guard = device.lock().await; + // attach device + let result = device_guard.attach(self.hypervisor.as_ref()).await; + // handle attach error + if let Err(e) = result { + if let DeviceConfig::BlockCfg(config) = device_guard.get_device_info().await { + self.shared_info.release_device_index(config.index); + }; + drop(device_guard); + self.devices.remove(device_id); + return Err(e); + } + Ok(()) + } + + pub async fn try_remove_device(&mut self, device_id: &str) -> Result<()> { + if let Some(dev) = self.devices.get(device_id) { + let mut device_guard = dev.lock().await; + let result = match device_guard.detach(self.hypervisor.as_ref()).await { + Ok(index) => { + if let Some(i) = index { + // release the declared block device index + self.shared_info.release_device_index(i); + } + Ok(()) + } + Err(e) => Err(e), + }; + if result.is_ok() { + drop(device_guard); + // if detach success, remove it from device manager + self.devices.remove(device_id); + } + return result; + } + Err(anyhow!( + "device with specified ID hasn't been created. {}", + device_id + )) + } + + pub async fn get_device_info(&self, device_id: &str) -> Result { + if let Some(dev) = self.devices.get(device_id) { + return Ok(dev.lock().await.get_device_info().await); + } + Err(anyhow!( + "device with specified ID hasn't been created. {}", + device_id + )) + } + + async fn find_device(&self, device_config: &DeviceConfig) -> Option { + for (device_id, dev) in &self.devices { + match dev.lock().await.get_device_info().await { + DeviceConfig::BlockCfg(config) => match device_config { + DeviceConfig::BlockCfg(ref config_new) => { + if config_new.path_on_host == config.path_on_host { + return Some(device_id.to_string()); + } + } + _ => { + continue; + } + }, + _ => { + // TODO: support find other device type + continue; + } + } + } + None + } + + async fn create_device(&mut self, device_config: &DeviceConfig) -> Result { + // device ID must be generated by manager instead of device itself + // in case of ID collision + let device_id = self.new_device_id()?; + let dev: ArcMutexDevice = match device_config { + DeviceConfig::BlockCfg(config) => self + .create_block_device(config, device_id.clone()) + .await + .context("failed to create device")?, + _ => { + return Err(anyhow!("invliad device type")); + } + }; + // register device to devices + self.devices.insert(device_id.clone(), dev.clone()); + Ok(device_id) + } + + async fn create_block_device( + &mut self, + config: &BlockConfig, + device_id: String, + ) -> Result { + let mut block_config = config.clone(); + // get hypervisor block driver + let block_driver = match self + .hypervisor + .hypervisor_config() + .await + .blockdev_info + .block_device_driver + .as_str() + { + // convert the block driver to kata type + VIRTIO_BLOCK_MMIO => KATA_MMIO_BLK_DEV_TYPE.to_string(), + VIRTIO_BLOCK_PCI => KATA_BLK_DEV_TYPE.to_string(), + _ => "".to_string(), + }; + block_config.driver_option = block_driver; + // generate virt path + let current_index = self.shared_info.declare_device_index()?; + block_config.index = current_index; + let drive_name = get_virt_drive_name(current_index as i32)?; + block_config.virt_path = format!("/dev/{}", drive_name); + // if the path on host is empty, we need to get device host path from the device major and minor number + // Otherwise, it might be rawfile based block device, the host path is already passed from the runtime, so we don't need to do anything here + if block_config.path_on_host.is_empty() { + block_config.path_on_host = get_host_path("b".to_owned(), config.major, config.minor) + .context("failed to get host path")?; + } + Ok(Arc::new(Mutex::new(BlockDevice::new( + device_id, + block_config, + )))) + } + + // device ID must be generated by device manager instead of device itself + // in case of ID collision + fn new_device_id(&self) -> Result { + for _ in 0..5 { + let rand_bytes = RandomBytes::new(8); + let id = format!("{:x}", rand_bytes); + + // check collision in devices + if self.devices.get(&id).is_none() { + return Ok(id); + } + } + + Err(anyhow!("ID are exhausted")) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs new file mode 100644 index 000000000..45487e3b5 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs @@ -0,0 +1,23 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod vhost_user; +mod virtio_blk; +pub use virtio_blk::{ + BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, VIRTIO_BLOCK_MMIO, + VIRTIO_BLOCK_PCI, +}; +mod virtio_net; +pub use virtio_net::{Address, NetworkConfig, NetworkDevice}; +mod vfio; +pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig, VfioDevice}; +mod virtio_fs; +pub use virtio_fs::{ + ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType, + ShareFsOperation, +}; +mod virtio_vsock; +pub use virtio_vsock::{HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice}; diff --git a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs similarity index 85% rename from src/runtime-rs/crates/hypervisor/src/device/vfio.rs rename to src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs index 608091379..ff3a579a8 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vfio.rs @@ -6,9 +6,13 @@ use std::{fs, path::Path, process::Command}; +use crate::device::Device; +use crate::device::DeviceConfig; +use crate::Hypervisor as hypervisor; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use anyhow::anyhow; use anyhow::{Context, Result}; +use async_trait::async_trait; fn override_driver(bdf: &str, driver: &str) -> Result<()> { let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf); @@ -25,7 +29,7 @@ const VFIO_UNBIND_PATH: &str = "/sys/bus/pci/drivers/vfio-pci/unbind"; pub const VFIO_PCI: &str = "vfio-pci"; -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum VfioBusMode { PCI, MMIO, @@ -40,11 +44,8 @@ impl VfioBusMode { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct VfioConfig { - /// Unique identifier of the device - pub id: String, - /// Sysfs path for mdev bus type device pub sysfs_path: String, @@ -55,6 +56,15 @@ pub struct VfioConfig { pub mode: VfioBusMode, } +#[derive(Debug, Clone)] +pub struct VfioDevice { + /// Unique identifier of the device + pub id: String, + + /// Config info for Vfio Device + pub config: VfioConfig, +} + /// binds the device to vfio driver after unbinding from host. /// Will be called by a network interface or a generic pcie device. pub fn bind_device_to_vfio(bdf: &str, host_driver: &str, _vendor_device_id: &str) -> Result<()> { @@ -145,3 +155,26 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str Ok(()) } + +#[async_trait] +impl Device for VfioConfig { + async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> { + todo!() + } + + async fn detach(&mut self, _h: &dyn hypervisor) -> Result> { + todo!() + } + + async fn get_device_info(&self) -> DeviceConfig { + todo!() + } + + async fn increase_attach_count(&mut self) -> Result { + todo!() + } + + async fn decrease_attach_count(&mut self) -> Result { + todo!() + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs new file mode 100644 index 000000000..d778c4459 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/vhost_user.rs @@ -0,0 +1,61 @@ +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use crate::device::Device; +use crate::device::DeviceConfig; +use crate::Hypervisor as hypervisor; +use anyhow::Result; +use async_trait::async_trait; + +#[derive(Debug, Clone, Default)] +/// VhostUserConfig represents data shared by most vhost-user devices +pub struct VhostUserConfig { + /// Device id + pub dev_id: String, + /// Socket path + pub socket_path: String, + /// Mac_address is only meaningful for vhost user net device + pub mac_address: String, + /// These are only meaningful for vhost user fs devices + pub tag: String, + pub cache: String, + pub device_type: String, + /// Pci_addr is the PCI address used to identify the slot at which the drive is attached. + pub pci_addr: Option, + /// Block index of the device if assigned + pub index: u8, + pub cache_size: u32, + pub queue_siez: u32, +} + +#[derive(Debug, Clone, Default)] +pub struct VhostUserDevice { + pub device_id: String, + pub config: VhostUserConfig, +} + +#[async_trait] +impl Device for VhostUserConfig { + async fn attach(&mut self, _h: &dyn hypervisor) -> Result<()> { + todo!() + } + + async fn detach(&mut self, _h: &dyn hypervisor) -> Result> { + todo!() + } + + async fn get_device_info(&self) -> DeviceConfig { + todo!() + } + + async fn increase_attach_count(&mut self) -> Result { + todo!() + } + + async fn decrease_attach_count(&mut self) -> Result { + todo!() + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs new file mode 100644 index 000000000..2ff98a1e7 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs @@ -0,0 +1,134 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio"; +use crate::device::Device; +use crate::device::{DeviceConfig, DeviceType}; +use crate::Hypervisor as hypervisor; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +/// VIRTIO_BLOCK_PCI indicates block driver is virtio-pci based +pub const VIRTIO_BLOCK_PCI: &str = "virtio-blk-pci"; +pub const KATA_MMIO_BLK_DEV_TYPE: &str = "mmioblk"; +pub const KATA_BLK_DEV_TYPE: &str = "blk"; + +#[derive(Debug, Clone, Default)] +pub struct BlockConfig { + /// Path of the drive. + pub path_on_host: String, + + /// If set to true, the drive is opened in read-only mode. Otherwise, the + /// drive is opened as read-write. + pub is_readonly: bool, + + /// Don't close `path_on_host` file when dropping the device. + pub no_drop: bool, + + /// device index + pub index: u64, + + /// driver type for block device + pub driver_option: String, + + /// device path in guest + pub virt_path: String, + + /// device attach count + pub attach_count: u64, + + /// device major number + pub major: i64, + + /// device minor number + pub minor: i64, +} + +#[derive(Debug, Clone, Default)] +pub struct BlockDevice { + pub device_id: String, + pub attach_count: u64, + pub config: BlockConfig, +} + +impl BlockDevice { + // new creates a new VirtioBlkDevice + pub fn new(device_id: String, config: BlockConfig) -> Self { + BlockDevice { + device_id, + attach_count: 0, + config, + } + } +} + +#[async_trait] +impl Device for BlockDevice { + async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> { + // increase attach count, skip attach the device if the device is already attached + if self + .increase_attach_count() + .await + .context("failed to increase attach count")? + { + return Ok(()); + } + if let Err(e) = h.add_device(DeviceType::Block(self.clone())).await { + self.decrease_attach_count().await?; + return Err(e); + } + return Ok(()); + } + + async fn detach(&mut self, h: &dyn hypervisor) -> Result> { + // get the count of device detached, skip detach once it reaches the 0 + if self + .decrease_attach_count() + .await + .context("failed to decrease attach count")? + { + return Ok(None); + } + if let Err(e) = h.remove_device(DeviceType::Block(self.clone())).await { + self.increase_attach_count().await?; + return Err(e); + } + Ok(Some(self.config.index)) + } + + async fn get_device_info(&self) -> DeviceConfig { + DeviceConfig::BlockCfg(self.config.clone()) + } + + async fn increase_attach_count(&mut self) -> Result { + match self.attach_count { + 0 => { + // do real attach + self.attach_count += 1; + Ok(false) + } + std::u64::MAX => Err(anyhow!("device was attached too many times")), + _ => { + self.attach_count += 1; + Ok(true) + } + } + } + + async fn decrease_attach_count(&mut self) -> Result { + match self.attach_count { + 0 => Err(anyhow!("detaching a device that wasn't attached")), + 1 => { + // do real wrok + self.attach_count -= 1; + Ok(false) + } + _ => { + self.attach_count -= 1; + Ok(true) + } + } + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs similarity index 57% rename from src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs rename to src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs index 85f516456..d2d3cc762 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs @@ -11,14 +11,14 @@ pub enum ShareFsOperation { Update, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum ShareFsMountType { PASSTHROUGH, RAFS, } /// ShareFsMountConfig: share fs mount config -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ShareFsMountConfig { /// source: the passthrough fs exported dir or rafs meta file of rafs pub source: String, @@ -41,3 +41,35 @@ pub struct ShareFsMountConfig { /// prefetch_list_path: path to file that contains file lists that should be prefetched by rafs pub prefetch_list_path: Option, } + +#[derive(Debug, Clone)] +pub struct ShareFsMountDevice { + pub config: ShareFsMountConfig, +} + +/// ShareFsDeviceConfig: share fs device config +#[derive(Debug, Clone)] +pub struct ShareFsDeviceConfig { + /// fs_type: virtiofs or inline-virtiofs + pub fs_type: String, + + /// socket_path: socket path for virtiofs + pub sock_path: String, + + /// mount_tag: a label used as a hint to the guest. + pub mount_tag: String, + + /// host_path: the host filesystem path for this volume. + pub host_path: String, + + /// queue_size: queue size + pub queue_size: u64, + + /// queue_num: queue number + pub queue_num: u64, +} + +#[derive(Debug, Clone)] +pub struct ShareFsDevice { + pub config: ShareFsDeviceConfig, +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/network.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs similarity index 80% rename from src/runtime-rs/crates/hypervisor/src/device/network.rs rename to src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs index 6c13a9ca1..3be861ced 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/network.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_net.rs @@ -6,6 +6,7 @@ use std::fmt; +#[derive(Clone)] pub struct Address(pub [u8; 6]); impl fmt::Debug for Address { @@ -19,14 +20,20 @@ impl fmt::Debug for Address { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct NetworkConfig { - /// Unique identifier of the device - pub id: String, - /// Host level path for the guest network interface. pub host_dev_name: String, /// Guest MAC address. pub guest_mac: Option
, } + +#[derive(Debug, Clone)] +pub struct NetworkDevice { + /// Unique identifier of the device + pub id: String, + + /// Network Device config info + pub config: NetworkConfig, +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/vsock.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs similarity index 84% rename from src/runtime-rs/crates/hypervisor/src/device/vsock.rs rename to src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs index cd0553adf..d94e8f864 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/vsock.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs @@ -11,9 +11,6 @@ use tokio::fs::{File, OpenOptions}; #[derive(Debug)] pub struct HybridVsockConfig { - /// Unique identifier of the device - pub id: String, - /// A 32-bit Context Identifier (CID) used to identify the guest. pub guest_cid: u32, @@ -22,10 +19,16 @@ pub struct HybridVsockConfig { } #[derive(Debug)] -pub struct VsockConfig { +pub struct HybridVsockDevice { /// Unique identifier of the device pub id: String, + /// config information for HybridVsockDevice + pub config: HybridVsockConfig, +} + +#[derive(Debug)] +pub struct VsockConfig { /// A 32-bit Context Identifier (CID) used to identify the guest. pub guest_cid: u32, @@ -33,6 +36,15 @@ pub struct VsockConfig { pub vhost_fd: File, } +#[derive(Debug)] +pub struct VsockDevice { + /// Unique identifier of the device + pub id: String, + + /// config information for VsockDevice + pub config: VsockConfig, +} + const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock"; // From @@ -50,7 +62,7 @@ nix::ioctl_write_ptr!( const CID_RETRY_COUNT: u32 = 50; -impl VsockConfig { +impl VsockDevice { pub async fn new(id: String) -> Result { let vhost_fd = OpenOptions::new() .read(true) @@ -72,10 +84,12 @@ impl VsockConfig { unsafe { vhost_vsock_set_guest_cid(vhost_fd.as_raw_fd(), &(rand_cid as u64)) }; match guest_cid { Ok(_) => { - return Ok(VsockConfig { + return Ok(VsockDevice { id, - guest_cid: rand_cid, - vhost_fd, + config: VsockConfig { + guest_cid: rand_cid, + vhost_fd, + }, }); } Err(nix::Error::EADDRINUSE) => { diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs index bbd14fb1d..d341d9a12 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs @@ -1,37 +1,67 @@ -// Copyright (c) 2019-2022 Alibaba Cloud -// Copyright (c) 2019-2022 Ant Group +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group // // SPDX-License-Identifier: Apache-2.0 // -mod block; -pub use block::BlockConfig; -mod network; -pub use network::{Address, NetworkConfig}; -mod share_fs_device; -pub use share_fs_device::ShareFsDeviceConfig; -mod vfio; -pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig}; -mod share_fs_mount; -pub use share_fs_mount::{ShareFsMountConfig, ShareFsMountType, ShareFsOperation}; -mod vsock; -pub use vsock::{HybridVsockConfig, VsockConfig}; - use std::fmt; +use crate::{ + BlockConfig, BlockDevice, HybridVsockConfig, HybridVsockDevice, Hypervisor as hypervisor, + NetworkConfig, NetworkDevice, ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, + ShareFsMountDevice, VfioConfig, VfioDevice, VsockConfig, VsockDevice, +}; +use anyhow::Result; +use async_trait::async_trait; + +pub mod device_manager; +pub mod driver; +pub mod util; + #[derive(Debug)] -pub enum Device { - Block(BlockConfig), - Network(NetworkConfig), - ShareFsDevice(ShareFsDeviceConfig), - Vfio(VfioConfig), - ShareFsMount(ShareFsMountConfig), - Vsock(VsockConfig), - HybridVsock(HybridVsockConfig), +pub enum DeviceConfig { + BlockCfg(BlockConfig), + NetworkCfg(NetworkConfig), + ShareFsCfg(ShareFsDeviceConfig), + VfioCfg(VfioConfig), + ShareFsMountCfg(ShareFsMountConfig), + VsockCfg(VsockConfig), + HybridVsockCfg(HybridVsockConfig), } -impl fmt::Display for Device { +#[derive(Debug)] +pub enum DeviceType { + Block(BlockDevice), + Vfio(VfioDevice), + Network(NetworkDevice), + ShareFs(ShareFsDevice), + ShareFsMount(ShareFsMountDevice), + HybridVsock(HybridVsockDevice), + Vsock(VsockDevice), +} + +impl fmt::Display for DeviceType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } + +#[async_trait] +pub trait Device: Send + Sync { + // attach is to plug device into VM + async fn attach(&mut self, h: &dyn hypervisor) -> Result<()>; + // detach is to unplug device from VM + async fn detach(&mut self, h: &dyn hypervisor) -> Result>; + // get_device_info returns device config + async fn get_device_info(&self) -> DeviceConfig; + // increase_attach_count is used to increase the attach count for a device + // return values: + // * true: no need to do real attach when current attach count is zero, skip following actions. + // * err error: error while do increase attach count + async fn increase_attach_count(&mut self) -> Result; + // decrease_attach_count is used to decrease the attach count for a device + // return values: + // * false: no need to do real dettach when current attach count is not zero, skip following actions. + // * err error: error while do decrease attach count + async fn decrease_attach_count(&mut self) -> Result; +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs b/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs deleted file mode 100644 index 4bf73eab7..000000000 --- a/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2019-2022 Alibaba Cloud -// Copyright (c) 2019-2022 Ant Group -// -// SPDX-License-Identifier: Apache-2.0 -// - -/// ShareFsDeviceConfig: share fs device config -#[derive(Debug)] -pub struct ShareFsDeviceConfig { - /// fs_type: virtiofs or inline-virtiofs - pub fs_type: String, - - /// socket_path: socket path for virtiofs - pub sock_path: String, - - /// mount_tag: a label used as a hint to the guest. - pub mount_tag: String, - - /// host_path: the host filesystem path for this volume. - pub host_path: String, - - /// queue_size: queue size - pub queue_size: u64, - - /// queue_num: queue number - pub queue_num: u64, -} diff --git a/src/runtime-rs/crates/hypervisor/src/device/util.rs b/src/runtime-rs/crates/hypervisor/src/device/util.rs new file mode 100644 index 000000000..3aa5f7b0a --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/util.rs @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2023 Alibaba Cloud +// Copyright (c) 2019-2023 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Result}; +use ini::Ini; + +const SYS_DEV_PREFIX: &str = "/sys/dev"; + +// get_host_path is used to fetch the host path for the device. +// The path passed in the spec refers to the path that should appear inside the container. +// We need to find the actual device path on the host based on the major-minor numbers of the device. +pub fn get_host_path(dev_type: String, major: i64, minor: i64) -> Result { + let path_comp = match dev_type.as_str() { + "c" | "u" => "char", + "b" => "block", + // for device type p will return an empty string + _ => return Ok(String::new()), + }; + let format = format!("{}:{}", major, minor); + let sys_dev_path = std::path::Path::new(SYS_DEV_PREFIX) + .join(path_comp) + .join(format) + .join("uevent"); + std::fs::metadata(&sys_dev_path)?; + let conf = Ini::load_from_file(&sys_dev_path)?; + let dev_name = conf + .section::(None) + .ok_or_else(|| anyhow!("has no section"))? + .get("DEVNAME") + .ok_or_else(|| anyhow!("has no DEVNAME"))?; + Ok(format!("/dev/{}", dev_name)) +} + +// get_virt_drive_name returns the disk name format for virtio-blk +// Reference: https://github.com/torvalds/linux/blob/master/drivers/block/virtio_blk.c @c0aa3e0916d7e531e69b02e426f7162dfb1c6c0 +pub(crate) fn get_virt_drive_name(mut index: i32) -> Result { + if index < 0 { + return Err(anyhow!("Index cannot be negative")); + } + + // Prefix used for virtio-block devices + const PREFIX: &str = "vd"; + + // Refer to DISK_NAME_LEN: https://github.com/torvalds/linux/blob/08c521a2011ff492490aa9ed6cc574be4235ce2b/include/linux/genhd.h#L61 + let disk_name_len = 32usize; + let base = 26i32; + + let suff_len = disk_name_len - PREFIX.len(); + let mut disk_letters = vec![0u8; suff_len]; + + let mut i = 0usize; + while i < suff_len && index >= 0 { + let letter: u8 = b'a' + (index % base) as u8; + disk_letters[i] = letter; + index = (index / base) - 1; + i += 1; + } + if index >= 0 { + return Err(anyhow!("Index not supported")); + } + disk_letters.truncate(i); + disk_letters.reverse(); + Ok(String::from(PREFIX) + std::str::from_utf8(&disk_letters)?) +} + +#[cfg(test)] +mod tests { + use crate::device::util::get_virt_drive_name; + + #[actix_rt::test] + async fn test_get_virt_drive_name() { + for &(input, output) in [ + (0i32, "vda"), + (25, "vdz"), + (27, "vdab"), + (704, "vdaac"), + (18277, "vdzzz"), + ] + .iter() + { + let out = get_virt_drive_name(input).unwrap(); + assert_eq!(&out, output); + } + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index d2d2cd86d..45b77f09e 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -6,8 +6,9 @@ use super::vmm_instance::VmmInstance; use crate::{ - device::Device, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState, + device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState, DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, VM_ROOTFS_DRIVER_BLK, + VM_ROOTFS_DRIVER_MMIO, }; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -56,7 +57,7 @@ pub struct DragonballInner { pub(crate) run_dir: String, /// pending device - pub(crate) pending_devices: Vec, + pub(crate) pending_devices: Vec, /// cached block device pub(crate) cached_block_devices: HashSet, @@ -265,7 +266,7 @@ impl DragonballInner { .get_resource(path, DRAGONBALL_ROOT_FS) .context("get resource")?; - if driver == VM_ROOTFS_DRIVER_BLK { + if driver == VM_ROOTFS_DRIVER_BLK || driver == VM_ROOTFS_DRIVER_MMIO { let blk_cfg = BlockDeviceConfigInfo { path_on_host: PathBuf::from(jail_drive), drive_id: DRAGONBALL_ROOT_FS.to_string(), diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 48d9a3508..94861536c 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -15,7 +15,7 @@ use dragonball::api::v1::{ use super::DragonballInner; use crate::{ - device::Device, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig, + device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountType, ShareFsOperation, VmmState, }; @@ -31,7 +31,7 @@ pub(crate) fn drive_index_to_id(index: u64) -> String { } impl DragonballInner { - pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> { + pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> { if self.state == VmmState::NotReady { info!(sl!(), "VMM not ready, queueing device {}", device); @@ -44,41 +44,43 @@ impl DragonballInner { info!(sl!(), "dragonball add device {:?}", &device); match device { - Device::Network(config) => self.add_net_device(&config).context("add net device"), - Device::Vfio(_config) => { + DeviceType::Network(network) => self + .add_net_device(&network.config, network.id) + .context("add net device"), + DeviceType::Vfio(_) => { todo!() } - Device::Block(config) => self + DeviceType::Block(block) => self .add_block_device( - config.path_on_host.as_str(), - config.id.as_str(), - config.is_readonly, - config.no_drop, + block.config.path_on_host.as_str(), + block.device_id.as_str(), + block.config.is_readonly, + block.config.no_drop, ) .context("add block device"), - Device::HybridVsock(config) => self.add_hvsock(&config).context("add vsock"), - Device::ShareFsDevice(config) => self - .add_share_fs_device(&config) + DeviceType::HybridVsock(hvsock) => self.add_hvsock(&hvsock.config).context("add vsock"), + DeviceType::ShareFs(sharefs) => self + .add_share_fs_device(&sharefs.config) .context("add share fs device"), - Device::ShareFsMount(config) => self - .add_share_fs_mount(&config) + DeviceType::ShareFsMount(sharefs_mount) => self + .add_share_fs_mount(&sharefs_mount.config) .context("add share fs mount"), - Device::Vsock(_) => { + DeviceType::Vsock(_) => { todo!() } } } - pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> { + pub(crate) async fn remove_device(&mut self, device: DeviceType) -> Result<()> { info!(sl!(), "remove device {} ", device); match device { - Device::Block(config) => { - let drive_id = drive_index_to_id(config.index); + DeviceType::Block(block) => { + let drive_id = drive_index_to_id(block.config.index); self.remove_block_drive(drive_id.as_str()) .context("remove block drive") } - Device::Vfio(_config) => { + DeviceType::Vfio(_config) => { todo!() } _ => Err(anyhow!("unsupported device {:?}", device)), @@ -121,9 +123,9 @@ impl DragonballInner { Ok(()) } - fn add_net_device(&mut self, config: &NetworkConfig) -> Result<()> { + fn add_net_device(&mut self, config: &NetworkConfig, device_id: String) -> Result<()> { let iface_cfg = VirtioNetDeviceConfigInfo { - iface_id: config.id.clone(), + iface_id: device_id, host_dev_name: config.host_dev_name.clone(), guest_mac: match &config.guest_mac { Some(mac) => MacAddr::from_bytes(&mac.0).ok(), diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs index adc016807..18aa7139d 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs @@ -13,7 +13,9 @@ use anyhow::{Context, Ok, Result}; use kata_types::capabilities::Capabilities; use super::inner::DragonballInner; -use crate::{utils, VcpuThreadIds, VmmState}; +use crate::{ + device::DeviceType, utils, HybridVsockConfig, HybridVsockDevice, VcpuThreadIds, VmmState, +}; use shim_interface::KATA_PATH; const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock"; @@ -32,10 +34,12 @@ impl DragonballInner { // prepare vsock let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/"); - let d = crate::device::Device::HybridVsock(crate::device::HybridVsockConfig { + let d = DeviceType::HybridVsock(HybridVsockDevice { id: format!("vsock-{}", &self.id), - guest_cid: 3, - uds_path, + config: HybridVsockConfig { + guest_cid: 3, + uds_path, + }, }); self.add_device(d).await.context("add device")?; diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index 951af2bc7..c6df95cc9 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -20,7 +20,7 @@ use kata_types::capabilities::Capabilities; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use tokio::sync::RwLock; -use crate::{device::Device, Hypervisor, VcpuThreadIds}; +use crate::{DeviceType, Hypervisor, VcpuThreadIds}; pub struct Dragonball { inner: Arc>, @@ -77,12 +77,12 @@ impl Hypervisor for Dragonball { inner.save_vm().await } - async fn add_device(&self, device: Device) -> Result<()> { + async fn add_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.add_device(device).await } - async fn remove_device(&self, device: Device) -> Result<()> { + async fn remove_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.remove_device(device).await } diff --git a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs index 7ad17cb8f..554d61660 100644 --- a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs +++ b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs @@ -7,7 +7,7 @@ use anyhow::{anyhow, Result}; use crate::{ - VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_PMEM, VM_ROOTFS_FILESYSTEM_EROFS, + VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_MMIO, VM_ROOTFS_DRIVER_PMEM, VM_ROOTFS_FILESYSTEM_EROFS, VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS, VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM, }; use kata_types::config::LOG_VPORT_OPTION; @@ -91,7 +91,7 @@ impl KernelParams { } } } - VM_ROOTFS_DRIVER_BLK => { + VM_ROOTFS_DRIVER_BLK | VM_ROOTFS_DRIVER_MMIO => { params.push(Param::new("root", VM_ROOTFS_ROOT_BLK)); match rootfs_type { VM_ROOTFS_FILESYSTEM_EXT4 | VM_ROOTFS_FILESYSTEM_XFS => { diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 364cce0f2..2001433e5 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -11,7 +11,8 @@ logging::logger_with_subsystem!(sl, "hypervisor"); pub mod device; pub mod hypervisor_persist; -pub use device::*; +pub use device::driver::*; +use device::DeviceType; pub mod dragonball; mod kernel_param; pub mod qemu; @@ -31,8 +32,9 @@ use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH; // Config which driver to use as vm root dev -const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk"; +const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk-pci"; const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem"; +const VM_ROOTFS_DRIVER_MMIO: &str = "virtio-blk-mmio"; //Configure the root corresponding to the driver const VM_ROOTFS_ROOT_BLK: &str = "/dev/vda1"; @@ -78,8 +80,8 @@ pub trait Hypervisor: Send + Sync { async fn resume_vm(&self) -> Result<()>; // device manager - async fn add_device(&self, device: device::Device) -> Result<()>; - async fn remove_device(&self, device: device::Device) -> Result<()>; + async fn add_device(&self, device: DeviceType) -> Result<()>; + async fn remove_device(&self, device: DeviceType) -> Result<()>; // utils async fn get_agent_socket(&self) -> Result; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 0c0efbde6..456bf6373 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -133,16 +133,16 @@ impl QemuInner { } } -use crate::device::Device; +use crate::device::DeviceType; // device manager part of Hypervisor impl QemuInner { - pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> { + pub(crate) async fn add_device(&mut self, device: DeviceType) -> Result<()> { info!(sl!(), "QemuInner::add_device() {}", device); todo!() } - pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> { + pub(crate) async fn remove_device(&mut self, device: DeviceType) -> Result<()> { info!(sl!(), "QemuInner::remove_device() {} ", device); todo!() } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index c73fb23d4..77217f153 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -5,7 +5,7 @@ mod inner; -use crate::device::Device; +use crate::device::DeviceType; use crate::hypervisor_persist::HypervisorState; use crate::Hypervisor; use crate::{HypervisorConfig, VcpuThreadIds}; @@ -73,12 +73,12 @@ impl Hypervisor for Qemu { inner.save_vm().await } - async fn add_device(&self, device: Device) -> Result<()> { + async fn add_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.add_device(device).await } - async fn remove_device(&self, device: Device) -> Result<()> { + async fn remove_device(&self, device: DeviceType) -> Result<()> { let mut inner = self.inner.write().await; inner.remove_device(device).await } diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index baafd28b7..9847ce61b 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -30,7 +30,7 @@ serde = { version = "1.0.138", features = ["derive"] } serde_json = "1.0.82" slog = "2.5.2" slog-scope = "4.4.0" -tokio = { version = "1.8.0", features = ["process"] } +tokio = { version = "1.28.1", features = ["process"] } uuid = { version = "0.4", features = ["v4"] } agent = { path = "../agent" } diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index 9514b6013..a022f722f 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -7,13 +7,15 @@ use crate::network::NetworkConfig; use crate::resource_persist::ResourceState; use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig}; +use agent::types::Device; use agent::{Agent, Storage}; use anyhow::Result; use async_trait::async_trait; +use hypervisor::device::device_manager::DeviceManager; use hypervisor::Hypervisor; use kata_types::config::TomlConfig; use kata_types::mount::Mount; -use oci::LinuxResources; +use oci::{Linux, LinuxResources}; use persist::sandbox_persist::Persist; use std::sync::Arc; use tokio::sync::RwLock; @@ -51,6 +53,11 @@ impl ResourceManager { inner.config() } + pub async fn get_device_manager(&self) -> Arc> { + let inner = self.inner.read().await; + inner.get_device_manager() + } + pub async fn prepare_before_start_vm(&self, device_configs: Vec) -> Result<()> { let mut inner = self.inner.write().await; inner.prepare_before_start_vm(device_configs).await @@ -93,6 +100,11 @@ impl ResourceManager { inner.handler_volumes(cid, spec).await } + pub async fn handler_devices(&self, cid: &str, linux: &Linux) -> Result> { + let inner = self.inner.read().await; + inner.handler_devices(cid, linux).await + } + pub async fn dump(&self) { let inner = self.inner.read().await; inner.dump().await diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 6c6e4067a..7e50485ae 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -4,18 +4,21 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::{sync::Arc, thread}; +use std::{sync::Arc, thread, vec}; use crate::{network::NetworkConfig, resource_persist::ResourceState}; -use agent::{Agent, Storage}; +use agent::{types::Device, Agent, Storage}; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; +use hypervisor::{ + device::{device_manager::DeviceManager, DeviceConfig}, + BlockConfig, Hypervisor, +}; use kata_types::config::TomlConfig; use kata_types::mount::Mount; -use oci::LinuxResources; +use oci::{Linux, LinuxResources}; use persist::sandbox_persist::Persist; -use tokio::runtime; +use tokio::{runtime, sync::RwLock}; use crate::{ cgroups::{CgroupArgs, CgroupsResource}, @@ -32,6 +35,7 @@ pub(crate) struct ResourceManagerInner { toml_config: Arc, agent: Arc, hypervisor: Arc, + device_manager: Arc>, network: Option>, share_fs: Option>, @@ -48,11 +52,17 @@ impl ResourceManagerInner { toml_config: Arc, ) -> Result { let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; + + // create device manager + let dev_manager = + DeviceManager::new(hypervisor.clone()).context("failed to create device manager")?; + Ok(Self { sid: sid.to_string(), toml_config, agent, hypervisor, + device_manager: Arc::new(RwLock::new(dev_manager)), network: None, share_fs: None, rootfs_resource: RootFsResource::new(), @@ -65,6 +75,10 @@ impl ResourceManagerInner { self.toml_config.clone() } + pub fn get_device_manager(&self) -> Arc> { + self.device_manager.clone() + } + pub async fn prepare_before_start_vm( &mut self, device_configs: Vec, @@ -212,6 +226,7 @@ impl ResourceManagerInner { self.rootfs_resource .handler_rootfs( &self.share_fs, + self.device_manager.as_ref(), self.hypervisor.as_ref(), &self.sid, cid, @@ -228,10 +243,71 @@ impl ResourceManagerInner { spec: &oci::Spec, ) -> Result>> { self.volume_resource - .handler_volumes(&self.share_fs, cid, spec) + .handler_volumes( + &self.share_fs, + cid, + spec, + self.device_manager.as_ref(), + &self.sid, + ) .await } + pub async fn handler_devices(&self, _cid: &str, linux: &Linux) -> Result> { + let mut devices = vec![]; + for d in linux.devices.iter() { + match d.r#type.as_str() { + "b" => { + let device_info = DeviceConfig::BlockCfg(BlockConfig { + major: d.major, + minor: d.minor, + ..Default::default() + }); + let device_id = self + .device_manager + .write() + .await + .new_device(&device_info) + .await + .context("failed to create deviec")?; + + self.device_manager + .write() + .await + .try_add_device(&device_id) + .await + .context("failed to add deivce")?; + + // get complete device information + let dev_info = self + .device_manager + .read() + .await + .get_device_info(&device_id) + .await + .context("failed to get device info")?; + + // create agent device + if let DeviceConfig::BlockCfg(config) = dev_info { + let agent_device = Device { + id: device_id.clone(), + container_path: d.path.clone(), + field_type: config.driver_option, + vm_path: config.virt_path, + ..Default::default() + }; + devices.push(agent_device); + } + } + _ => { + // TODO enable other devices type + continue; + } + } + } + Ok(devices) + } + pub async fn update_cgroups( &self, cid: &str, @@ -298,7 +374,8 @@ impl Persist for ResourceManagerInner { Ok(Self { sid: resource_args.sid, agent: resource_args.agent, - hypervisor: resource_args.hypervisor, + hypervisor: resource_args.hypervisor.clone(), + device_manager: Arc::new(RwLock::new(DeviceManager::new(resource_args.hypervisor)?)), network: None, share_fs: None, rootfs_resource: RootFsResource::new(), diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs index eebabe59c..a8f08088b 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs @@ -9,11 +9,13 @@ use std::io::{self, Error}; use super::endpoint_persist::{EndpointState, IpVlanEndpointState}; use anyhow::{Context, Result}; use async_trait::async_trait; +use hypervisor::device::DeviceType; +use hypervisor::NetworkDevice; use super::Endpoint; use crate::network::network_model::TC_FILTER_NET_MODEL_STR; use crate::network::{utils, NetworkPair}; -use hypervisor::{device::NetworkConfig, Device, Hypervisor}; +use hypervisor::{device::driver::NetworkConfig, Hypervisor}; // IPVlanEndpoint is the endpoint bridged to VM #[derive(Debug)] @@ -44,7 +46,6 @@ impl IPVlanEndpoint { ) })?; Ok(NetworkConfig { - id: self.net_pair.virt_iface.name.clone(), host_dev_name: iface.name.clone(), guest_mac: Some(guest_mac), }) @@ -67,9 +68,12 @@ impl Endpoint for IPVlanEndpoint { .await .context("error adding network model")?; let config = self.get_network_config().context("get network config")?; - h.add_device(Device::Network(config)) - .await - .context("error adding device by hypervisor")?; + h.add_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error adding device by hypervisor")?; Ok(()) } @@ -82,9 +86,12 @@ impl Endpoint for IPVlanEndpoint { let config = self .get_network_config() .context("error getting network config")?; - h.remove_device(Device::Network(config)) - .await - .context("error removing device by hypervisor")?; + h.remove_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error removing device by hypervisor")?; Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs index 364a5792b..13716e877 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs @@ -11,7 +11,9 @@ use super::Endpoint; use crate::network::{utils, NetworkPair}; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::{device::NetworkConfig, Device, Hypervisor}; +use hypervisor::device::DeviceType; +use hypervisor::NetworkDevice; +use hypervisor::{device::driver::NetworkConfig, Hypervisor}; #[derive(Debug)] pub struct MacVlanEndpoint { @@ -41,7 +43,6 @@ impl MacVlanEndpoint { ) })?; Ok(NetworkConfig { - id: self.net_pair.virt_iface.name.clone(), host_dev_name: iface.name.clone(), guest_mac: Some(guest_mac), }) @@ -64,9 +65,13 @@ impl Endpoint for MacVlanEndpoint { .await .context("add network model")?; let config = self.get_network_config().context("get network config")?; - h.add_device(Device::Network(config)) - .await - .context("Error add device")?; + h.add_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error adding device by hypervisor")?; + Ok(()) } @@ -76,9 +81,13 @@ impl Endpoint for MacVlanEndpoint { .await .context("del network model")?; let config = self.get_network_config().context("get network config")?; - h.remove_device(Device::Network(config)) - .await - .context("remove device")?; + h.remove_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error removing device by hypervisor")?; + Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs index 1be181c0d..4db8b865f 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs @@ -8,7 +8,9 @@ use std::path::Path; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use hypervisor::{device, Hypervisor}; +use hypervisor::device::DeviceType; +use hypervisor::{device::driver, Hypervisor}; +use hypervisor::{VfioConfig, VfioDevice}; use super::endpoint_persist::{EndpointState, PhysicalEndpointState}; use super::Endpoint; @@ -94,7 +96,7 @@ impl Endpoint for PhysicalEndpoint { async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()> { // bind physical interface from host driver and bind to vfio - device::bind_device_to_vfio( + driver::bind_device_to_vfio( &self.bdf, &self.driver, &self.vendor_device_id.vendor_device_id(), @@ -108,12 +110,14 @@ impl Endpoint for PhysicalEndpoint { }; // add vfio device - let d = device::Device::Vfio(device::VfioConfig { + let d = DeviceType::Vfio(VfioDevice { id: format!("physical_nic_{}", self.name().await), - sysfs_path: "".to_string(), - bus_slot_func: self.bdf.clone(), - mode: device::VfioBusMode::new(mode) - .with_context(|| format!("new vfio bus mode {:?}", mode))?, + config: VfioConfig { + sysfs_path: "".to_string(), + bus_slot_func: self.bdf.clone(), + mode: driver::VfioBusMode::new(mode) + .with_context(|| format!("new vfio bus mode {:?}", mode))?, + }, }); hypervisor.add_device(d).await.context("add device")?; Ok(()) @@ -128,7 +132,7 @@ impl Endpoint for PhysicalEndpoint { // we do not need to enter the network namespace to bind back the // physical interface to host driver. - device::bind_device_to_host( + driver::bind_device_to_host( &self.bdf, &self.driver, &self.vendor_device_id.vendor_device_id(), diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs index 38ce2e335..4415e6d29 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs @@ -11,7 +11,9 @@ use super::Endpoint; use crate::network::{utils, NetworkPair}; use anyhow::{Context, Result}; use async_trait::async_trait; -use hypervisor::{device::NetworkConfig, Device, Hypervisor}; +use hypervisor::device::DeviceType; +use hypervisor::NetworkDevice; +use hypervisor::{device::driver::NetworkConfig, Hypervisor}; #[derive(Debug)] pub struct VethEndpoint { @@ -41,7 +43,6 @@ impl VethEndpoint { ) })?; Ok(NetworkConfig { - id: self.net_pair.virt_iface.name.clone(), host_dev_name: iface.name.clone(), guest_mac: Some(guest_mac), }) @@ -64,9 +65,12 @@ impl Endpoint for VethEndpoint { .await .context("add network model")?; let config = self.get_network_config().context("get network config")?; - h.add_device(Device::Network(config)) - .await - .context("Error add device")?; + h.add_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error adding device by hypervisor")?; Ok(()) } @@ -76,9 +80,12 @@ impl Endpoint for VethEndpoint { .await .context("del network model")?; let config = self.get_network_config().context("get network config")?; - h.remove_device(Device::Network(config)) - .await - .context("remove device")?; + h.remove_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error removing device by hypervisor")?; Ok(()) } async fn save(&self) -> Option { diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs index 8a90f4a75..6aca674be 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs @@ -8,12 +8,14 @@ use std::io::{self, Error}; use anyhow::{Context, Result}; use async_trait::async_trait; +use hypervisor::device::DeviceType; +use hypervisor::NetworkDevice; use super::endpoint_persist::{EndpointState, VlanEndpointState}; use super::Endpoint; use crate::network::network_model::TC_FILTER_NET_MODEL_STR; use crate::network::{utils, NetworkPair}; -use hypervisor::{device::NetworkConfig, Device, Hypervisor}; +use hypervisor::{device::driver::NetworkConfig, Hypervisor}; #[derive(Debug)] pub struct VlanEndpoint { pub(crate) net_pair: NetworkPair, @@ -41,7 +43,6 @@ impl VlanEndpoint { ) })?; Ok(NetworkConfig { - id: self.net_pair.virt_iface.name.clone(), host_dev_name: iface.name.clone(), guest_mac: Some(guest_mac), }) @@ -64,9 +65,12 @@ impl Endpoint for VlanEndpoint { .await .context("error adding network model")?; let config = self.get_network_config().context("get network config")?; - h.add_device(Device::Network(config)) - .await - .context("error adding device by hypervisor")?; + h.add_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error adding device by hypervisor")?; Ok(()) } @@ -79,9 +83,12 @@ impl Endpoint for VlanEndpoint { let config = self .get_network_config() .context("error getting network config")?; - h.remove_device(Device::Network(config)) - .await - .context("error removing device by hypervisor")?; + h.remove_device(DeviceType::Network(NetworkDevice { + id: self.net_pair.virt_iface.name.clone(), + config, + })) + .await + .context("error removing device by hypervisor")?; Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/network/utils/address.rs b/src/runtime-rs/crates/resource/src/network/utils/address.rs index ef3b68278..3046d3685 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/address.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/address.rs @@ -80,9 +80,7 @@ pub(crate) fn parse_ip(ip: &[u8], family: u8) -> Result { octets.copy_from_slice(&ip[..16]); Ok(IpAddr::V6(Ipv6Addr::from(octets))) } - _ => { - return Err(anyhow!("unknown IP network family {}", family)); - } + _ => Err(anyhow!("unknown IP network family {}", family)), } } diff --git a/src/runtime-rs/crates/resource/src/network/utils/mod.rs b/src/runtime-rs/crates/resource/src/network/utils/mod.rs index 74635a5d9..341038cb9 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/mod.rs @@ -25,7 +25,7 @@ pub(crate) fn parse_mac(s: &str) -> Option { pub(crate) fn get_mac_addr(b: &[u8]) -> Result { if b.len() != 6 { - return Err(anyhow!("invalid mac address {:?}", b)); + Err(anyhow!("invalid mac address {:?}", b)) } else { Ok(format!( "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", diff --git a/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs new file mode 100644 index 000000000..0e161fcb3 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/rootfs/block_rootfs.rs @@ -0,0 +1,134 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use super::{Rootfs, ROOTFS}; +use crate::share_fs::{do_get_guest_path, do_get_host_path}; +use agent::Storage; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use hypervisor::{ + device::{device_manager::DeviceManager, DeviceConfig}, + BlockConfig, +}; +use kata_types::mount::Mount; +use nix::sys::stat::{self, SFlag}; +use std::fs; +use tokio::sync::RwLock; + +pub(crate) struct BlockRootfs { + guest_path: String, + device_id: String, + mount: oci::Mount, + storage: Option, +} + +impl BlockRootfs { + pub async fn new( + d: &RwLock, + sid: &str, + cid: &str, + dev_id: u64, + rootfs: &Mount, + ) -> Result { + let container_path = do_get_guest_path(ROOTFS, cid, false, false); + let host_path = do_get_host_path(ROOTFS, sid, cid, false, false); + // Create rootfs dir on host to make sure mount point in guest exists, as readonly dir is + // shared to guest via virtiofs, and guest is unable to create rootfs dir. + fs::create_dir_all(&host_path) + .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?; + + let block_device_config = &mut BlockConfig { + major: stat::major(dev_id) as i64, + minor: stat::minor(dev_id) as i64, + ..Default::default() + }; + + let device_id = d + .write() + .await + .new_device(&DeviceConfig::BlockCfg(block_device_config.clone())) + .await + .context("failed to create deviec")?; + + d.write() + .await + .try_add_device(device_id.as_str()) + .await + .context("failed to add deivce")?; + + let mut storage = Storage { + fs_type: rootfs.fs_type.clone(), + mount_point: container_path.clone(), + options: rootfs.options.clone(), + ..Default::default() + }; + + // get complete device information + let dev_info = d + .read() + .await + .get_device_info(device_id.as_str()) + .await + .context("failed to get device info")?; + + if let DeviceConfig::BlockCfg(config) = dev_info { + storage.driver = config.driver_option; + storage.source = config.virt_path; + } + + Ok(Self { + guest_path: container_path.clone(), + device_id, + mount: oci::Mount { + ..Default::default() + }, + storage: Some(storage), + }) + } +} + +#[async_trait] +impl Rootfs for BlockRootfs { + async fn get_guest_rootfs_path(&self) -> Result { + Ok(self.guest_path.clone()) + } + + async fn get_rootfs_mount(&self) -> Result> { + Ok(vec![self.mount.clone()]) + } + + async fn get_storage(&self) -> Option { + self.storage.clone() + } + + async fn get_device_id(&self) -> Result> { + Ok(Some(self.device_id.clone())) + } + + async fn cleanup(&self, device_manager: &RwLock) -> Result<()> { + device_manager + .write() + .await + .try_remove_device(&self.device_id) + .await + } +} + +pub(crate) fn is_block_rootfs(file: &str) -> Option { + if file.is_empty() { + return None; + } + match stat::stat(file) { + Ok(fstat) => { + if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK { + let dev_id = fstat.st_rdev; + return Some(dev_id); + } + } + Err(_) => return None, + }; + None +} diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs index b18085ed8..a924b021b 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs @@ -6,18 +6,18 @@ mod nydus_rootfs; mod share_fs_rootfs; - use agent::Storage; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; use kata_types::mount::Mount; +mod block_rootfs; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; use std::{sync::Arc, vec::Vec}; use tokio::sync::RwLock; use crate::share_fs::ShareFs; -use self::nydus_rootfs::NYDUS_ROOTFS_TYPE; +use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE}; const ROOTFS: &str = "rootfs"; const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower"; @@ -27,7 +27,8 @@ pub trait Rootfs: Send + Sync { async fn get_guest_rootfs_path(&self) -> Result; async fn get_rootfs_mount(&self) -> Result>; async fn get_storage(&self) -> Option; - async fn cleanup(&self) -> Result<()>; + async fn cleanup(&self, device_manager: &RwLock) -> Result<()>; + async fn get_device_id(&self) -> Result>; } #[derive(Default)] @@ -56,7 +57,8 @@ impl RootFsResource { pub async fn handler_rootfs( &self, share_fs: &Option>, - hypervisor: &dyn Hypervisor, + device_manager: &RwLock, + h: &dyn Hypervisor, sid: &str, cid: &str, root: &oci::Root, @@ -67,7 +69,7 @@ impl RootFsResource { // if rootfs_mounts is empty mounts_vec if mounts_vec.is_empty() => { if let Some(share_fs) = share_fs { - // share fs rootfs + // handle share fs rootfs Ok(Arc::new( share_fs_rootfs::ShareFsRootfs::new( share_fs, @@ -79,22 +81,33 @@ impl RootFsResource { .context("new share fs rootfs")?, )) } else { - return Err(anyhow!("share fs is unavailable")); + Err(anyhow!("share fs is unavailable")) } } mounts_vec if is_single_layer_rootfs(mounts_vec) => { // Safe as single_layer_rootfs must have one layer let layer = &mounts_vec[0]; - let rootfs: Arc = if let Some(share_fs) = share_fs { - // nydus rootfs - if layer.fs_type == NYDUS_ROOTFS_TYPE { + let mut inner = self.inner.write().await; + let rootfs = if let Some(dev_id) = is_block_rootfs(&layer.source) { + // handle block rootfs + info!(sl!(), "block device: {}", dev_id); + let block_rootfs: Arc = Arc::new( + block_rootfs::BlockRootfs::new(device_manager, sid, cid, dev_id, layer) + .await + .context("new block rootfs")?, + ); + Ok(block_rootfs) + } else if let Some(share_fs) = share_fs { + // handle nydus rootfs + let share_rootfs: Arc = if layer.fs_type == NYDUS_ROOTFS_TYPE { Arc::new( - nydus_rootfs::NydusRootfs::new(share_fs, hypervisor, sid, cid, layer) + nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer) .await .context("new nydus rootfs")?, ) - } else { - // share fs rootfs + } + // handle sharefs rootfs + else { Arc::new( share_fs_rootfs::ShareFsRootfs::new( share_fs, @@ -105,21 +118,18 @@ impl RootFsResource { .await .context("new share fs rootfs")?, ) - } + }; + Ok(share_rootfs) } else { - return Err(anyhow!("unsupported rootfs {:?}", &layer)); - }; - - let mut inner = self.inner.write().await; - inner.rootfs.push(Arc::clone(&rootfs)); + Err(anyhow!("unsupported rootfs {:?}", &layer)) + }?; + inner.rootfs.push(rootfs.clone()); Ok(rootfs) } - _ => { - return Err(anyhow!( - "unsupported rootfs mounts count {}", - rootfs_mounts.len() - )) - } + _ => Err(anyhow!( + "unsupported rootfs mounts count {}", + rootfs_mounts.len() + )), } } diff --git a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs index 008443b87..96f29e8d6 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs @@ -16,8 +16,9 @@ use crate::{ use agent::Storage; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; -use hypervisor::Hypervisor; +use hypervisor::{device::device_manager::DeviceManager, Hypervisor}; use kata_types::mount::{Mount, NydusExtraOptions}; +use tokio::sync::RwLock; // Used for nydus rootfs pub(crate) const NYDUS_ROOTFS_TYPE: &str = "fuse.nydus-overlayfs"; @@ -150,7 +151,11 @@ impl Rootfs for NydusRootfs { Some(self.rootfs.clone()) } - async fn cleanup(&self) -> Result<()> { + async fn get_device_id(&self) -> Result> { + Ok(None) + } + + async fn cleanup(&self, _device_manager: &RwLock) -> Result<()> { // TODO: Clean up NydusRootfs after the container is killed warn!(sl!(), "Cleaning up NydusRootfs is still unimplemented."); Ok(()) diff --git a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs index b5d4136c1..385c058d3 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs @@ -4,12 +4,15 @@ // SPDX-License-Identifier: Apache-2.0 // +use std::sync::Arc; + use agent::Storage; use anyhow::{Context, Result}; use async_trait::async_trait; +use hypervisor::device::device_manager::DeviceManager; use kata_sys_util::mount::{umount_timeout, Mounter}; use kata_types::mount::Mount; -use std::sync::Arc; +use tokio::sync::RwLock; use super::{Rootfs, ROOTFS}; use crate::share_fs::{ShareFs, ShareFsRootfsConfig}; @@ -74,7 +77,11 @@ impl Rootfs for ShareFsRootfs { None } - async fn cleanup(&self) -> Result<()> { + async fn get_device_id(&self) -> Result> { + Ok(None) + } + + async fn cleanup(&self, _device_manager: &RwLock) -> Result<()> { // Umount the mount point shared to guest let share_fs_mount = self.share_fs.get_share_fs_mount(); share_fs_mount diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs index 350c7ea71..12bb64420 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/mod.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs @@ -12,7 +12,9 @@ mod share_virtio_fs_standalone; use share_virtio_fs_standalone::ShareVirtioFsStandalone; mod utils; use tokio::sync::Mutex; -pub use utils::{do_get_guest_path, do_get_guest_share_path, get_host_rw_shared_path}; +pub use utils::{ + do_get_guest_path, do_get_guest_share_path, do_get_host_path, get_host_rw_shared_path, +}; mod virtio_fs_share_mount; use virtio_fs_share_mount::VirtiofsShareMount; pub use virtio_fs_share_mount::EPHEMERAL_PATH; diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs index 4aed2cc19..81ab8b7fd 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -8,7 +8,13 @@ use std::path::Path; use anyhow::{Context, Result}; use hypervisor::{ - device::{Device as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation}, + device::{ + driver::{ + ShareFsDevice, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType, + ShareFsOperation, + }, + DeviceType, + }, Hypervisor, ShareFsDeviceConfig, }; use kata_sys_util::mount; @@ -42,15 +48,19 @@ pub(crate) async fn prepare_virtiofs( mount::bind_mount_unchecked(&host_rw_dest, &host_ro_dest, true) .context("bind mount shared_fs directory")?; - let share_fs_device = HypervisorDevice::ShareFsDevice(ShareFsDeviceConfig { - sock_path: generate_sock_path(root), - mount_tag: String::from(MOUNT_GUEST_TAG), - host_path: String::from(host_ro_dest.to_str().unwrap()), - fs_type: fs_type.to_string(), - queue_size: 0, - queue_num: 0, - }); - h.add_device(share_fs_device).await.context("add device")?; + let share_fs_device = ShareFsDevice { + config: ShareFsDeviceConfig { + sock_path: generate_sock_path(root), + mount_tag: String::from(MOUNT_GUEST_TAG), + host_path: String::from(host_ro_dest.to_str().unwrap()), + fs_type: fs_type.to_string(), + queue_size: 0, + queue_num: 0, + }, + }; + h.add_device(DeviceType::ShareFs(share_fs_device)) + .await + .context("add device")?; Ok(()) } @@ -66,16 +76,18 @@ pub(crate) async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Resul let ro_source = utils::get_host_ro_shared_path(id).join(PASSTHROUGH_FS_DIR); let source = String::from(ro_source.to_str().unwrap()); - let virtio_fs = HypervisorDevice::ShareFsMount(ShareFsMountConfig { - source: source.clone(), - fstype: ShareFsMountType::PASSTHROUGH, - mount_point: mnt, - config: None, - tag: String::from(MOUNT_GUEST_TAG), - op: ShareFsOperation::Mount, - prefetch_list_path: None, - }); - h.add_device(virtio_fs) + let virtio_fs = ShareFsMountDevice { + config: ShareFsMountConfig { + source: source.clone(), + fstype: ShareFsMountType::PASSTHROUGH, + mount_point: mnt, + config: None, + tag: String::from(MOUNT_GUEST_TAG), + op: ShareFsOperation::Mount, + prefetch_list_path: None, + }, + }; + h.add_device(DeviceType::ShareFsMount(virtio_fs)) .await .with_context(|| format!("fail to attach passthrough fs {:?}", source)) } @@ -91,16 +103,18 @@ pub async fn rafs_mount( sl!(), "Attaching rafs meta file {} to virtio-fs device, rafs mount point {}", rafs_meta, rafs_mnt ); - let virtio_fs = HypervisorDevice::ShareFsMount(ShareFsMountConfig { - source: rafs_meta.clone(), - fstype: ShareFsMountType::RAFS, - mount_point: rafs_mnt, - config: Some(config_content), - tag: String::from(MOUNT_GUEST_TAG), - op: ShareFsOperation::Mount, - prefetch_list_path, - }); - h.add_device(virtio_fs) + let virtio_fs = ShareFsMountDevice { + config: ShareFsMountConfig { + source: rafs_meta.clone(), + fstype: ShareFsMountType::RAFS, + mount_point: rafs_mnt, + config: Some(config_content), + tag: String::from(MOUNT_GUEST_TAG), + op: ShareFsOperation::Mount, + prefetch_list_path, + }, + }; + h.add_device(DeviceType::ShareFsMount(virtio_fs)) .await .with_context(|| format!("fail to attach rafs {:?}", rafs_meta))?; Ok(()) diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs index 47f3df533..c93cbec54 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -97,7 +97,7 @@ pub fn do_get_guest_share_path(target: &str, cid: &str, is_rafs: bool) -> String do_get_guest_any_path(target, cid, false, is_rafs, true) } -pub(crate) fn do_get_host_path( +pub fn do_get_host_path( target: &str, sid: &str, cid: &str, diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs index da8ef03f9..14f09fc81 100644 --- a/src/runtime-rs/crates/resource/src/volume/block_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs @@ -6,37 +6,151 @@ use anyhow::Result; use async_trait::async_trait; +use std::{collections::HashMap, fs, path::Path}; -use super::Volume; +use crate::share_fs::{do_get_guest_path, do_get_host_path}; +use super::{share_fs_volume::generate_mount_path, Volume}; +use agent::Storage; +use anyhow::{anyhow, Context}; +use hypervisor::{ + device::{device_manager::DeviceManager, DeviceConfig}, + BlockConfig, +}; +use nix::sys::stat::{self, SFlag}; +use tokio::sync::RwLock; #[derive(Debug)] -pub(crate) struct BlockVolume {} +pub(crate) struct BlockVolume { + storage: Option, + mount: oci::Mount, + device_id: String, +} /// BlockVolume: block device volume impl BlockVolume { - pub(crate) fn new(_m: &oci::Mount) -> Result { - Ok(Self {}) + pub(crate) async fn new( + d: &RwLock, + m: &oci::Mount, + read_only: bool, + cid: &str, + sid: &str, + ) -> Result { + let fstat = stat::stat(m.source.as_str()).context(format!("stat {}", m.source))?; + info!(sl!(), "device stat: {:?}", fstat); + let mut options = HashMap::new(); + if read_only { + options.insert("read_only".to_string(), "true".to_string()); + } + + let block_device_config = &mut BlockConfig { + major: stat::major(fstat.st_rdev) as i64, + minor: stat::minor(fstat.st_rdev) as i64, + ..Default::default() + }; + + let device_id = d + .write() + .await + .new_device(&DeviceConfig::BlockCfg(block_device_config.clone())) + .await + .context("failed to create deviec")?; + + d.write() + .await + .try_add_device(device_id.as_str()) + .await + .context("failed to add deivce")?; + + let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); + let file_name = generate_mount_path(cid, file_name); + let guest_path = do_get_guest_path(&file_name, cid, true, false); + let host_path = do_get_host_path(&file_name, sid, cid, true, read_only); + fs::create_dir_all(&host_path) + .map_err(|e| anyhow!("failed to create rootfs dir {}: {:?}", host_path, e))?; + + // get complete device information + let dev_info = d + .read() + .await + .get_device_info(&device_id) + .await + .context("failed to get device info")?; + + // storage + let mut storage = Storage::default(); + + if let DeviceConfig::BlockCfg(config) = dev_info { + storage.driver = config.driver_option; + storage.source = config.virt_path; + } + + storage.options = if read_only { + vec!["ro".to_string()] + } else { + Vec::new() + }; + + storage.mount_point = guest_path.clone(); + + // If the volume had specified the filesystem type, use it. Otherwise, set it + // to ext4 since but right now we only support it. + if m.r#type != "bind" { + storage.fs_type = m.r#type.clone(); + } else { + storage.fs_type = "ext4".to_string(); + } + + // mount + let mount = oci::Mount { + destination: m.destination.clone(), + r#type: m.r#type.clone(), + source: guest_path.clone(), + options: m.options.clone(), + }; + + Ok(Self { + storage: Some(storage), + mount, + device_id, + }) } } #[async_trait] impl Volume for BlockVolume { - fn get_volume_mount(&self) -> anyhow::Result> { - todo!() + fn get_volume_mount(&self) -> Result> { + Ok(vec![self.mount.clone()]) } fn get_storage(&self) -> Result> { - todo!() + let s = if let Some(s) = self.storage.as_ref() { + vec![s.clone()] + } else { + vec![] + }; + Ok(s) } - async fn cleanup(&self) -> Result<()> { - // TODO: Clean up BlockVolume - warn!(sl!(), "Cleaning up BlockVolume is still unimplemented."); - Ok(()) + async fn cleanup(&self, device_manager: &RwLock) -> Result<()> { + device_manager + .write() + .await + .try_remove_device(&self.device_id) + .await + } + + fn get_device_id(&self) -> Result> { + Ok(Some(self.device_id.clone())) } } -pub(crate) fn is_block_volume(_m: &oci::Mount) -> bool { - // attach block device +pub(crate) fn is_block_volume(m: &oci::Mount) -> bool { + if m.r#type != "bind" { + return false; + } + if let Ok(fstat) = stat::stat(m.source.as_str()).context(format!("stat {}", m.source)) { + info!(sl!(), "device stat: {:?}", fstat); + return SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK; + } false } diff --git a/src/runtime-rs/crates/resource/src/volume/default_volume.rs b/src/runtime-rs/crates/resource/src/volume/default_volume.rs index 8855a8e03..827d2b121 100644 --- a/src/runtime-rs/crates/resource/src/volume/default_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/default_volume.rs @@ -4,6 +4,9 @@ // SPDX-License-Identifier: Apache-2.0 // +use hypervisor::device::device_manager::DeviceManager; +use tokio::sync::RwLock; + use anyhow::Result; use async_trait::async_trait; @@ -33,9 +36,13 @@ impl Volume for DefaultVolume { Ok(vec![]) } - async fn cleanup(&self) -> Result<()> { + async fn cleanup(&self, _device_manager: &RwLock) -> Result<()> { // TODO: Clean up DefaultVolume warn!(sl!(), "Cleaning up DefaultVolume is still unimplemented."); Ok(()) } + + fn get_device_id(&self) -> Result> { + Ok(None) + } } diff --git a/src/runtime-rs/crates/resource/src/volume/hugepage.rs b/src/runtime-rs/crates/resource/src/volume/hugepage.rs index a827b2657..ca8502e7e 100644 --- a/src/runtime-rs/crates/resource/src/volume/hugepage.rs +++ b/src/runtime-rs/crates/resource/src/volume/hugepage.rs @@ -15,9 +15,10 @@ use agent::Storage; use anyhow::{anyhow, Context, Ok, Result}; use async_trait::async_trait; use byte_unit::Byte; -use hypervisor::HUGETLBFS; +use hypervisor::{device::device_manager::DeviceManager, HUGETLBFS}; use kata_sys_util::{fs::get_base_name, mount::PROC_MOUNTS_FILE}; use kata_types::mount::KATA_EPHEMERAL_VOLUME_TYPE; +use tokio::sync::RwLock; use super::{Volume, BIND}; @@ -88,9 +89,13 @@ impl Volume for Hugepage { Ok(s) } - async fn cleanup(&self) -> Result<()> { + async fn cleanup(&self, _device_manager: &RwLock) -> Result<()> { Ok(()) } + + fn get_device_id(&self) -> Result> { + Ok(None) + } } pub(crate) fn get_huge_page_option(m: &oci::Mount) -> Result>> { diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 2868ddee3..ea7d4f5d6 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -12,10 +12,11 @@ mod shm_volume; use async_trait::async_trait; use anyhow::{Context, Result}; +use hypervisor::device::device_manager::DeviceManager; use std::{sync::Arc, vec::Vec}; use tokio::sync::RwLock; -use crate::share_fs::ShareFs; +use crate::{share_fs::ShareFs, volume::block_volume::is_block_volume}; use self::hugepage::{get_huge_page_limits_map, get_huge_page_option}; @@ -25,7 +26,8 @@ const BIND: &str = "bind"; pub trait Volume: Send + Sync { fn get_volume_mount(&self) -> Result>; fn get_storage(&self) -> Result>; - async fn cleanup(&self) -> Result<()>; + fn get_device_id(&self) -> Result>; + async fn cleanup(&self, device_manager: &RwLock) -> Result<()>; } #[derive(Default)] @@ -48,20 +50,25 @@ impl VolumeResource { share_fs: &Option>, cid: &str, spec: &oci::Spec, + d: &RwLock, + sid: &str, ) -> Result>> { let mut volumes: Vec> = vec![]; let oci_mounts = &spec.mounts; + info!(sl!(), " oci mount is : {:?}", oci_mounts.clone()); // handle mounts for m in oci_mounts { + let read_only = m.options.iter().any(|opt| opt == "ro"); let volume: Arc = if shm_volume::is_shim_volume(m) { let shm_size = shm_volume::DEFAULT_SHM_SIZE; Arc::new( shm_volume::ShmVolume::new(m, shm_size) .with_context(|| format!("new shm volume {:?}", m))?, ) - } else if share_fs_volume::is_share_fs_volume(m) { + } else if is_block_volume(m) { + // handle block volume Arc::new( - share_fs_volume::ShareFsVolume::new(share_fs, m, cid) + block_volume::BlockVolume::new(d, m, read_only, cid, sid) .await .with_context(|| format!("new share fs volume {:?}", m))?, ) @@ -76,10 +83,11 @@ impl VolumeResource { hugepage::Hugepage::new(m, hugepage_limits, options) .with_context(|| format!("handle hugepages {:?}", m))?, ) - } else if block_volume::is_block_volume(m) { + } else if share_fs_volume::is_share_fs_volume(m) { Arc::new( - block_volume::BlockVolume::new(m) - .with_context(|| format!("new block volume {:?}", m))?, + share_fs_volume::ShareFsVolume::new(share_fs, m, cid, read_only) + .await + .with_context(|| format!("new share fs volume {:?}", m))?, ) } else if is_skip_volume(m) { info!(sl!(), "skip volume {:?}", m); diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs index f3f70424d..0748e374d 100644 --- a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs @@ -12,6 +12,8 @@ use std::{ use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; +use hypervisor::device::device_manager::DeviceManager; +use tokio::sync::RwLock; use super::Volume; use crate::share_fs::{MountedInfo, ShareFs, ShareFsVolumeConfig}; @@ -36,6 +38,7 @@ impl ShareFsVolume { share_fs: &Option>, m: &oci::Mount, cid: &str, + readonly: bool, ) -> Result { // The file_name is in the format of "sandbox-{uuid}-{file_name}" let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); @@ -69,8 +72,6 @@ impl ShareFsVolume { } } Some(share_fs) => { - let readonly = m.options.iter().any(|opt| opt == "ro"); - let share_fs_mount = share_fs.get_share_fs_mount(); let mounted_info_set = share_fs.mounted_info_set(); let mut mounted_info_set = mounted_info_set.lock().await; @@ -159,7 +160,7 @@ impl Volume for ShareFsVolume { Ok(self.storages.clone()) } - async fn cleanup(&self) -> Result<()> { + async fn cleanup(&self, _device_manager: &RwLock) -> Result<()> { let share_fs = match self.share_fs.as_ref() { Some(fs) => fs, None => return Ok(()), @@ -226,6 +227,10 @@ impl Volume for ShareFsVolume { Ok(()) } + + fn get_device_id(&self) -> Result> { + Ok(None) + } } pub(crate) fn is_share_fs_volume(m: &oci::Mount) -> bool { diff --git a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs index 5805106d2..fb12b3614 100644 --- a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs @@ -8,6 +8,8 @@ use std::path::Path; use anyhow::Result; use async_trait::async_trait; +use hypervisor::device::device_manager::DeviceManager; +use tokio::sync::RwLock; use super::Volume; use crate::share_fs::DEFAULT_KATA_GUEST_SANDBOX_DIR; @@ -99,11 +101,15 @@ impl Volume for ShmVolume { Ok(s) } - async fn cleanup(&self) -> Result<()> { + async fn cleanup(&self, _device_manager: &RwLock) -> Result<()> { // TODO: Clean up ShmVolume warn!(sl!(), "Cleaning up ShmVolume is still unimplemented."); Ok(()) } + + fn get_device_id(&self) -> Result> { + Ok(None) + } } pub(crate) fn is_shim_volume(m: &oci::Mount) -> bool { diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml index 768122684..f2a4ea522 100644 --- a/src/runtime-rs/crates/runtimes/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/Cargo.toml @@ -11,7 +11,7 @@ lazy_static = "1.4.0" netns-rs = "0.1.0" slog = "2.5.2" slog-scope = "4.4.0" -tokio = { version = "1.8.0", features = ["rt-multi-thread"] } +tokio = { version = "1.28.1", features = ["rt-multi-thread"] } hyper = { version = "0.14.20", features = ["stream", "server", "http1"] } hyperlocal = "0.8" serde_json = "1.0.88" diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index 440db1486..a60e1f5f1 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -19,7 +19,7 @@ slog = "2.5.2" slog-scope = "4.4.0" strum = { version = "0.24.0", features = ["derive"] } thiserror = "^1.0" -tokio = { version = "1.8.0", features = ["rt-multi-thread", "process", "fs"] } +tokio = { version = "1.28.1", features = ["rt-multi-thread", "process", "fs"] } ttrpc = { version = "0.7.1" } persist = {path = "../../persist"} agent = { path = "../../agent" } diff --git a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml index 58e6f6012..de3c03ffd 100644 --- a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] anyhow = "^1.0" async-trait = "0.1.48" -tokio = { version = "1.8.0" } +tokio = { version = "1.28.1" } common = { path = "../common" } -kata-types = { path = "../../../../libs/kata-types" } \ No newline at end of file +kata-types = { path = "../../../../libs/kata-types" } diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index b32c36773..1fa7ce4a0 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -397,12 +397,11 @@ fn load_config(spec: &oci::Spec, option: &Option>) -> Result path } else if let Some(option) = option { // get rid of the special characters in options to get the config path - let path = if option.len() > 2 { + if option.len() > 2 { from_utf8(&option[2..])?.to_string() } else { String::from("") - }; - path + } } else { String::from("") }; diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml index f3d8d9375..d648a2206 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1.0.27" serde_json = "1.0.82" slog = "2.5.2" slog-scope = "4.4.0" -tokio = { version = "1.8.0" } +tokio = { version = "1.28.1" } toml = "0.4.2" url = "2.1.1" async-std = "1.12.0" diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 2f9f03b0e..6dfea9fd0 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -140,7 +140,15 @@ impl Container { } spec.mounts = oci_mounts; - // TODO: handler device + let linux = spec + .linux + .as_ref() + .context("OCI spec missing linux field")?; + + let devices_agent = self + .resource_manager + .handler_devices(&config.container_id, linux) + .await?; // update cgroups self.resource_manager @@ -158,6 +166,7 @@ impl Container { storages, oci: Some(spec), sandbox_pidns, + devices: devices_agent, ..Default::default() }; @@ -174,7 +183,8 @@ impl Container { match process.process_type { ProcessType::Container => { if let Err(err) = inner.start_container(&process.container_id).await { - let _ = inner.stop_process(process, true).await; + let device_manager = self.resource_manager.get_device_manager().await; + let _ = inner.stop_process(process, true, &device_manager).await; return Err(err); } @@ -186,7 +196,8 @@ impl Container { } ProcessType::Exec => { if let Err(e) = inner.start_exec_process(process).await { - let _ = inner.stop_process(process, true).await; + let device_manager = self.resource_manager.get_device_manager().await; + let _ = inner.stop_process(process, true, &device_manager).await; return Err(e).context("enter process"); } @@ -268,7 +279,10 @@ impl Container { all: bool, ) -> Result<()> { let mut inner = self.inner.write().await; - inner.signal_process(container_process, signal, all).await + let device_manager = self.resource_manager.get_device_manager().await; + inner + .signal_process(container_process, signal, all, &device_manager) + .await } pub async fn exec_process( @@ -305,8 +319,9 @@ impl Container { pub async fn stop_process(&self, container_process: &ContainerProcess) -> Result<()> { let mut inner = self.inner.write().await; + let device_manager = self.resource_manager.get_device_manager().await; inner - .stop_process(container_process, true) + .stop_process(container_process, true, &device_manager) .await .context("stop process") } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index bb6c2ed07..12d4810fb 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -12,6 +12,7 @@ use common::{ error::Error, types::{ContainerID, ContainerProcess, ProcessExitStatus, ProcessStatus, ProcessType}, }; +use hypervisor::device::device_manager::DeviceManager; use nix::sys::signal::Signal; use resource::{rootfs::Rootfs, volume::Volume}; use tokio::sync::RwLock; @@ -193,6 +194,7 @@ impl ContainerInner { &mut self, process: &ContainerProcess, force: bool, + device_manager: &RwLock, ) -> Result<()> { let logger = logger_with_process(process); info!(logger, "begin to stop process"); @@ -212,7 +214,7 @@ impl ContainerInner { // send kill signal to container // ignore the error of sending signal, since the process would // have been killed and exited yet. - self.signal_process(process, Signal::SIGKILL as u32, false) + self.signal_process(process, Signal::SIGKILL as u32, false, device_manager) .await .map_err(|e| { warn!(logger, "failed to signal kill. {:?}", e); @@ -242,6 +244,7 @@ impl ContainerInner { process: &ContainerProcess, signal: u32, all: bool, + device_manager: &RwLock, ) -> Result<()> { let mut process_id: agent::ContainerProcessID = process.clone().into(); if all { @@ -253,8 +256,12 @@ impl ContainerInner { .signal_process(agent::SignalProcessRequest { process_id, signal }) .await?; - self.clean_volumes().await.context("clean volumes")?; - self.clean_rootfs().await.context("clean rootfs")?; + self.clean_volumes(device_manager) + .await + .context("clean volumes")?; + self.clean_rootfs(device_manager) + .await + .context("clean rootfs")?; Ok(()) } @@ -278,10 +285,10 @@ impl ContainerInner { Ok(()) } - async fn clean_volumes(&mut self) -> Result<()> { + async fn clean_volumes(&mut self, device_manager: &RwLock) -> Result<()> { let mut unhandled = Vec::new(); for v in self.volumes.iter() { - if let Err(err) = v.cleanup().await { + if let Err(err) = v.cleanup(device_manager).await { unhandled.push(Arc::clone(v)); warn!( sl!(), @@ -297,10 +304,10 @@ impl ContainerInner { Ok(()) } - async fn clean_rootfs(&mut self) -> Result<()> { + async fn clean_rootfs(&mut self, device_manager: &RwLock) -> Result<()> { let mut unhandled = Vec::new(); for rootfs in self.rootfs.iter() { - if let Err(err) = rootfs.cleanup().await { + if let Err(err) = rootfs.cleanup(device_manager).await { unhandled.push(Arc::clone(rootfs)); warn!( sl!(), diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs index 438a817e2..32856f27c 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs @@ -132,7 +132,7 @@ impl Process { info!(self.logger, "run io copy for {}", io_name); let io_name = io_name.to_string(); let logger = self.logger.new(o!("io_name" => io_name)); - let _ = tokio::spawn(async move { + tokio::spawn(async move { match tokio::io::copy(&mut reader, &mut writer).await { Err(e) => { warn!(logger, "run_io_copy: failed to copy stream: {}", e); @@ -156,7 +156,7 @@ impl Process { let exit_notifier = self.exit_watcher_tx.take(); let status = self.status.clone(); - let _ = tokio::spawn(async move { + tokio::spawn(async move { // wait on all of the container's io stream terminated info!(logger, "begin wait group io"); wg.wait().await; diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs b/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs index 81fb3d58b..874ccb7f1 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs @@ -47,7 +47,7 @@ impl HealthCheck { let stop_rx = self.stop_rx.clone(); let keep_abnormal = self.keep_abnormal; - let _ = tokio::spawn(async move { + tokio::spawn(async move { let mut version_check_threshold_count = 0; loop { diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index c5ec38e46..8202d854d 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -285,7 +285,7 @@ impl Sandbox for VirtSandbox { let agent = self.agent.clone(); let sender = self.msg_sender.clone(); info!(sl!(), "oom watcher start"); - let _ = tokio::spawn(async move { + tokio::spawn(async move { loop { match agent .get_oom_event(agent::Empty::new()) diff --git a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml index b8174ee82..4f098295a 100644 --- a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] anyhow = "^1.0" async-trait = "0.1.48" -tokio = { version = "1.8.0" } +tokio = { version = "1.28.1" } common = { path = "../common" } -kata-types = { path = "../../../../libs/kata-types" } \ No newline at end of file +kata-types = { path = "../../../../libs/kata-types" } diff --git a/src/runtime-rs/crates/service/Cargo.toml b/src/runtime-rs/crates/service/Cargo.toml index cb414abe3..8449328c0 100644 --- a/src/runtime-rs/crates/service/Cargo.toml +++ b/src/runtime-rs/crates/service/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "^1.0" async-trait = "0.1.48" slog = "2.5.2" slog-scope = "4.4.0" -tokio = { version = "1.8.0", features = ["rt-multi-thread"] } +tokio = { version = "1.28.1", features = ["rt-multi-thread"] } ttrpc = { version = "0.7.1" } common = { path = "../runtimes/common" } diff --git a/src/runtime-rs/crates/shim-ctl/Cargo.toml b/src/runtime-rs/crates/shim-ctl/Cargo.toml index b08e15daa..b1e844b0c 100644 --- a/src/runtime-rs/crates/shim-ctl/Cargo.toml +++ b/src/runtime-rs/crates/shim-ctl/Cargo.toml @@ -10,5 +10,5 @@ anyhow = "^1.0" common = { path = "../runtimes/common" } logging = { path = "../../../libs/logging"} runtimes = { path = "../runtimes" } -tokio = { version = "1.8.0", features = [ "rt", "rt-multi-thread" ] } +tokio = { version = "1.28.1", features = [ "rt", "rt-multi-thread" ] } diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml index 84521eb00..1f5dafb3f 100644 --- a/src/runtime-rs/crates/shim/Cargo.toml +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -27,7 +27,7 @@ slog-async = "2.5.2" slog-scope = "4.4.0" slog-stdlog = "4.1.0" thiserror = "1.0.30" -tokio = { version = "1.8.0", features = [ "rt", "rt-multi-thread" ] } +tokio = { version = "1.28.1", features = [ "rt", "rt-multi-thread" ] } unix_socket2 = "0.5.4" kata-types = { path = "../../../libs/kata-types"} diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 57ee327b4..055e7d560 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -97,16 +97,15 @@ GENERATED_VARS = \ VERSION \ CONFIG_ACRN_IN \ CONFIG_QEMU_IN \ + CONFIG_QEMU_NVIDIA_GPU_IN \ + CONFIG_QEMU_SEV_IN \ CONFIG_QEMU_TDX_IN \ - CONFIG_QEMU_GPU_IN \ + CONFIG_QEMU_SNP_IN \ CONFIG_CLH_IN \ CONFIG_FC_IN \ CONFIG_CLH_TDX_IN \ - CONFIG_QEMU_TDX_IN \ - CONFIG_QEMU_SEV_IN \ CONFIG_QEMU_SE_IN \ CONFIG_REMOTE_IN \ - CONFIG_QEMU_SNP_IN \ $(USER_VARS) SCRIPTS += $(COLLECT_SCRIPT) SCRIPTS_DIR := $(BINDIR) @@ -122,8 +121,7 @@ KERNELDIR := $(PKGDATADIR) IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME) INITRDPATH := $(PKGDATADIR)/$(INITRDNAME) -INITRDSEVPATH := $(PKGDATADIR)/$(INITRDSEVNAME) -IMAGETDXPATH := $(PKGDATADIR)/$(IMAGETDXNAME) +INITRDSEVPATH := $(PKGDATADIR)/$(INITRDSEVNAME) ROOTFSTYPE_EXT4 := \"ext4\" ROOTFSTYPE_XFS := \"xfs\" @@ -133,8 +131,6 @@ DEFROOTFSTYPE := $(ROOTFSTYPE_EXT4) FIRMWAREPATH := FIRMWAREVOLUMEPATH := TDSHIMFIRMWAREPATH := ${PREFIXDEPS}/share/td-shim/td-shim.bin -SEVFIRMWAREPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd -SNPFIRMWAREPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd AGENTCONFIGFILEPATH := /etc/agent-config.toml AGENTCONFIGFILEKERNELPARAM := agent.config_file=$(AGENTCONFIGFILEPATH) @@ -152,9 +148,12 @@ SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verifica SNPKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SNP) KERNELPARAMS += $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS) -FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF_CODE.fd +FIRMWARETDVFPATH := $(PREFIXDEPS)/share/tdvf/OVMF.fd FIRMWARETDVFVOLUMEPATH := $(PREFIXDEPS)/share/tdvf/OVMF_VARS.fd +FIRMWARESEVPATH := $(PREFIXDEPS)/share/ovmf/AMDSEV.fd +FIRMWARESNPPATH := $(PREFIXDEPS)/share/ovmf/OVMF.fd + # Name of default configuration file the runtime will use. CONFIG_FILE = configuration.toml @@ -179,6 +178,9 @@ QEMUSNPVALIDHYPERVISORPATHS := [\"$(QEMUSNPPATH)\"] QEMUTDXPATH := $(QEMUBINDIR)/$(QEMUTDXCMD) QEMUTDXVALIDHYPERVISORPATHS := [\"$(QEMUTDXPATH)\"] +QEMUSNPPATH := $(QEMUBINDIR)/$(QEMUSNPCMD) +QEMUSNPVALIDHYPERVISORPATHS := [\"$(QEMUSNPPATH)\"] + QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) CLHPATH := $(CLHBINDIR)/$(CLHCMD) @@ -237,10 +239,9 @@ DEFDISABLEBLOCK := false DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFSHAREDFS_QEMU_TDX_VIRTIOFS := virtio-9p +DEFSHAREDFS_QEMU_SEV_VIRTIOFS := virtio-9p +DEFSHAREDFS_QEMU_SNP_VIRTIOFS := virtio-9p DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd -ifeq ($(ARCH),ppc64le) -DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/qemu/virtiofsd -endif DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"] # Default DAX mapping cache size in MiB #if value is 0, DAX is not enabled @@ -390,7 +391,31 @@ ifneq (,$(QEMUCMD)) CONFIG_QEMU_GPU = config/$(CONFIG_FILE_QEMU_GPU) CONFIG_QEMU_GPU_IN = $(CONFIG_QEMU_GPU).in - CONFIGS += $(CONFIG_QEMU_GPU) + CONFIG_PATH_QEMU_SEV = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SEV)) + CONFIG_PATHS += $(CONFIG_PATH_QEMU_SEV) + + SYSCONFIG_QEMU_SEV = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SEV)) + SYSCONFIG_PATHS_SEV += $(SYSCONFIG_QEMU_SEV) + + CONFIGS += $(CONFIG_QEMU_SEV) + + CONFIG_FILE_QEMU_SNP = configuration-qemu-snp.toml + CONFIG_QEMU_SNP = config/$(CONFIG_FILE_QEMU_SNP) + CONFIG_QEMU_SNP_IN = $(CONFIG_QEMU_SNP).in + + CONFIG_PATH_QEMU_SNP = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_SNP)) + CONFIG_PATHS += $(CONFIG_PATH_QEMU_SNP) + + SYSCONFIG_QEMU_SNP = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_SNP)) + SYSCONFIG_PATHS_SNP += $(SYSCONFIG_QEMU_SNP) + + CONFIGS += $(CONFIG_QEMU_SNP) + + CONFIG_FILE_QEMU_NVIDIA_GPU = configuration-qemu-nvidia-gpu.toml + CONFIG_QEMU_NVIDIA_GPU = config/$(CONFIG_FILE_QEMU_NVIDIA_GPU) + CONFIG_QEMU_NVIDIA_GPU_IN = $(CONFIG_QEMU_NVIDIA_GPU).in + + CONFIGS += $(CONFIG_QEMU_NVIDIA_GPU) # qemu-specific options (all should be suffixed by "_QEMU") DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi @@ -401,6 +426,10 @@ ifneq (,$(QEMUCMD)) KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) KERNELPATH = $(KERNELDIR)/$(KERNELNAME) + KERNELSEVTYPE = compressed + KERNELSEVNAME = $(call MAKE_KERNEL_SEV_NAME,$(KERNELSEVTYPE)) + KERNELSEVPATH = $(KERNELDIR)/$(KERNELSEVNAME) + KERNELTDXTYPE = compressed KERNELTDXNAME = $(call MAKE_KERNEL_TDX_NAME,$(KERNELTDXTYPE)) KERNELTDXPATH = $(KERNELDIR)/$(KERNELTDXNAME) @@ -409,6 +438,10 @@ ifneq (,$(QEMUCMD)) KERNELSEVNAME = $(call MAKE_KERNEL_SEV_NAME,$(KERNELSEVTYPE)) KERNELSEVPATH = $(KERNELDIR)/$(KERNELSEVNAME) + KERNELSNPTYPE = compressed + KERNELSNPNAME = $(call MAKE_KERNEL_SNP_NAME,$(KERNELSNPTYPE)) + KERNELSNPPATH = $(KERNELDIR)/$(KERNELSNPNAME) + KERNELSENAME = kata-containers-secure.img KERNELSEPATH = $(KERNELDIR)/$(KERNELSENAME) endif @@ -575,8 +608,8 @@ USER_VARS += IMAGEPATH USER_VARS += IMAGETDXNAME USER_VARS += IMAGETDXPATH USER_VARS += INITRDNAME -USER_VARS += INITRDPATH USER_VARS += INITRDSEVNAME +USER_VARS += INITRDPATH USER_VARS += INITRDSEVPATH USER_VARS += DEFROOTFSTYPE USER_VARS += MACHINETYPE @@ -587,20 +620,22 @@ USER_VARS += KERNELTYPE_ACRN USER_VARS += KERNELTYPE_CLH USER_VARS += KERNELPATH_ACRN USER_VARS += KERNELPATH +USER_VARS += KERNELSEVPATH USER_VARS += KERNELTDXPATH USER_VARS += KERNELSEVPATH USER_VARS += KERNELSEPATH +USER_VARS += KERNELSNPPATH USER_VARS += KERNELPATH_CLH USER_VARS += KERNELTDXPATH_CLH USER_VARS += KERNELPATH_FC USER_VARS += KERNELVIRTIOFSPATH USER_VARS += FIRMWAREPATH +USER_VARS += FIRMWARESEVPATH USER_VARS += FIRMWARETDVFPATH USER_VARS += FIRMWAREVOLUMEPATH USER_VARS += TDSHIMFIRMWAREPATH USER_VARS += FIRMWARETDVFVOLUMEPATH -USER_VARS += SEVFIRMWAREPATH -USER_VARS += SNPFIRMWAREPATH +USER_VARS += FIRMWARESNPPATH USER_VARS += MACHINEACCELERATORS USER_VARS += CPUFEATURES USER_VARS += TDXCPUFEATURES @@ -628,10 +663,13 @@ USER_VARS += PROJECT_URL USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUTDXCMD +USER_VARS += QEMUSNPCMD USER_VARS += QEMUPATH USER_VARS += QEMUTDXPATH +USER_VARS += QEMUSNPPATH USER_VARS += QEMUVALIDHYPERVISORPATHS USER_VARS += QEMUTDXVALIDHYPERVISORPATHS +USER_VARS += QEMUSNPVALIDHYPERVISORPATHS USER_VARS += QEMUVIRTIOFSCMD USER_VARS += QEMUVIRTIOFSPATH USER_VARS += QEMUSNPPATH @@ -666,6 +704,8 @@ USER_VARS += DEFBLOCKDEVICEAIO_QEMU USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS USER_VARS += DEFSHAREDFS_QEMU_TDX_VIRTIOFS +USER_VARS += DEFSHAREDFS_QEMU_SEV_VIRTIOFS +USER_VARS += DEFSHAREDFS_QEMU_SNP_VIRTIOFS USER_VARS += DEFVIRTIOFSDAEMON USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS USER_VARS += DEFVIRTIOFSCACHESIZE @@ -688,6 +728,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT USER_VARS += DEFSTATICRESOURCEMGMT_FC USER_VARS += DEFSTATICRESOURCEMGMT_TEE USER_VARS += DEFBINDMOUNTS +USER_VARS += DEFSERVICEOFFLOAD USER_VARS += DEFVFIOMODE USER_VARS += BUILDFLAGS USER_VARS += DEFSERVICEOFFLOAD @@ -789,10 +830,19 @@ define MAKE_KERNEL_VIRTIOFS_NAME $(if $(findstring uncompressed,$1),vmlinux-virtiofs.container,vmlinuz-virtiofs.container) endef +define MAKE_KERNEL_SEV_NAME +$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container) +endef + define MAKE_KERNEL_TDX_NAME $(if $(findstring uncompressed,$1),vmlinux-tdx.container,vmlinuz-tdx.container) endef +# SNP configuration uses the SEV kernel +define MAKE_KERNEL_SNP_NAME +$(if $(findstring uncompressed,$1),vmlinux-sev.container,vmlinuz-sev.container) +endef + GENERATED_FILES += pkg/katautils/config-settings.go $(RUNTIME_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) | show-summary @@ -1027,4 +1077,4 @@ ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS))) endif @printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR)) @printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR)) - @printf "\n" + @printf "\n" \ No newline at end of file diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index c129f8f43..f17480aba 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/procfs" "github.com/urfave/cli" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" @@ -113,6 +114,7 @@ type HypervisorInfo struct { Msize9p uint32 MemorySlots uint32 PCIeRootPort uint32 + ColdPlugVFIO hv.PCIePort HotplugVFIOOnRootBus bool Debug bool } @@ -305,17 +307,17 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) { } return HypervisorInfo{ - Debug: config.HypervisorConfig.Debug, - MachineType: config.HypervisorConfig.HypervisorMachineType, - Version: version, - Path: hypervisorPath, - BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, - Msize9p: config.HypervisorConfig.Msize9p, - MemorySlots: config.HypervisorConfig.MemSlots, - EntropySource: config.HypervisorConfig.EntropySource, - SharedFS: config.HypervisorConfig.SharedFS, - VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, - + Debug: config.HypervisorConfig.Debug, + MachineType: config.HypervisorConfig.HypervisorMachineType, + Version: version, + Path: hypervisorPath, + BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, + Msize9p: config.HypervisorConfig.Msize9p, + MemorySlots: config.HypervisorConfig.MemSlots, + EntropySource: config.HypervisorConfig.EntropySource, + SharedFS: config.HypervisorConfig.SharedFS, + VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, + ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO, HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus, PCIeRootPort: config.HypervisorConfig.PCIeRootPort, SocketPath: socketPath, diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index 321bc507b..3760104d0 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/BurntSushi/toml" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -74,6 +75,7 @@ func createConfig(configPath string, fileData string) error { } func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) { + var coldPlugVFIO hv.PCIePort const logPath = "/log/path" hypervisorPath := filepath.Join(prefixDir, "hypervisor") kernelPath := filepath.Join(prefixDir, "kernel") @@ -86,6 +88,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) + coldPlugVFIO = hv.NoPort disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := filepath.Join(prefixDir, "virtiofsd") @@ -129,6 +132,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC BlockDeviceDriver: blockStorageDriver, EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, + ColdPlugVFIO: coldPlugVFIO, PCIeRootPort: pcieRootPort, DisableNewNetNs: disableNewNetNs, DefaultVCPUCount: hypConfig.NumVCPUs, @@ -191,12 +195,13 @@ func genericGetExpectedHostDetails(tmpdir string, expectedVendor string, expecte expectedSupportVSocks, _ := vcUtils.SupportsVsocks() expectedHostDetails := HostInfo{ - Kernel: expectedKernelVersion, - Architecture: expectedArch, - Distro: expectedDistro, - CPU: expectedCPU, - VMContainerCapable: expectedVMContainerCapable, - SupportVSocks: expectedSupportVSocks, + AvailableGuestProtections: vc.AvailableGuestProtections(), + Kernel: expectedKernelVersion, + Architecture: expectedArch, + Distro: expectedDistro, + CPU: expectedCPU, + VMContainerCapable: expectedVMContainerCapable, + SupportVSocks: expectedSupportVSocks, } testProcCPUInfo := filepath.Join(tmpdir, "cpuinfo") @@ -273,6 +278,7 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo { HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus, PCIeRootPort: config.HypervisorConfig.PCIeRootPort, + ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO, } if os.Geteuid() == 0 { diff --git a/src/runtime/config/configuration-qemu-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in similarity index 100% rename from src/runtime/config/configuration-qemu-gpu.toml.in rename to src/runtime/config/configuration-qemu-nvidia-gpu.toml.in diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in index d40555fa2..7571cc6d0 100644 --- a/src/runtime/config/configuration-qemu-sev.toml.in +++ b/src/runtime/config/configuration-qemu-sev.toml.in @@ -5,7 +5,7 @@ # XXX: WARNING: this file is auto-generated. # XXX: -# XXX: Source file: "@CONFIG_QEMU_IN@" +# XXX: Source file: "@CONFIG_QEMU_SEV_IN@" # XXX: Project: # XXX: Name: @PROJECT_NAME@ # XXX: Type: @PROJECT_TYPE@ @@ -84,7 +84,7 @@ kernel_params = "@SEVKERNELPARAMS@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty -firmware = "@SEVFIRMWAREPATH@" +firmware = "@FIRMWARESEVPATH@" # Path to the firmware volume. # firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables @@ -186,7 +186,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus -shared_fs = "virtio-9p" +shared_fs = "@DEFSHAREDFS_QEMU_SEV_VIRTIOFS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" @@ -520,8 +520,8 @@ kernel_modules=[] #debug_console_enabled = true # Agent connection dialing timeout value in seconds -# (default: 30) -#dial_timeout = 30 +# (default: 90) +dial_timeout = 90 [runtime] # If enabled, the runtime will log additional debug messages to the @@ -669,4 +669,4 @@ service_offload = @DEFSERVICEOFFLOAD@ # # Keys can be remotely provisioned. The Kata agent fetches them from e.g. # a HTTPS URL: -#provision=https://my-key-broker.foo/tenant/ +#provision=https://my-key-broker.foo/tenant/ \ No newline at end of file diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index d01de64b8..d4fd77a88 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -1,25 +1,31 @@ +# Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # Copyright 2022 Advanced Micro Devices, Inc. +# Copyright (c) 2023 IBM Inc. # # SPDX-License-Identifier: Apache-2.0 # # XXX: WARNING: this file is auto-generated. # XXX: -# XXX: Source file: "@CONFIG_QEMU_IN@" +# XXX: Source file: "@CONFIG_QEMU_SNP_IN@" # XXX: Project: # XXX: Name: @PROJECT_NAME@ # XXX: Type: @PROJECT_TYPE@ -# Basing SNP updates on this post from Joana -# https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md - [hypervisor.qemu] path = "@QEMUSNPPATH@" -kernel = "@KERNELSEVPATH@" -#SNP should be able to use SEV kernel and initrd with things being passed via kernel params +kernel = "@KERNELSNPPATH@" +#image = "@IMAGEPATH@" initrd = "@INITRDSEVPATH@" machine_type = "@MACHINETYPE@" +# rootfs filesystem type: +# - ext4 (default) +# - xfs +# - erofs +rootfs_type=@DEFROOTFSTYPE@ + # Enable confidential guest support. # Toggling that setting may trigger different hardware features, ranging # from memory encryption to both memory and CPU-state encryption and integrity. @@ -76,7 +82,7 @@ kernel_params = "@SNPKERNELPARAMS@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty -firmware = "@SNPFIRMWAREPATH@" +firmware = "@FIRMWARESNPPATH@" # Path to the firmware volume. # firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables @@ -178,10 +184,10 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus -shared_fs = "virtio-9p" +shared_fs = "@DEFSHAREDFS_QEMU_SNP_VIRTIOFS@" # Path to vhost-user-fs daemon. -# virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" +virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon # The default if not set is empty (all annotations rejected.) @@ -191,6 +197,9 @@ valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ +# Default size of virtqueues +virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ + # Extra args for virtiofsd daemon # # Format example: @@ -203,7 +212,7 @@ virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ # Cache mode: # -# - none +# - never # Metadata, data, and pathname lookup are not cached in guest. They are # always fetched from host and any changes are immediately pushed to host. # @@ -221,6 +230,20 @@ virtio_fs_cache = "@DEFVIRTIOFSCACHE@" # or nvdimm. block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@" +# aio is the I/O mechanism used by qemu +# Options: +# +# - threads +# Pthread based disk I/O. +# +# - native +# Native Linux I/O. +# +# - io_uring +# Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and +# qemu >=5.0. +block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@" + # Specifies cache-related options will be set to block devices or not. # Default false #block_device_cache_set = true @@ -284,11 +307,16 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@ +# The timeout for reconnecting on non-server spdk sockets when the remote end goes away. +# qemu will delay this many seconds and then attempt to reconnect. +# Zero disables reconnecting, and the default is zero. +vhost_user_reconnect_timeout_sec = 0 + # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. # This option will be ignored if VM templating is enabled. -file_mem_backend = "@DEFFILEMEMBACKEND@" +file_mem_backend = "" # List of valid annotations values for the file_mem_backend annotation # The default if not set is empty (all annotations rejected.) @@ -300,7 +328,7 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ pflashes = [] # This option changes the default hypervisor and kernel parameters -# to enable debug output where available. +# to enable debug output where available. And Debug also enable the hmp socket. # # Default false #enable_debug = true @@ -512,8 +540,8 @@ kernel_modules=[] #debug_console_enabled = true # Agent connection dialing timeout value in seconds -# (default: 30) -#dial_timeout = 30 +# (default: 90) +dial_timeout = 90 [runtime] # If enabled, the runtime will log additional debug messages to the @@ -546,6 +574,11 @@ internetworking_model="@DEFNETWORKMODEL_QEMU@" # (default: true) disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ +# vCPUs pinning settings +# if enabled, each vCPU thread will be scheduled to a fixed CPU +# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet) +# enable_vcpus_pinning = false + # Apply a custom SELinux security policy to the container process inside the VM. # This is used when you want to apply a type other than the default `container_t`, # so general users should not uncomment and apply it. @@ -661,4 +694,4 @@ service_offload = @DEFSERVICEOFFLOAD@ # # Keys can be remotely provisioned. The Kata agent fetches them from e.g. # a HTTPS URL: -#provision=https://my-key-broker.foo/tenant/ +#provision=https://my-key-broker.foo/tenant/ \ No newline at end of file diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index add7ed607..7ef2cb610 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -380,6 +380,11 @@ pflashes = [] # Default false #hotplug_vfio_on_root_bus = true +# In a confidential compute environment hot-plugging can compromise +# security. Enable cold-plugging of VFIO devices to a root-port. +# The default setting is "no-port", which means disabled. +#cold_plug_vfio = "root-port" + # Before hot plugging a PCIe device, you need to add a pcie_root_port device. # Use this parameter when using some large PCI bar devices, such as Nvidia GPU # The value means the number of pcie_root_port diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go index ccad5ceea..e3e8e9369 100644 --- a/src/runtime/pkg/containerd-shim-v2/create_test.go +++ b/src/runtime/pkg/containerd-shim-v2/create_test.go @@ -20,6 +20,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" @@ -308,6 +309,7 @@ func TestCreateContainerConfigFail(t *testing.T) { } func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) { + var coldPlugVFIO hv.PCIePort if dir == "" { return "", fmt.Errorf("BUG: need directory") } @@ -332,6 +334,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") + coldPlugVFIO = hv.RootPort configFileOptions := ktu.RuntimeConfigOptions{ Hypervisor: "qemu", @@ -350,6 +353,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err DisableNewNetNs: disableNewNetNs, SharedFS: sharedFS, VirtioFSDaemon: virtioFSdaemon, + ColdPlugVFIO: coldPlugVFIO, } runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions) diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index 79e00adbd..bfffa31a2 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -10,10 +10,12 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) @@ -133,3 +135,99 @@ func GetAPVFIODevices(sysfsdev string) ([]string, error) { // Split by newlines, omitting final newline return strings.Split(string(data[:len(data)-1]), "\n"), nil } + +// Ignore specific PCI devices, supply the pciClass and the bitmask to check +// against the device class, deviceBDF for meaningfull info message +func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (bool, error) { + if pciClass == "" { + return false, nil + } + pciClassID, err := strconv.ParseUint(pciClass, 0, 32) + if err != nil { + return false, err + } + // ClassID is 16 bits, remove the two trailing zeros + pciClassID = pciClassID >> 8 + if pciClassID&bitmask == bitmask { + deviceLogger().Infof("Ignoring PCI (Host) Bridge deviceBDF %v Class %x", deviceBDF, pciClassID) + return true, nil + } + return false, nil +} + +// GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group +// We can reuse this function at various levels, sandbox, container. +// Only the VFIO module is allowed to do bus assignments, all other modules need to +// ignore it if used as helper function to get VFIO information. +func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignment bool) ([]*config.VFIODev, error) { + + vfioDevs := []*config.VFIODev{} + + vfioGroup := filepath.Base(device.HostPath) + iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices") + + deviceFiles, err := os.ReadDir(iommuDevicesPath) + if err != nil { + return nil, err + } + + // Pass all devices in iommu group + for i, deviceFile := range deviceFiles { + //Get bdf of device eg 0000:00:1c.0 + deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath) + if err != nil { + return nil, err + } + id := utils.MakeNameID("vfio", device.ID+strconv.Itoa(i), maxDevIDSize) + + pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass) + // We need to ignore Host or PCI Bridges that are in the same IOMMU group as the + // passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge. + // Class 0x0604 is PCI bridge, 0x0600 is Host bridge + ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600) + if err != nil { + return nil, err + } + if ignorePCIDevice { + continue + } + + var vfio config.VFIODev + + switch vfioDeviceType { + case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: + isPCIe := isPCIeDevice(deviceBDF) + // Do not directly assign to `vfio` -- need to access field still + vfioPCI := config.VFIOPCIDev{ + ID: id, + Type: vfioDeviceType, + BDF: deviceBDF, + SysfsDev: deviceSysfsDev, + IsPCIe: isPCIe, + Class: pciClass, + } + if isPCIe && !ignoreBusAssignment { + vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) + AllPCIeDevs[deviceBDF] = true + } + vfio = vfioPCI + case config.VFIOAPDeviceMediatedType: + devices, err := GetAPVFIODevices(deviceSysfsDev) + if err != nil { + return nil, err + } + vfio = config.VFIOAPDev{ + ID: id, + SysfsDev: deviceSysfsDev, + Type: config.VFIOAPDeviceMediatedType, + APDevices: devices, + } + default: + return nil, fmt.Errorf("Failed to append device: VFIO device type unrecognized") + } + + vfioDevs = append(vfioDevs, &vfio) + } + + return vfioDevs, nil +} diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 1099f8f0b..106220dcf 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "github.com/sirupsen/logrus" @@ -54,25 +53,6 @@ func NewVFIODevice(devInfo *config.DeviceInfo) *VFIODevice { } } -// Ignore specific PCI devices, supply the pciClass and the bitmask to check -// against the device class, deviceBDF for meaningfull info message -func (device *VFIODevice) checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (bool, error) { - if pciClass == "" { - return false, nil - } - pciClassID, err := strconv.ParseUint(pciClass, 0, 32) - if err != nil { - return false, err - } - // ClassID is 16 bits, remove the two trailing zeros - pciClassID = pciClassID >> 8 - if pciClassID&bitmask == bitmask { - deviceLogger().Infof("Ignoring PCI (Host) Bridge deviceBDF %v Class %x", deviceBDF, pciClassID) - return true, nil - } - return false, nil -} - // Attach is standard interface of api.Device, it's used to add device to some // DeviceReceiver func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceReceiver) (retErr error) { @@ -90,72 +70,11 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece } }() - vfioGroup := filepath.Base(device.DeviceInfo.HostPath) - iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices") - - deviceFiles, err := os.ReadDir(iommuDevicesPath) + device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo, false) if err != nil { return err } - // Pass all devices in iommu group - for i, deviceFile := range deviceFiles { - //Get bdf of device eg 0000:00:1c.0 - deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath) - if err != nil { - return err - } - id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize) - - pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass) - // We need to ignore Host or PCI Bridges that are in the same IOMMU group as the - // passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge. - // Class 0x0604 is PCI bridge, 0x0600 is Host bridge - ignorePCIDevice, err := device.checkIgnorePCIClass(pciClass, deviceBDF, 0x0600) - if err != nil { - return err - } - if ignorePCIDevice { - continue - } - - var vfio config.VFIODev - - switch vfioDeviceType { - case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType: - isPCIe := isPCIeDevice(deviceBDF) - // Do not directly assign to `vfio` -- need to access field still - vfioPCI := config.VFIOPCIDev{ - ID: id, - Type: vfioDeviceType, - BDF: deviceBDF, - SysfsDev: deviceSysfsDev, - IsPCIe: isPCIe, - Class: pciClass, - } - if isPCIe { - vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) - AllPCIeDevs[deviceBDF] = true - } - vfio = vfioPCI - case config.VFIOAPDeviceMediatedType: - devices, err := GetAPVFIODevices(deviceSysfsDev) - if err != nil { - return err - } - vfio = config.VFIOAPDev{ - ID: id, - SysfsDev: deviceSysfsDev, - Type: config.VFIOAPDeviceMediatedType, - APDevices: devices, - } - default: - return fmt.Errorf("Failed to append device: VFIO device type unrecognized") - } - - device.VfioDevs = append(device.VfioDevs, &vfio) - } - coldPlug := device.DeviceInfo.ColdPlug deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device") diff --git a/src/runtime/pkg/device/manager/manager.go b/src/runtime/pkg/device/manager/manager.go index 34a51d300..baf1209a7 100644 --- a/src/runtime/pkg/device/manager/manager.go +++ b/src/runtime/pkg/device/manager/manager.go @@ -116,7 +116,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device if devInfo.ID, err = dm.newDeviceID(); err != nil { return nil, err } - if isVFIO(devInfo.HostPath) { + if IsVFIO(devInfo.HostPath) { return drivers.NewVFIODevice(&devInfo), nil } else if isVhostUserBlk(devInfo) { if devInfo.DriverOptions == nil { diff --git a/src/runtime/pkg/device/manager/utils.go b/src/runtime/pkg/device/manager/utils.go index 17d14741c..e78205d0c 100644 --- a/src/runtime/pkg/device/manager/utils.go +++ b/src/runtime/pkg/device/manager/utils.go @@ -17,8 +17,8 @@ const ( vfioPath = "/dev/vfio/" ) -// isVFIO checks if the device provided is a vfio group. -func isVFIO(hostPath string) bool { +// IsVFIO checks if the device provided is a vfio group. +func IsVFIO(hostPath string) bool { // Ignore /dev/vfio/vfio character device if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) { return false diff --git a/src/runtime/pkg/device/manager/utils_test.go b/src/runtime/pkg/device/manager/utils_test.go index 273283823..b57992b3d 100644 --- a/src/runtime/pkg/device/manager/utils_test.go +++ b/src/runtime/pkg/device/manager/utils_test.go @@ -31,7 +31,7 @@ func TestIsVFIO(t *testing.T) { } for _, d := range data { - isVFIO := isVFIO(d.path) + isVFIO := IsVFIO(d.path) assert.Equal(t, d.expected, isVFIO) } } diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go index b1f58e7fa..482b7e9e2 100644 --- a/src/runtime/pkg/hypervisors/hypervisor_state.go +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -5,6 +5,8 @@ package hypervisors +import "fmt" + // Bridge is a bridge where devices can be hot plugged type Bridge struct { // DeviceAddr contains information about devices plugged and its address in the bridge @@ -26,6 +28,34 @@ type CPUDevice struct { ID string } +// PCIePort distinguish only between root and switch port +type PCIePort string + +const ( + // RootPort attach VFIO devices to a root-port + RootPort PCIePort = "root-port" + // SwitchPort attach VFIO devices to a switch-port + SwitchPort = "switch-port" + // BridgePort is the default + BridgePort = "bridge-port" + // NoPort is for disabling VFIO hotplug/coldplug + NoPort = "no-port" +) + +func (p PCIePort) String() string { + switch p { + case RootPort: + return "root-port" + case SwitchPort: + return "switch-port" + case BridgePort: + return "bridge-port" + case NoPort: + return "no-port" + } + return fmt.Sprintf("", string(p)) +} + type HypervisorState struct { BlockIndexMap map[int]struct{} @@ -41,10 +71,10 @@ type HypervisorState struct { // HotpluggedCPUs is the list of CPUs that were hot-added HotpluggedVCPUs []CPUDevice - HotpluggedMemory int - VirtiofsDaemonPid int - Pid int - PCIeRootPort int - + HotpluggedMemory int + VirtiofsDaemonPid int + Pid int + PCIeRootPort int + ColdPlugVFIO PCIePort HotplugVFIOOnRootBus bool } diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index 4e3a784a2..4c8257a40 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -14,6 +14,7 @@ import ( "strconv" "testing" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" ) @@ -224,6 +225,7 @@ type RuntimeConfigOptions struct { JaegerPassword string PFlash []string PCIeRootPort uint32 + ColdPlugVFIO hv.PCIePort DefaultVCPUCount uint32 DefaultMaxVCPUCount uint32 DefaultMemSize uint32 @@ -317,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + ` hotplug_vfio_on_root_bus = ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + ` pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + ` + cold_plug_vfio = "` + config.ColdPlugVFIO.String() + `" msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + ` enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + ` guest_hook_path = "` + config.DefaultGuestHookPath + `" diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 4ec977ae5..a59ce8a17 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -9,6 +9,10 @@ package katautils +import ( + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" +) + // name is the name of the runtime var NAME = "@RUNTIME_NAME@" @@ -110,3 +114,5 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" // Default config file used by stateless systems. var defaultRuntimeConfiguration = "@CONFIG_PATH@" + +const defaultColdPlugVFIO = hv.NoPort diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index fbe304b5e..f918a5bbc 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -20,6 +20,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -78,97 +79,98 @@ type factory struct { } type hypervisor struct { - Path string `toml:"path"` - JailerPath string `toml:"jailer_path"` - Kernel string `toml:"kernel"` - CtlPath string `toml:"ctlpath"` - Initrd string `toml:"initrd"` - Image string `toml:"image"` - RootfsType string `toml:"rootfs_type"` - Firmware string `toml:"firmware"` - FirmwareVolume string `toml:"firmware_volume"` - MachineAccelerators string `toml:"machine_accelerators"` - CPUFeatures string `toml:"cpu_features"` - KernelParams string `toml:"kernel_params"` - MachineType string `toml:"machine_type"` - BlockDeviceDriver string `toml:"block_device_driver"` - EntropySource string `toml:"entropy_source"` - SharedFS string `toml:"shared_fs"` - VirtioFSDaemon string `toml:"virtio_fs_daemon"` - VirtioFSCache string `toml:"virtio_fs_cache"` - VhostUserStorePath string `toml:"vhost_user_store_path"` - FileBackedMemRootDir string `toml:"file_mem_backend"` - GuestHookPath string `toml:"guest_hook_path"` - GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` - SeccompSandbox string `toml:"seccompsandbox"` - GuestPreAttestationURI string `toml:"guest_pre_attestation_kbs_uri"` - GuestPreAttestationMode string `toml:"guest_pre_attestation_kbs_mode"` - GuestPreAttestationKeyset string `toml:"guest_pre_attestation_keyset"` - SEVCertChainPath string `toml:"sev_cert_chain"` - BlockDeviceAIO string `toml:"block_device_aio"` - RemoteHypervisorSocket string `toml:"remote_hypervisor_socket"` - HypervisorPathList []string `toml:"valid_hypervisor_paths"` - JailerPathList []string `toml:"valid_jailer_paths"` - CtlPathList []string `toml:"valid_ctlpaths"` - VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` - VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` - PFlashList []string `toml:"pflashes"` - VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` - FileBackedMemRootList []string `toml:"valid_file_mem_backends"` - EntropySourceList []string `toml:"valid_entropy_sources"` - EnableAnnotations []string `toml:"enable_annotations"` - RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` - TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` - MemOffset uint64 `toml:"memory_offset"` - DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` - DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` - DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` - DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` - DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` - NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` - NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` - NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` - NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` - VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` - VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` - DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` - MemorySize uint32 `toml:"default_memory"` - MemSlots uint32 `toml:"memory_slots"` - DefaultBridges uint32 `toml:"default_bridges"` - Msize9p uint32 `toml:"msize_9p"` - PCIeRootPort uint32 `toml:"pcie_root_port"` - GuestPreAttestationGRPCTimeout uint32 `toml:"guest_pre_attestation_grpc_timeout"` - SEVGuestPolicy uint32 `toml:"sev_guest_policy"` - SNPGuestPolicy uint64 `toml:"snp_guest_policy"` - RemoteHypervisorTimeout uint32 `toml:"remote_hypervisor_timeout"` - NumVCPUs int32 `toml:"default_vcpus"` - BlockDeviceCacheSet bool `toml:"block_device_cache_set"` - BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` - BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` - EnableVhostUserStore bool `toml:"enable_vhost_user_store"` - VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` - DisableBlockDeviceUse bool `toml:"disable_block_device_use"` - MemPrealloc bool `toml:"enable_mem_prealloc"` - HugePages bool `toml:"enable_hugepages"` - VirtioMem bool `toml:"enable_virtio_mem"` - IOMMU bool `toml:"enable_iommu"` - IOMMUPlatform bool `toml:"enable_iommu_platform"` - Debug bool `toml:"enable_debug"` - DisableNestingChecks bool `toml:"disable_nesting_checks"` - EnableIOThreads bool `toml:"enable_iothreads"` - DisableImageNvdimm bool `toml:"disable_image_nvdimm"` - HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` - DisableVhostNet bool `toml:"disable_vhost_net"` - GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` - ConfidentialGuest bool `toml:"confidential_guest"` - SevSnpGuest bool `toml:"sev_snp_guest"` - GuestSwap bool `toml:"enable_guest_swap"` - Rootless bool `toml:"rootless"` - DisableSeccomp bool `toml:"disable_seccomp"` - DisableSeLinux bool `toml:"disable_selinux"` - DisableGuestSeLinux bool `toml:"disable_guest_selinux"` - LegacySerial bool `toml:"use_legacy_serial"` - GuestPreAttestation bool `toml:"guest_pre_attestation"` + Path string `toml:"path"` + JailerPath string `toml:"jailer_path"` + Kernel string `toml:"kernel"` + CtlPath string `toml:"ctlpath"` + Initrd string `toml:"initrd"` + Image string `toml:"image"` + RootfsType string `toml:"rootfs_type"` + Firmware string `toml:"firmware"` + FirmwareVolume string `toml:"firmware_volume"` + MachineAccelerators string `toml:"machine_accelerators"` + CPUFeatures string `toml:"cpu_features"` + KernelParams string `toml:"kernel_params"` + MachineType string `toml:"machine_type"` + BlockDeviceDriver string `toml:"block_device_driver"` + EntropySource string `toml:"entropy_source"` + SharedFS string `toml:"shared_fs"` + VirtioFSDaemon string `toml:"virtio_fs_daemon"` + VirtioFSCache string `toml:"virtio_fs_cache"` + VhostUserStorePath string `toml:"vhost_user_store_path"` + FileBackedMemRootDir string `toml:"file_mem_backend"` + GuestHookPath string `toml:"guest_hook_path"` + GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` + SeccompSandbox string `toml:"seccompsandbox"` + GuestPreAttestationURI string `toml:"guest_pre_attestation_kbs_uri"` + GuestPreAttestationMode string `toml:"guest_pre_attestation_kbs_mode"` + GuestPreAttestationKeyset string `toml:"guest_pre_attestation_keyset"` + SEVCertChainPath string `toml:"sev_cert_chain"` + BlockDeviceAIO string `toml:"block_device_aio"` + RemoteHypervisorSocket string `toml:"remote_hypervisor_socket"` + HypervisorPathList []string `toml:"valid_hypervisor_paths"` + JailerPathList []string `toml:"valid_jailer_paths"` + CtlPathList []string `toml:"valid_ctlpaths"` + VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` + VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` + PFlashList []string `toml:"pflashes"` + VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` + FileBackedMemRootList []string `toml:"valid_file_mem_backends"` + EntropySourceList []string `toml:"valid_entropy_sources"` + EnableAnnotations []string `toml:"enable_annotations"` + RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` + TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` + MemOffset uint64 `toml:"memory_offset"` + DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` + DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` + DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` + DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` + DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` + NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` + NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` + NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` + NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` + VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` + VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` + DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` + MemorySize uint32 `toml:"default_memory"` + MemSlots uint32 `toml:"memory_slots"` + DefaultBridges uint32 `toml:"default_bridges"` + Msize9p uint32 `toml:"msize_9p"` + PCIeRootPort uint32 `toml:"pcie_root_port"` + GuestPreAttestationGRPCTimeout uint32 `toml:"guest_pre_attestation_grpc_timeout"` + SEVGuestPolicy uint32 `toml:"sev_guest_policy"` + SNPGuestPolicy uint64 `toml:"snp_guest_policy"` + RemoteHypervisorTimeout uint32 `toml:"remote_hypervisor_timeout"` + NumVCPUs int32 `toml:"default_vcpus"` + BlockDeviceCacheSet bool `toml:"block_device_cache_set"` + BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` + BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` + EnableVhostUserStore bool `toml:"enable_vhost_user_store"` + VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` + DisableBlockDeviceUse bool `toml:"disable_block_device_use"` + MemPrealloc bool `toml:"enable_mem_prealloc"` + HugePages bool `toml:"enable_hugepages"` + VirtioMem bool `toml:"enable_virtio_mem"` + IOMMU bool `toml:"enable_iommu"` + IOMMUPlatform bool `toml:"enable_iommu_platform"` + Debug bool `toml:"enable_debug"` + DisableNestingChecks bool `toml:"disable_nesting_checks"` + EnableIOThreads bool `toml:"enable_iothreads"` + DisableImageNvdimm bool `toml:"disable_image_nvdimm"` + HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` + ColdPlugVFIO hv.PCIePort `toml:"cold_plug_vfio"` + DisableVhostNet bool `toml:"disable_vhost_net"` + GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` + ConfidentialGuest bool `toml:"confidential_guest"` + SevSnpGuest bool `toml:"sev_snp_guest"` + GuestSwap bool `toml:"enable_guest_swap"` + Rootless bool `toml:"rootless"` + DisableSeccomp bool `toml:"disable_seccomp"` + DisableSeLinux bool `toml:"disable_selinux"` + DisableGuestSeLinux bool `toml:"disable_guest_selinux"` + LegacySerial bool `toml:"use_legacy_serial"` + GuestPreAttestation bool `toml:"guest_pre_attestation"` } type runtime struct { @@ -296,6 +298,13 @@ func (h hypervisor) firmware() (string, error) { return ResolvePath(p) } +func (h hypervisor) coldPlugVFIO() hv.PCIePort { + if h.ColdPlugVFIO == "" { + return defaultColdPlugVFIO + } + return h.ColdPlugVFIO +} + func (h hypervisor) firmwareVolume() (string, error) { p := h.FirmwareVolume @@ -871,6 +880,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { Msize9p: h.msize9p(), DisableImageNvdimm: h.DisableImageNvdimm, HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, + ColdPlugVFIO: h.coldPlugVFIO(), PCIeRootPort: h.PCIeRootPort, DisableVhostNet: h.DisableVhostNet, EnableVhostUserStore: h.EnableVhostUserStore, @@ -1073,6 +1083,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { EnableIOThreads: h.EnableIOThreads, Msize9p: h.msize9p(), HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, + ColdPlugVFIO: h.coldPlugVFIO(), PCIeRootPort: h.PCIeRootPort, DisableVhostNet: true, GuestHookPath: h.guestHookPath(), @@ -1317,6 +1328,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { EnableIOThreads: defaultEnableIOThreads, Msize9p: defaultMsize9p, HotplugVFIOOnRootBus: defaultHotplugVFIOOnRootBus, + ColdPlugVFIO: defaultColdPlugVFIO, PCIeRootPort: defaultPCIeRootPort, GuestHookPath: defaultGuestHookPath, VhostUserStorePath: defaultVhostUserStorePath, @@ -1698,9 +1710,32 @@ func checkConfig(config oci.RuntimeConfig) error { return err } + coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO + machineType := config.HypervisorConfig.HypervisorMachineType + if err := checkPCIeConfig(coldPlugVFIO, machineType); err != nil { + return err + } + return nil } +// checkPCIeConfig ensures the PCIe configuration is valid. +// Only allow one of the following settings for cold-plug: +// no-port, root-port, switch-port +func checkPCIeConfig(vfioPort hv.PCIePort, machineType string) error { + // Currently only QEMU q35 supports advanced PCIe topologies + // firecracker, dragonball do not have right now any PCIe support + if machineType != "q35" { + return nil + } + if vfioPort == hv.NoPort || vfioPort == hv.RootPort || vfioPort == hv.SwitchPort { + return nil + } + + return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s", + vfioPort, hv.NoPort, hv.RootPort, hv.SwitchPort) +} + // checkNetNsConfig performs sanity checks on disable_new_netns config. // Because it is an expert option and conflicts with some other common configs. func checkNetNsConfig(config oci.RuntimeConfig) error { diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index d9557afbe..97f134579 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -70,7 +71,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf if hypervisor == "" { return config, fmt.Errorf("BUG: need hypervisor") } - + var coldPlugVFIO hv.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") kernelParams := "foo=bar xyz" @@ -85,6 +86,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) + coldPlugVFIO = hv.RootPort disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") @@ -107,6 +109,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, + ColdPlugVFIO: coldPlugVFIO, DisableNewNetNs: disableNewNetNs, DefaultVCPUCount: defaultVCPUCount, DefaultMaxVCPUCount: defaultMaxVCPUCount, @@ -170,6 +173,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, + ColdPlugVFIO: coldPlugVFIO, Msize9p: defaultMsize9p, MemSlots: defaultMemSlots, EntropySource: defaultEntropySource, @@ -566,6 +570,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { BlockDeviceAIO: defaultBlockDeviceAIO, DisableGuestSeLinux: defaultDisableGuestSeLinux, SNPGuestPolicy: defaultSNPGuestPolicy, + ColdPlugVFIO: defaultColdPlugVFIO, } expectedAgentConfig := vc.KataAgentConfig{ @@ -599,7 +604,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { func TestNewQemuHypervisorConfig(t *testing.T) { dir := t.TempDir() - + var coldPlugVFIO hv.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") @@ -608,6 +613,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) { enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) + coldPlugVFIO = hv.RootPort orgVHostVSockDevicePath := utils.VHostVSockDevicePath blockDeviceAIO := "io_uring" defer func() { @@ -627,6 +633,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) { EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, + ColdPlugVFIO: coldPlugVFIO, RxRateLimiterMaxRate: rxRateLimiterMaxRate, TxRateLimiterMaxRate: txRateLimiterMaxRate, SharedFS: "virtio-fs", diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index 61197f37a..b443a3f88 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -993,6 +993,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st DisableGuestSeccomp: runtime.DisableGuestSeccomp, + EnableVCPUsPinning: runtime.EnableVCPUsPinning, + GuestSeLinuxLabel: runtime.GuestSeLinuxLabel, Experimental: runtime.Experimental, diff --git a/src/runtime/pkg/resourcecontrol/utils_linux.go b/src/runtime/pkg/resourcecontrol/utils_linux.go index 59ca788f8..0acbc6c6a 100644 --- a/src/runtime/pkg/resourcecontrol/utils_linux.go +++ b/src/runtime/pkg/resourcecontrol/utils_linux.go @@ -156,7 +156,7 @@ func IsCgroupV1() (bool, error) { func SetThreadAffinity(threadID int, cpuSetSlice []int) error { unixCPUSet := unix.CPUSet{} - for cpuId := range cpuSetSlice { + for _, cpuId := range cpuSetSlice { unixCPUSet.Set(cpuId) } diff --git a/src/runtime/pkg/signals/signals_test.go b/src/runtime/pkg/signals/signals_test.go index fab557829..f13e02b04 100644 --- a/src/runtime/pkg/signals/signals_test.go +++ b/src/runtime/pkg/signals/signals_test.go @@ -7,7 +7,9 @@ package signals import ( "bytes" + "errors" "os" + "os/exec" "reflect" goruntime "runtime" "sort" @@ -135,3 +137,69 @@ func TestSignalBacktrace(t *testing.T) { assert.True(strings.Contains(b, "contention:")) assert.True(strings.Contains(b, `level=error`)) } + +func TestSignalHandlePanic(t *testing.T) { + assert := assert.New(t) + + savedLog := signalLog + defer func() { + signalLog = savedLog + }() + + signalLog = logrus.WithFields(logrus.Fields{ + "name": "name", + "pid": os.Getpid(), + "source": "throttler", + "test-logger": true}) + + // Create buffer to save logger output. + buf := &bytes.Buffer{} + + savedOut := signalLog.Logger.Out + defer func() { + signalLog.Logger.Out = savedOut + }() + + // Capture output to buffer. + signalLog.Logger.Out = buf + + HandlePanic(nil) + + b := buf.String() + assert.True(len(b) == 0) +} + +func TestSignalHandlePanicWithError(t *testing.T) { + assert := assert.New(t) + + if os.Getenv("CALL_EXIT") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestSignalHandlePanicWithError") + cmd.Env = append(os.Environ(), "CALL_EXIT=1") + + err := cmd.Run() + assert.True(err != nil) + + exitError, ok := err.(*exec.ExitError) + assert.True(ok) + assert.True(exitError.ExitCode() == 1) + + return + } + + signalLog = logrus.WithFields(logrus.Fields{ + "name": "name", + "pid": os.Getpid(), + "source": "throttler", + "test-logger": true}) + + // Create buffer to save logger output. + buf := &bytes.Buffer{} + + // Capture output to buffer. + signalLog.Logger.Out = buf + + dieCallBack := func() {} + defer HandlePanic(dieCallBack) + e := errors.New("test-panic") + panic(e) +} diff --git a/src/runtime/virtcontainers/README.md b/src/runtime/virtcontainers/README.md index c9556829d..cbf8da778 100644 --- a/src/runtime/virtcontainers/README.md +++ b/src/runtime/virtcontainers/README.md @@ -77,7 +77,7 @@ For further details, see the [API documentation](documentation/api/1.0/api.md). Typically the former is the Docker default networking model while the later is used on Kubernetes deployments. -[cnm]: https://github.com/docker/libnetwork/blob/master/docs/design.md +[cnm]: https://github.com/moby/libnetwork/blob/master/docs/design.md [cni]: https://github.com/containernetworking/cni/ ## CNM diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index 6355f791f..43ea782c2 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -139,6 +139,9 @@ type agent interface { // resumeContainer will resume a paused container resumeContainer(ctx context.Context, sandbox *Sandbox, c Container) error + // removeStaleVirtiofsShareMounts will tell the agent to remove stale virtiofs share mounts in the guest. + removeStaleVirtiofsShareMounts(ctx context.Context) error + // configure will update agent settings based on provided arguments configure(ctx context.Context, h Hypervisor, id, sharePath string, config KataAgentConfig) error diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index a1c3c4b64..b5c800e95 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -9,6 +9,7 @@ package virtcontainers import ( "context" + "fmt" "net/http" "os" "path/filepath" @@ -422,6 +423,7 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { assert.Error(err, "persist.GetDriver() expected error") clh.id = "cleanVMID" + clh.config.VMid = "cleanVMID" err = clh.cleanupVM(true) assert.NoError(err, "persist.GetDriver() unexpected error") @@ -438,77 +440,86 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { assert.True(os.IsNotExist(err), "persist.GetDriver() unexpected error") } -func TestClhCreateVMWithInitrd(t *testing.T) { - assert := assert.New(t) - - clhConfig, err := newClhConfig() - assert.NoError(err) - clhConfig.ImagePath = "" - clhConfig.InitrdPath = testClhInitrdPath - - store, err := persist.GetDriver() - assert.NoError(err) - - clhConfig.VMStorePath = store.RunVMStoragePath() - clhConfig.RunStorePath = store.RunStoragePath() - - network, err := NewNetwork() - assert.NoError(err) - - clh := &cloudHypervisor{ - config: clhConfig, - } - - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: clhConfig, - }, - } - - err = clh.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig) - assert.NoError(err) - assert.Exactly(clhConfig, clh.config) -} - func TestClhCreateVM(t *testing.T) { assert := assert.New(t) - clhConfig, err := newClhConfig() - assert.NoError(err) - assert.NotEmpty(clhConfig.ImagePath) - store, err := persist.GetDriver() assert.NoError(err) - clhConfig.VMStorePath = store.RunVMStoragePath() - clhConfig.RunStorePath = store.RunStoragePath() - network, err := NewNetwork() assert.NoError(err) clh := &cloudHypervisor{ - config: clhConfig, - } - - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: clhConfig, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), }, } - err = clh.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig) + config0, err := newClhConfig() assert.NoError(err) - assert.Exactly(clhConfig, clh.config) + + config1, err := newClhConfig() + assert.NoError(err) + config1.ImagePath = "" + config1.InitrdPath = testClhInitrdPath + + config2, err := newClhConfig() + assert.NoError(err) + config2.Debug = true + + config3, err := newClhConfig() + assert.NoError(err) + config3.Debug = true + config3.ConfidentialGuest = true + + config4, err := newClhConfig() + assert.NoError(err) + config4.SGXEPCSize = 1 + + config5, err := newClhConfig() + assert.NoError(err) + config5.SharedFS = config.VirtioFSNydus + + type testData struct { + config HypervisorConfig + expectError bool + configMatch bool + } + + data := []testData{ + {config0, false, true}, + {config1, false, true}, + {config2, false, true}, + {config3, true, false}, + {config4, false, true}, + {config5, false, true}, + } + + for i, d := range data { + msg := fmt.Sprintf("test[%d]", i) + + err = clh.CreateVM(context.Background(), "testSandbox", network, &d.config) + + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + + if d.configMatch { + assert.Exactly(d.config, clh.config, msg) + } + } } func TestCloudHypervisorStartSandbox(t *testing.T) { assert := assert.New(t) clhConfig, err := newClhConfig() assert.NoError(err) + clhConfig.Debug = true + clhConfig.DisableSeccomp = true store, err := persist.GetDriver() assert.NoError(err) @@ -530,6 +541,44 @@ func TestCloudHypervisorStartSandbox(t *testing.T) { err = clh.StartVM(context.Background(), 10) assert.NoError(err) + + _, err = clh.loadVirtiofsDaemon("/tmp/xyzabc") + assert.NoError(err) + + err = clh.stopVirtiofsDaemon(context.Background()) + assert.NoError(err) + + _, _, err = clh.GetVMConsole(context.Background(), "test") + assert.NoError(err) + + _, err = clh.GetThreadIDs(context.Background()) + assert.NoError(err) + + assert.True(clh.getClhStopSandboxTimeout().Nanoseconds() != 0) + + pid := clh.GetPids() + assert.True(pid[0] != 0) + + pid2 := *clh.GetVirtioFsPid() + assert.True(pid2 == 0) + + mem := clh.GetTotalMemoryMB(context.Background()) + assert.True(mem == 0) + + err = clh.PauseVM(context.Background()) + assert.NoError(err) + + err = clh.SaveVM() + assert.NoError(err) + + err = clh.ResumeVM(context.Background()) + assert.NoError(err) + + err = clh.Check() + assert.NoError(err) + + err = clh.Cleanup(context.Background()) + assert.NoError(err) } func TestCloudHypervisorResizeMemory(t *testing.T) { diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 7aaacf6fc..55eb5f3a0 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -1026,6 +1026,10 @@ func (c *Container) stop(ctx context.Context, force bool) error { } } + if err := c.sandbox.agent.removeStaleVirtiofsShareMounts(ctx); err != nil && !force { + return err + } + if err := c.detachDevices(ctx); err != nil && !force { return err } diff --git a/src/runtime/virtcontainers/container_linux_test.go b/src/runtime/virtcontainers/container_linux_test.go index 581461e61..7666e451d 100644 --- a/src/runtime/virtcontainers/container_linux_test.go +++ b/src/runtime/virtcontainers/container_linux_test.go @@ -125,6 +125,7 @@ func TestUnmountHostMountsRemoveBindHostPath(t *testing.T) { ctx: context.Background(), id: "foobar", config: &SandboxConfig{}, + agent: newMockAgent(), } fsShare, err := NewFilesystemShare(sandbox) diff --git a/src/runtime/virtcontainers/container_test.go b/src/runtime/virtcontainers/container_test.go index ac2ce540c..6a04ec1d4 100644 --- a/src/runtime/virtcontainers/container_test.go +++ b/src/runtime/virtcontainers/container_test.go @@ -302,6 +302,7 @@ func TestMountSharedDirMounts(t *testing.T) { sandbox := &Sandbox{ ctx: context.Background(), id: "foobar", + agent: newMockAgent(), hypervisor: &mockHypervisor{}, config: &SandboxConfig{ HypervisorConfig: HypervisorConfig{ diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index 9255a96ca..d3071a86f 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -292,6 +292,10 @@ type HypervisorConfig struct { // The PCIe Root Port device is used to hot-plug the PCIe device PCIeRootPort uint32 + // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the + // root port, switch or no port + ColdPlugVFIO hv.PCIePort + // BootToBeTemplate used to indicate if the VM is created to be a template VM BootToBeTemplate bool diff --git a/src/runtime/virtcontainers/factory/factory_test.go b/src/runtime/virtcontainers/factory/factory_test.go index 80cff101d..e7c40046b 100644 --- a/src/runtime/virtcontainers/factory/factory_test.go +++ b/src/runtime/virtcontainers/factory/factory_test.go @@ -8,6 +8,7 @@ package factory import ( "context" "os" + "strings" "testing" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -331,3 +332,126 @@ func TestDeepCompare(t *testing.T) { assert.Nil(err) assert.False(utils.DeepCompare(f1, f2)) } + +func TestFactoryConfig(t *testing.T) { + assert := assert.New(t) + + // Valid config + var config Config + config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + KernelPath: "foo", + ImagePath: "bar", + } + ctx := context.Background() + vf, err := NewFactory(ctx, config, false) + assert.Nil(err) + + f, ok := vf.(*factory) + assert.True(ok) + + vmc := f.Config() + + assert.Equal(config.VMConfig.HypervisorConfig.KernelPath, vmc.HypervisorConfig.KernelPath) + assert.Equal(config.VMConfig.HypervisorConfig.ImagePath, vmc.HypervisorConfig.ImagePath) +} + +func TestFactoryGetBaseVM(t *testing.T) { + assert := assert.New(t) + + // Set configs + var config Config + testDir := t.TempDir() + + hyperConfig := vc.HypervisorConfig{ + KernelPath: testDir, + ImagePath: testDir, + } + vmConfig := vc.VMConfig{ + HypervisorType: vc.MockHypervisor, + HypervisorConfig: hyperConfig, + } + config.VMConfig = vmConfig + config.TemplatePath = testDir + + err := vmConfig.Valid() + assert.Nil(err) + + ctx := context.Background() + + url, err := mock.GenerateKataMockHybridVSock() + assert.NoError(err) + defer mock.RemoveKataMockHybridVSock(url) + vc.MockHybridVSockPath = url + + hybridVSockTTRPCMock := mock.HybridVSockTTRPCMock{} + err = hybridVSockTTRPCMock.Start(url) + assert.NoError(err) + defer hybridVSockTTRPCMock.Stop() + + // New factory + vf, err := NewFactory(ctx, config, false) + assert.Nil(err) + + f, ok := vf.(*factory) + assert.True(ok) + + // Check VM Config + assert.Equal(f.Config(), vmConfig) + + // GetBaseVM + vm, err := f.GetBaseVM(ctx, vmConfig) + assert.Nil(err) + + // Get VM Status + defer func() { + r := recover() + assert.NotNil(r) + + // Close + err = vm.Stop(ctx) + assert.Nil(err) + }() + vmStatus := f.GetVMStatus() + assert.NotNil(vmStatus) // line of code to make golang happy. This is never executed. +} + +func TestNewFactoryWithCache(t *testing.T) { + assert := assert.New(t) + + // Config + var config Config + config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + KernelPath: "foo", + ImagePath: "bar", + } + ctx := context.Background() + + // cache>0 and fetch only should throw error + config.Cache = 1 + vf, err := NewFactory(ctx, config, true) + + assert.Nil(vf) + assert.Error(err) + b := err.Error() + assert.True(strings.Contains(b, "cache factory does not support fetch")) +} + +func TestNewFactoryWrongCacheEndpoint(t *testing.T) { + assert := assert.New(t) + + // Config + var config Config + config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ + KernelPath: "foo", + ImagePath: "bar", + } + ctx := context.Background() + + config.VMCache = true + vf, err := NewFactory(ctx, config, false) + + assert.Nil(vf) + assert.Error(err) + b := err.Error() + assert.True(strings.Contains(b, "rpc error")) // sanity check +} diff --git a/src/runtime/virtcontainers/fc_test.go b/src/runtime/virtcontainers/fc_test.go index 64bde2232..5550b6895 100644 --- a/src/runtime/virtcontainers/fc_test.go +++ b/src/runtime/virtcontainers/fc_test.go @@ -8,6 +8,7 @@ package virtcontainers import ( + "context" "strings" "testing" @@ -54,11 +55,184 @@ func TestFCParseVersion(t *testing.T) { fc := firecracker{} + // correct versions for rawVersion, v := range map[string]string{"Firecracker v0.23.1": "0.23.1", "Firecracker v0.25.0\nSupported snapshot data format versions: 0.23.0": "0.25.0"} { parsedVersion, err := fc.parseVersion(rawVersion) assert.NoError(err) assert.Equal(parsedVersion, v) } + + // wrong version str + rawVersion := "Firecracker_v0.23.0" + parsedVersion, err := fc.parseVersion(rawVersion) + assert.Error(err) + assert.Equal(parsedVersion, "") +} + +func TestFCCheckVersion(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + + // correct version + v := "0.23.0" + err := fc.checkVersion(v) + assert.NoError(err) + + // version too low + v = "0.1.1" + err = fc.checkVersion(v) + assert.Error(err) + b := err.Error() + assert.True(strings.Contains(b, "version 0.1.1 is not supported")) // sanity + + // version is malformed + v = "Firecracker v0.23.0" + err = fc.checkVersion(v) + assert.Error(err) + b = err.Error() + assert.True(strings.Contains(b, "Malformed firecracker version:")) // sanity +} + +func TestFCGetVersionNumber(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + _, err := fc.getVersionNumber() + assert.Error(err) +} + +func TestFCDriveIndexToID(t *testing.T) { + assert := assert.New(t) + + d := fcDriveIndexToID(5) + assert.Equal(d, "drive_5") +} + +func TestFCPauseVM(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + err := fc.PauseVM(ctx) + assert.NoError(err) +} + +func TestFCSaveVM(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + err := fc.SaveVM() + assert.NoError(err) +} + +func TestFCResumeVM(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + err := fc.ResumeVM(ctx) + assert.NoError(err) +} + +func TestFCGetVirtioFsPid(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + pid := fc.GetVirtioFsPid() + assert.Nil(pid) +} + +func TestFCIsRateLimiterBuiltin(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + rl := fc.IsRateLimiterBuiltin() + assert.True(rl) +} + +func TestFCCheck(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + err := fc.Check() + assert.NoError(err) +} + +func TestFCGetPids(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + pids := fc.GetPids() + assert.Equal(len(pids), 1) +} + +func TestFCCleanup(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + err := fc.Cleanup(ctx) + assert.NoError(err) +} + +func TestFCToGrpc(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + _, err := fc.toGrpc(ctx) + assert.Error(err) +} + +func TestFCHypervisorConfig(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + config := fc.HypervisorConfig() + assert.Equal(fc.config, config) +} + +func TestFCGetTotalMemoryMB(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + + var initialMemSize uint32 = 1024 + + fc.config.MemorySize = 1024 + memSize := fc.GetTotalMemoryMB(ctx) + assert.Equal(memSize, initialMemSize) +} + +func TestFCClient(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + + conn := fc.client(ctx) + assert.Equal(conn, fc.connection) +} + +func TestFCVmRunning(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + ctx := context.Background() + sr := fc.vmRunning(ctx) + assert.False(sr) +} + +func TestFCCreateJailedDrive(t *testing.T) { + assert := assert.New(t) + + fc := firecracker{} + + driveID := fcDriveIndexToID(0) + _, err := fc.createJailedDrive(driveID) + assert.NoError(err) } func TestFcSetConfig(t *testing.T) { diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index e6055e352..fe0b96fac 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -420,6 +420,7 @@ type HypervisorConfig struct { DisableSeLinux bool DisableGuestSeLinux bool LegacySerial bool + ColdPlugVFIO hv.PCIePort } // vcpu mapping from vcpu number to thread number diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 251af379d..b54ad9fe8 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -118,42 +118,43 @@ var ( ) const ( - grpcCheckRequest = "grpc.CheckRequest" - grpcExecProcessRequest = "grpc.ExecProcessRequest" - grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" - grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" - grpcCreateContainerRequest = "grpc.CreateContainerRequest" - grpcStartContainerRequest = "grpc.StartContainerRequest" - grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" - grpcSignalProcessRequest = "grpc.SignalProcessRequest" - grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" - grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" - grpcListInterfacesRequest = "grpc.ListInterfacesRequest" - grpcListRoutesRequest = "grpc.ListRoutesRequest" - grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" - grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" - grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" - grpcUpdateEphemeralMountsRequest = "grpc.UpdateEphemeralMountsRequest" - grpcWaitProcessRequest = "grpc.WaitProcessRequest" - grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" - grpcWriteStreamRequest = "grpc.WriteStreamRequest" - grpcCloseStdinRequest = "grpc.CloseStdinRequest" - grpcStatsContainerRequest = "grpc.StatsContainerRequest" - grpcPauseContainerRequest = "grpc.PauseContainerRequest" - grpcResumeContainerRequest = "grpc.ResumeContainerRequest" - grpcPullImageRequest = "grpc.PullImageRequest" - grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" - grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" - grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" - grpcCopyFileRequest = "grpc.CopyFileRequest" - grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" - grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" - grpcGetMetricsRequest = "grpc.GetMetricsRequest" - grpcAddSwapRequest = "grpc.AddSwapRequest" - grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" - grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" - grpcGetIPTablesRequest = "grpc.GetIPTablesRequest" - grpcSetIPTablesRequest = "grpc.SetIPTablesRequest" + grpcCheckRequest = "grpc.CheckRequest" + grpcExecProcessRequest = "grpc.ExecProcessRequest" + grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" + grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" + grpcCreateContainerRequest = "grpc.CreateContainerRequest" + grpcStartContainerRequest = "grpc.StartContainerRequest" + grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" + grpcSignalProcessRequest = "grpc.SignalProcessRequest" + grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" + grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" + grpcUpdateEphemeralMountsRequest = "grpc.UpdateEphemeralMountsRequest" + grpcRemoveStaleVirtiofsShareMountsRequest = "grpc.RemoveStaleVirtiofsShareMountsRequest" + grpcListInterfacesRequest = "grpc.ListInterfacesRequest" + grpcListRoutesRequest = "grpc.ListRoutesRequest" + grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" + grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" + grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" + grpcWaitProcessRequest = "grpc.WaitProcessRequest" + grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" + grpcWriteStreamRequest = "grpc.WriteStreamRequest" + grpcCloseStdinRequest = "grpc.CloseStdinRequest" + grpcStatsContainerRequest = "grpc.StatsContainerRequest" + grpcPauseContainerRequest = "grpc.PauseContainerRequest" + grpcResumeContainerRequest = "grpc.ResumeContainerRequest" + grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" + grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" + grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" + grpcCopyFileRequest = "grpc.CopyFileRequest" + grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" + grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" + grpcGetMetricsRequest = "grpc.GetMetricsRequest" + grpcAddSwapRequest = "grpc.AddSwapRequest" + grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" + grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" + grpcGetIPTablesRequest = "grpc.GetIPTablesRequest" + grpcSetIPTablesRequest = "grpc.SetIPTablesRequest" + grpcPullImageRequest = "grpc.PullImageRequest" ) // newKataAgent returns an agent from an agent type. @@ -1959,6 +1960,11 @@ func (k *kataAgent) reseedRNG(ctx context.Context, data []byte) error { return err } +func (k *kataAgent) removeStaleVirtiofsShareMounts(ctx context.Context) error { + _, err := k.sendReq(ctx, &grpc.RemoveStaleVirtiofsShareMountsRequest{}) + return err +} + type reqFunc func(context.Context, interface{}) (interface{}, error) func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { @@ -2071,6 +2077,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcSetIPTablesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.SetIPTables(ctx, req.(*grpc.SetIPTablesRequest)) } + k.reqHandlers[grpcRemoveStaleVirtiofsShareMountsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.RemoveStaleVirtiofsShareMounts(ctx, req.(*grpc.RemoveStaleVirtiofsShareMountsRequest)) + } } func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) { diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index b2564e5a8..c7fa059dc 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -846,6 +846,7 @@ func TestAgentCreateContainer(t *testing.T) { }, }, hypervisor: &mockHypervisor{}, + agent: newMockAgent(), } fsShare, err := NewFilesystemShare(sandbox) diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index b71a52f7e..ac9f7503e 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -142,6 +142,11 @@ func (n *mockAgent) waitProcess(ctx context.Context, c *Container, processID str return 0, nil } +// removeStaleVirtiofsShareMounts is the Noop agent removeStaleVirtiofsShareMounts implementation. It does nothing. +func (n *mockAgent) removeStaleVirtiofsShareMounts(ctx context.Context) error { + return nil +} + // winsizeProcess is the Noop agent process tty resizer. It does nothing. func (n *mockAgent) winsizeProcess(ctx context.Context, c *Container, processID string, height, width uint32) error { return nil diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 18c83e251..cbba44e60 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -487,6 +487,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { DisableNestingChecks: hconf.DisableNestingChecks, DisableImageNvdimm: hconf.DisableImageNvdimm, HotplugVFIOOnRootBus: hconf.HotplugVFIOOnRootBus, + ColdPlugVFIO: hconf.ColdPlugVFIO, PCIeRootPort: hconf.PCIeRootPort, BootToBeTemplate: hconf.BootToBeTemplate, BootFromTemplate: hconf.BootFromTemplate, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 5bef01219..e4facc6b9 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -7,6 +7,7 @@ package persistapi import ( + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/opencontainers/runc/libcontainer/configs" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -198,6 +199,10 @@ type HypervisorConfig struct { // root bus instead of a bridge. HotplugVFIOOnRootBus bool + // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the + // root port or a switch or no-port + ColdPlugVFIO hv.PCIePort + // BootToBeTemplate used to indicate if the VM is created to be a template VM BootToBeTemplate bool diff --git a/src/runtime/virtcontainers/persist/manager_test.go b/src/runtime/virtcontainers/persist/manager_test.go index 4347f9adc..ad7813792 100644 --- a/src/runtime/virtcontainers/persist/manager_test.go +++ b/src/runtime/virtcontainers/persist/manager_test.go @@ -7,7 +7,9 @@ package persist import ( + "errors" "os" + "strings" "testing" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" @@ -20,14 +22,29 @@ func TestGetDriverByName(t *testing.T) { assert.NotNil(t, err) assert.Nil(t, nonexist) + // testing correct driver is returned fsDriver, err := GetDriverByName("fs") assert.Nil(t, err) assert.NotNil(t, fsDriver) + + // testing case when expErr is set + expErr = errors.New("TEST-ERROR") + defer func() { + expErr = nil + }() + + nonexist, err = GetDriverByName("fs") + assert.NotNil(t, err) + assert.Nil(t, nonexist) + + b := err.Error() + assert.True(t, strings.Contains(b, "TEST-ERROR")) } func TestGetDriver(t *testing.T) { assert := assert.New(t) + // testing correct driver is returned fsd, err := GetDriver() assert.NoError(err) @@ -39,5 +56,32 @@ func TestGetDriver(t *testing.T) { } assert.NoError(err) - assert.Equal(expectedFS, fsd) + assert.Equal(expectedFS, fsd) // driver should match correct one for UID + + // testing case when expErr is set + expErr = errors.New("TEST-ERROR") + nonexist, err := GetDriver() + assert.NotNil(err) + assert.Nil(nonexist) + expErr = nil + + // testing case when driver can't be found on supportedDrivers variable + supportedDriversBU := supportedDrivers + supportedDrivers = nil + fsd, err = GetDriver() + assert.Nil(fsd) + assert.NotNil(err) + b := err.Error() + assert.True(strings.Contains(b, "Could not find a FS driver")) + supportedDrivers = supportedDriversBU + + // testing case when mock driver is activated + fs.EnableMockTesting(t.TempDir()) + mock, err := GetDriver() + assert.NoError(err) + expectedFS, err = fs.MockFSInit(fs.MockStorageRootPath()) + assert.NoError(err) + assert.Equal(expectedFS, mock) + + fs.EnableMockTesting("") } diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index 36af23886..a8dd81104 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/agent.proto +// source: agent.proto package grpc @@ -50,7 +50,7 @@ type CreateContainerRequest struct { func (m *CreateContainerRequest) Reset() { *m = CreateContainerRequest{} } func (*CreateContainerRequest) ProtoMessage() {} func (*CreateContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{0} + return fileDescriptor_56ede974c0020f77, []int{0} } func (m *CreateContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -89,7 +89,7 @@ type StartContainerRequest struct { func (m *StartContainerRequest) Reset() { *m = StartContainerRequest{} } func (*StartContainerRequest) ProtoMessage() {} func (*StartContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{1} + return fileDescriptor_56ede974c0020f77, []int{1} } func (m *StartContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -134,7 +134,7 @@ type RemoveContainerRequest struct { func (m *RemoveContainerRequest) Reset() { *m = RemoveContainerRequest{} } func (*RemoveContainerRequest) ProtoMessage() {} func (*RemoveContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{2} + return fileDescriptor_56ede974c0020f77, []int{2} } func (m *RemoveContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -176,7 +176,7 @@ type ExecProcessRequest struct { func (m *ExecProcessRequest) Reset() { *m = ExecProcessRequest{} } func (*ExecProcessRequest) ProtoMessage() {} func (*ExecProcessRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{3} + return fileDescriptor_56ede974c0020f77, []int{3} } func (m *ExecProcessRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -220,7 +220,7 @@ type SignalProcessRequest struct { func (m *SignalProcessRequest) Reset() { *m = SignalProcessRequest{} } func (*SignalProcessRequest) ProtoMessage() {} func (*SignalProcessRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{4} + return fileDescriptor_56ede974c0020f77, []int{4} } func (m *SignalProcessRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -260,7 +260,7 @@ type WaitProcessRequest struct { func (m *WaitProcessRequest) Reset() { *m = WaitProcessRequest{} } func (*WaitProcessRequest) ProtoMessage() {} func (*WaitProcessRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{5} + return fileDescriptor_56ede974c0020f77, []int{5} } func (m *WaitProcessRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -299,7 +299,7 @@ type WaitProcessResponse struct { func (m *WaitProcessResponse) Reset() { *m = WaitProcessResponse{} } func (*WaitProcessResponse) ProtoMessage() {} func (*WaitProcessResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{6} + return fileDescriptor_56ede974c0020f77, []int{6} } func (m *WaitProcessResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -339,7 +339,7 @@ type UpdateContainerRequest struct { func (m *UpdateContainerRequest) Reset() { *m = UpdateContainerRequest{} } func (*UpdateContainerRequest) ProtoMessage() {} func (*UpdateContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{7} + return fileDescriptor_56ede974c0020f77, []int{7} } func (m *UpdateContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -378,7 +378,7 @@ type StatsContainerRequest struct { func (m *StatsContainerRequest) Reset() { *m = StatsContainerRequest{} } func (*StatsContainerRequest) ProtoMessage() {} func (*StatsContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{8} + return fileDescriptor_56ede974c0020f77, []int{8} } func (m *StatsContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -417,7 +417,7 @@ type PauseContainerRequest struct { func (m *PauseContainerRequest) Reset() { *m = PauseContainerRequest{} } func (*PauseContainerRequest) ProtoMessage() {} func (*PauseContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{9} + return fileDescriptor_56ede974c0020f77, []int{9} } func (m *PauseContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -456,7 +456,7 @@ type ResumeContainerRequest struct { func (m *ResumeContainerRequest) Reset() { *m = ResumeContainerRequest{} } func (*ResumeContainerRequest) ProtoMessage() {} func (*ResumeContainerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{10} + return fileDescriptor_56ede974c0020f77, []int{10} } func (m *ResumeContainerRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -498,7 +498,7 @@ type CpuUsage struct { func (m *CpuUsage) Reset() { *m = CpuUsage{} } func (*CpuUsage) ProtoMessage() {} func (*CpuUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{11} + return fileDescriptor_56ede974c0020f77, []int{11} } func (m *CpuUsage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -539,7 +539,7 @@ type ThrottlingData struct { func (m *ThrottlingData) Reset() { *m = ThrottlingData{} } func (*ThrottlingData) ProtoMessage() {} func (*ThrottlingData) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{12} + return fileDescriptor_56ede974c0020f77, []int{12} } func (m *ThrottlingData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -579,7 +579,7 @@ type CpuStats struct { func (m *CpuStats) Reset() { *m = CpuStats{} } func (*CpuStats) ProtoMessage() {} func (*CpuStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{13} + return fileDescriptor_56ede974c0020f77, []int{13} } func (m *CpuStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -619,7 +619,7 @@ type PidsStats struct { func (m *PidsStats) Reset() { *m = PidsStats{} } func (*PidsStats) ProtoMessage() {} func (*PidsStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{14} + return fileDescriptor_56ede974c0020f77, []int{14} } func (m *PidsStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -661,7 +661,7 @@ type MemoryData struct { func (m *MemoryData) Reset() { *m = MemoryData{} } func (*MemoryData) ProtoMessage() {} func (*MemoryData) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{15} + return fileDescriptor_56ede974c0020f77, []int{15} } func (m *MemoryData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -705,7 +705,7 @@ type MemoryStats struct { func (m *MemoryStats) Reset() { *m = MemoryStats{} } func (*MemoryStats) ProtoMessage() {} func (*MemoryStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{16} + return fileDescriptor_56ede974c0020f77, []int{16} } func (m *MemoryStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -747,7 +747,7 @@ type BlkioStatsEntry struct { func (m *BlkioStatsEntry) Reset() { *m = BlkioStatsEntry{} } func (*BlkioStatsEntry) ProtoMessage() {} func (*BlkioStatsEntry) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{17} + return fileDescriptor_56ede974c0020f77, []int{17} } func (m *BlkioStatsEntry) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -793,7 +793,7 @@ type BlkioStats struct { func (m *BlkioStats) Reset() { *m = BlkioStats{} } func (*BlkioStats) ProtoMessage() {} func (*BlkioStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{18} + return fileDescriptor_56ede974c0020f77, []int{18} } func (m *BlkioStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -834,7 +834,7 @@ type HugetlbStats struct { func (m *HugetlbStats) Reset() { *m = HugetlbStats{} } func (*HugetlbStats) ProtoMessage() {} func (*HugetlbStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{19} + return fileDescriptor_56ede974c0020f77, []int{19} } func (m *HugetlbStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -877,7 +877,7 @@ type CgroupStats struct { func (m *CgroupStats) Reset() { *m = CgroupStats{} } func (*CgroupStats) ProtoMessage() {} func (*CgroupStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{20} + return fileDescriptor_56ede974c0020f77, []int{20} } func (m *CgroupStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -924,7 +924,7 @@ type NetworkStats struct { func (m *NetworkStats) Reset() { *m = NetworkStats{} } func (*NetworkStats) ProtoMessage() {} func (*NetworkStats) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{21} + return fileDescriptor_56ede974c0020f77, []int{21} } func (m *NetworkStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -964,7 +964,7 @@ type StatsContainerResponse struct { func (m *StatsContainerResponse) Reset() { *m = StatsContainerResponse{} } func (*StatsContainerResponse) ProtoMessage() {} func (*StatsContainerResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{22} + return fileDescriptor_56ede974c0020f77, []int{22} } func (m *StatsContainerResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1005,7 +1005,7 @@ type WriteStreamRequest struct { func (m *WriteStreamRequest) Reset() { *m = WriteStreamRequest{} } func (*WriteStreamRequest) ProtoMessage() {} func (*WriteStreamRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{23} + return fileDescriptor_56ede974c0020f77, []int{23} } func (m *WriteStreamRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1044,7 +1044,7 @@ type WriteStreamResponse struct { func (m *WriteStreamResponse) Reset() { *m = WriteStreamResponse{} } func (*WriteStreamResponse) ProtoMessage() {} func (*WriteStreamResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{24} + return fileDescriptor_56ede974c0020f77, []int{24} } func (m *WriteStreamResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1085,7 +1085,7 @@ type ReadStreamRequest struct { func (m *ReadStreamRequest) Reset() { *m = ReadStreamRequest{} } func (*ReadStreamRequest) ProtoMessage() {} func (*ReadStreamRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{25} + return fileDescriptor_56ede974c0020f77, []int{25} } func (m *ReadStreamRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1124,7 +1124,7 @@ type ReadStreamResponse struct { func (m *ReadStreamResponse) Reset() { *m = ReadStreamResponse{} } func (*ReadStreamResponse) ProtoMessage() {} func (*ReadStreamResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{26} + return fileDescriptor_56ede974c0020f77, []int{26} } func (m *ReadStreamResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1164,7 +1164,7 @@ type CloseStdinRequest struct { func (m *CloseStdinRequest) Reset() { *m = CloseStdinRequest{} } func (*CloseStdinRequest) ProtoMessage() {} func (*CloseStdinRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{27} + return fileDescriptor_56ede974c0020f77, []int{27} } func (m *CloseStdinRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1206,7 +1206,7 @@ type TtyWinResizeRequest struct { func (m *TtyWinResizeRequest) Reset() { *m = TtyWinResizeRequest{} } func (*TtyWinResizeRequest) ProtoMessage() {} func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{28} + return fileDescriptor_56ede974c0020f77, []int{28} } func (m *TtyWinResizeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1249,7 +1249,7 @@ type KernelModule struct { func (m *KernelModule) Reset() { *m = KernelModule{} } func (*KernelModule) ProtoMessage() {} func (*KernelModule) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{29} + return fileDescriptor_56ede974c0020f77, []int{29} } func (m *KernelModule) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1304,7 +1304,7 @@ type CreateSandboxRequest struct { func (m *CreateSandboxRequest) Reset() { *m = CreateSandboxRequest{} } func (*CreateSandboxRequest) ProtoMessage() {} func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{30} + return fileDescriptor_56ede974c0020f77, []int{30} } func (m *CreateSandboxRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1342,7 +1342,7 @@ type DestroySandboxRequest struct { func (m *DestroySandboxRequest) Reset() { *m = DestroySandboxRequest{} } func (*DestroySandboxRequest) ProtoMessage() {} func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{31} + return fileDescriptor_56ede974c0020f77, []int{31} } func (m *DestroySandboxRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1371,6 +1371,44 @@ func (m *DestroySandboxRequest) XXX_DiscardUnknown() { var xxx_messageInfo_DestroySandboxRequest proto.InternalMessageInfo +type RemoveStaleVirtiofsShareMountsRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RemoveStaleVirtiofsShareMountsRequest) Reset() { *m = RemoveStaleVirtiofsShareMountsRequest{} } +func (*RemoveStaleVirtiofsShareMountsRequest) ProtoMessage() {} +func (*RemoveStaleVirtiofsShareMountsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{32} +} +func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest.Merge(m, src) +} +func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_Size() int { + return m.Size() +} +func (m *RemoveStaleVirtiofsShareMountsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_RemoveStaleVirtiofsShareMountsRequest proto.InternalMessageInfo + type Interfaces struct { Interfaces []*protocols.Interface `protobuf:"bytes,1,rep,name=Interfaces,proto3" json:"Interfaces,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1381,7 +1419,7 @@ type Interfaces struct { func (m *Interfaces) Reset() { *m = Interfaces{} } func (*Interfaces) ProtoMessage() {} func (*Interfaces) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{32} + return fileDescriptor_56ede974c0020f77, []int{33} } func (m *Interfaces) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1420,7 +1458,7 @@ type Routes struct { func (m *Routes) Reset() { *m = Routes{} } func (*Routes) ProtoMessage() {} func (*Routes) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{33} + return fileDescriptor_56ede974c0020f77, []int{34} } func (m *Routes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1459,7 +1497,7 @@ type UpdateInterfaceRequest struct { func (m *UpdateInterfaceRequest) Reset() { *m = UpdateInterfaceRequest{} } func (*UpdateInterfaceRequest) ProtoMessage() {} func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{34} + return fileDescriptor_56ede974c0020f77, []int{35} } func (m *UpdateInterfaceRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1498,7 +1536,7 @@ type UpdateRoutesRequest struct { func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } func (*UpdateRoutesRequest) ProtoMessage() {} func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{35} + return fileDescriptor_56ede974c0020f77, []int{36} } func (m *UpdateRoutesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1537,7 +1575,7 @@ type UpdateEphemeralMountsRequest struct { func (m *UpdateEphemeralMountsRequest) Reset() { *m = UpdateEphemeralMountsRequest{} } func (*UpdateEphemeralMountsRequest) ProtoMessage() {} func (*UpdateEphemeralMountsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{36} + return fileDescriptor_56ede974c0020f77, []int{37} } func (m *UpdateEphemeralMountsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1575,7 +1613,7 @@ type ListInterfacesRequest struct { func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } func (*ListInterfacesRequest) ProtoMessage() {} func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{37} + return fileDescriptor_56ede974c0020f77, []int{38} } func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1613,7 +1651,7 @@ type ListRoutesRequest struct { func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } func (*ListRoutesRequest) ProtoMessage() {} func (*ListRoutesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{38} + return fileDescriptor_56ede974c0020f77, []int{39} } func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1652,7 +1690,7 @@ type ARPNeighbors struct { func (m *ARPNeighbors) Reset() { *m = ARPNeighbors{} } func (*ARPNeighbors) ProtoMessage() {} func (*ARPNeighbors) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{39} + return fileDescriptor_56ede974c0020f77, []int{40} } func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1691,7 +1729,7 @@ type AddARPNeighborsRequest struct { func (m *AddARPNeighborsRequest) Reset() { *m = AddARPNeighborsRequest{} } func (*AddARPNeighborsRequest) ProtoMessage() {} func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{40} + return fileDescriptor_56ede974c0020f77, []int{41} } func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1730,7 +1768,7 @@ type GetIPTablesRequest struct { func (m *GetIPTablesRequest) Reset() { *m = GetIPTablesRequest{} } func (*GetIPTablesRequest) ProtoMessage() {} func (*GetIPTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{41} + return fileDescriptor_56ede974c0020f77, []int{42} } func (m *GetIPTablesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1770,7 +1808,7 @@ type GetIPTablesResponse struct { func (m *GetIPTablesResponse) Reset() { *m = GetIPTablesResponse{} } func (*GetIPTablesResponse) ProtoMessage() {} func (*GetIPTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{42} + return fileDescriptor_56ede974c0020f77, []int{43} } func (m *GetIPTablesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1812,7 +1850,7 @@ type SetIPTablesRequest struct { func (m *SetIPTablesRequest) Reset() { *m = SetIPTablesRequest{} } func (*SetIPTablesRequest) ProtoMessage() {} func (*SetIPTablesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{43} + return fileDescriptor_56ede974c0020f77, []int{44} } func (m *SetIPTablesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1852,7 +1890,7 @@ type SetIPTablesResponse struct { func (m *SetIPTablesResponse) Reset() { *m = SetIPTablesResponse{} } func (*SetIPTablesResponse) ProtoMessage() {} func (*SetIPTablesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{44} + return fileDescriptor_56ede974c0020f77, []int{45} } func (m *SetIPTablesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1898,7 +1936,7 @@ type OnlineCPUMemRequest struct { func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } func (*OnlineCPUMemRequest) ProtoMessage() {} func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{45} + return fileDescriptor_56ede974c0020f77, []int{46} } func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1938,7 +1976,7 @@ type ReseedRandomDevRequest struct { func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } func (*ReseedRandomDevRequest) ProtoMessage() {} func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{46} + return fileDescriptor_56ede974c0020f77, []int{47} } func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1988,7 +2026,7 @@ type AgentDetails struct { func (m *AgentDetails) Reset() { *m = AgentDetails{} } func (*AgentDetails) ProtoMessage() {} func (*AgentDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{47} + return fileDescriptor_56ede974c0020f77, []int{48} } func (m *AgentDetails) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2034,7 +2072,7 @@ type GuestDetailsRequest struct { func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } func (*GuestDetailsRequest) ProtoMessage() {} func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{48} + return fileDescriptor_56ede974c0020f77, []int{49} } func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2076,7 +2114,7 @@ type GuestDetailsResponse struct { func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } func (*GuestDetailsResponse) ProtoMessage() {} func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{49} + return fileDescriptor_56ede974c0020f77, []int{50} } func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2117,7 +2155,7 @@ type MemHotplugByProbeRequest struct { func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } func (*MemHotplugByProbeRequest) ProtoMessage() {} func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{50} + return fileDescriptor_56ede974c0020f77, []int{51} } func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2159,7 +2197,7 @@ type SetGuestDateTimeRequest struct { func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } func (*SetGuestDateTimeRequest) ProtoMessage() {} func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{51} + return fileDescriptor_56ede974c0020f77, []int{52} } func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2205,7 +2243,7 @@ type FSGroup struct { func (m *FSGroup) Reset() { *m = FSGroup{} } func (*FSGroup) ProtoMessage() {} func (*FSGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{52} + return fileDescriptor_56ede974c0020f77, []int{53} } func (m *FSGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2273,7 +2311,7 @@ type Storage struct { func (m *Storage) Reset() { *m = Storage{} } func (*Storage) ProtoMessage() {} func (*Storage) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{53} + return fileDescriptor_56ede974c0020f77, []int{54} } func (m *Storage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2344,7 +2382,7 @@ type Device struct { func (m *Device) Reset() { *m = Device{} } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{54} + return fileDescriptor_56ede974c0020f77, []int{55} } func (m *Device) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2385,7 +2423,7 @@ type StringUser struct { func (m *StringUser) Reset() { *m = StringUser{} } func (*StringUser) ProtoMessage() {} func (*StringUser) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{55} + return fileDescriptor_56ede974c0020f77, []int{56} } func (m *StringUser) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2442,7 +2480,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (*CopyFileRequest) ProtoMessage() {} func (*CopyFileRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{56} + return fileDescriptor_56ede974c0020f77, []int{57} } func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2480,7 +2518,7 @@ type GetOOMEventRequest struct { func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } func (*GetOOMEventRequest) ProtoMessage() {} func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{57} + return fileDescriptor_56ede974c0020f77, []int{58} } func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2519,7 +2557,7 @@ type OOMEvent struct { func (m *OOMEvent) Reset() { *m = OOMEvent{} } func (*OOMEvent) ProtoMessage() {} func (*OOMEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{58} + return fileDescriptor_56ede974c0020f77, []int{59} } func (m *OOMEvent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2558,7 +2596,7 @@ type AddSwapRequest struct { func (m *AddSwapRequest) Reset() { *m = AddSwapRequest{} } func (*AddSwapRequest) ProtoMessage() {} func (*AddSwapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{59} + return fileDescriptor_56ede974c0020f77, []int{60} } func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2596,7 +2634,7 @@ type GetMetricsRequest struct { func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } func (*GetMetricsRequest) ProtoMessage() {} func (*GetMetricsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{60} + return fileDescriptor_56ede974c0020f77, []int{61} } func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2635,7 +2673,7 @@ type Metrics struct { func (m *Metrics) Reset() { *m = Metrics{} } func (*Metrics) ProtoMessage() {} func (*Metrics) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{61} + return fileDescriptor_56ede974c0020f77, []int{62} } func (m *Metrics) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2675,7 +2713,7 @@ type VolumeStatsRequest struct { func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } func (*VolumeStatsRequest) ProtoMessage() {} func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{62} + return fileDescriptor_56ede974c0020f77, []int{63} } func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2716,7 +2754,7 @@ type ResizeVolumeRequest struct { func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } func (*ResizeVolumeRequest) ProtoMessage() {} func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{63} + return fileDescriptor_56ede974c0020f77, []int{64} } func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2780,6 +2818,7 @@ func init() { proto.RegisterType((*KernelModule)(nil), "grpc.KernelModule") proto.RegisterType((*CreateSandboxRequest)(nil), "grpc.CreateSandboxRequest") proto.RegisterType((*DestroySandboxRequest)(nil), "grpc.DestroySandboxRequest") + proto.RegisterType((*RemoveStaleVirtiofsShareMountsRequest)(nil), "grpc.RemoveStaleVirtiofsShareMountsRequest") proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") proto.RegisterType((*Routes)(nil), "grpc.Routes") proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") @@ -2814,216 +2853,215 @@ func init() { proto.RegisterType((*ResizeVolumeRequest)(nil), "grpc.ResizeVolumeRequest") } -func init() { - proto.RegisterFile("github.com/kata-containers/kata-containers/src/libs/protocols/protos/agent.proto", fileDescriptor_712ce9a559fda969) -} +func init() { proto.RegisterFile("agent.proto", fileDescriptor_56ede974c0020f77) } -var fileDescriptor_712ce9a559fda969 = []byte{ - // 3249 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0xcb, 0x72, 0x1c, 0x47, - 0x72, 0x9a, 0x07, 0x30, 0x33, 0x39, 0x2f, 0x4c, 0x0f, 0x08, 0x0e, 0x47, 0x14, 0x44, 0x35, 0x25, - 0x0a, 0x94, 0x4c, 0x40, 0xa2, 0x14, 0xa2, 0x1e, 0x21, 0xd3, 0x00, 0x08, 0x01, 0x90, 0x04, 0x71, - 0xdc, 0x43, 0x58, 0x0e, 0x3b, 0xec, 0x8e, 0x46, 0x77, 0x61, 0xa6, 0x84, 0xe9, 0xae, 0x56, 0x75, - 0x35, 0x08, 0xc8, 0x11, 0x0e, 0x9f, 0xec, 0x9b, 0x8f, 0xbe, 0xe9, 0x07, 0x1c, 0xfe, 0x03, 0x5f, - 0x7d, 0x50, 0xf8, 0xb4, 0xc7, 0xbd, 0xec, 0xc6, 0x8a, 0x9f, 0xb0, 0x5f, 0xb0, 0x51, 0xaf, 0x7e, - 0xcc, 0x03, 0xd2, 0x22, 0x18, 0xb1, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x56, 0x56, 0x66, - 0xd5, 0xc0, 0x60, 0x84, 0xd9, 0x38, 0x3e, 0xd9, 0x74, 0x89, 0xbf, 0x75, 0xe6, 0x30, 0xe7, 0x81, - 0x4b, 0x02, 0xe6, 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x35, 0xc1, 0x27, 0xd1, 0x56, - 0x48, 0x09, 0x23, 0x2e, 0x99, 0xa8, 0xaf, 0x68, 0xcb, 0x19, 0xa1, 0x80, 0x6d, 0x0a, 0xc0, 0x28, - 0x8f, 0x68, 0xe8, 0xf6, 0x6b, 0xc4, 0xc5, 0x12, 0xd1, 0xaf, 0xb9, 0x91, 0xfe, 0xac, 0xb3, 0xcb, - 0x10, 0x45, 0x0a, 0x78, 0x75, 0x44, 0xc8, 0x68, 0x82, 0x24, 0x8f, 0x93, 0xf8, 0x74, 0x0b, 0xf9, - 0x21, 0xbb, 0x94, 0x83, 0xe6, 0x8f, 0x45, 0x58, 0xdb, 0xa5, 0xc8, 0x61, 0x68, 0x57, 0x2b, 0x60, - 0xa1, 0xef, 0x63, 0x14, 0x31, 0xe3, 0x0d, 0x68, 0x24, 0x4a, 0xd9, 0xd8, 0xeb, 0x15, 0xee, 0x14, - 0x36, 0x6a, 0x56, 0x3d, 0xc1, 0x1d, 0x7a, 0xc6, 0x4d, 0xa8, 0xa0, 0x0b, 0xe4, 0xf2, 0xd1, 0xa2, - 0x18, 0x5d, 0xe6, 0xe0, 0xa1, 0x67, 0xbc, 0x0f, 0xf5, 0x88, 0x51, 0x1c, 0x8c, 0xec, 0x38, 0x42, - 0xb4, 0x57, 0xba, 0x53, 0xd8, 0xa8, 0x3f, 0x5c, 0xd9, 0xe4, 0x2a, 0x6f, 0x0e, 0xc5, 0xc0, 0x71, - 0x84, 0xa8, 0x05, 0x51, 0xf2, 0x6d, 0xdc, 0x83, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x5e, 0xf9, - 0x4e, 0x69, 0xa3, 0xfe, 0xb0, 0x21, 0xc9, 0x9f, 0x08, 0xa4, 0xa5, 0x07, 0x8d, 0xfb, 0x50, 0x8d, - 0x18, 0xa1, 0xce, 0x08, 0x45, 0xbd, 0x25, 0x41, 0xd8, 0xd4, 0x7c, 0x05, 0xd6, 0x4a, 0x86, 0x8d, - 0xdb, 0x50, 0x7a, 0xba, 0x7b, 0xd8, 0x5b, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, - 0xb8, 0x0b, 0xcd, 0xc8, 0x09, 0xbc, 0x13, 0x72, 0x61, 0x87, 0xd8, 0x0b, 0xa2, 0x5e, 0xe5, 0x4e, - 0x61, 0xa3, 0x6a, 0x35, 0x14, 0x72, 0xc0, 0x71, 0xe6, 0xa7, 0x70, 0x63, 0xc8, 0x1c, 0xca, 0xae, - 0xe1, 0x1d, 0xf3, 0x18, 0xd6, 0x2c, 0xe4, 0x93, 0xf3, 0x6b, 0xb9, 0xb6, 0x07, 0x15, 0x86, 0x7d, - 0x44, 0x62, 0x26, 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0x4f, 0x01, 0x8c, 0xbd, 0x0b, 0xe4, 0x0e, - 0x28, 0x71, 0x51, 0x14, 0xfd, 0x85, 0x96, 0xeb, 0x6d, 0xa8, 0x84, 0x52, 0x81, 0x5e, 0x59, 0x90, - 0xab, 0x55, 0xd0, 0x5a, 0xe9, 0x51, 0xf3, 0x3b, 0x58, 0x1d, 0xe2, 0x51, 0xe0, 0x4c, 0x5e, 0xa2, - 0xbe, 0x6b, 0xb0, 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0x73, 0x00, 0xc6, 0xb7, 0x0e, - 0x66, 0x2f, 0x4f, 0x92, 0xf9, 0x00, 0xba, 0x39, 0x8e, 0x51, 0x48, 0x82, 0x08, 0x09, 0x05, 0x98, - 0xc3, 0xe2, 0x48, 0x30, 0x5b, 0xb2, 0x14, 0x64, 0x12, 0x58, 0x3b, 0x0e, 0xbd, 0x6b, 0xee, 0xa6, - 0x87, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x7b, 0xa0, 0x28, 0x9c, 0xba, 0x2a, 0x9d, 0xfa, 0x35, - 0x0e, 0xe2, 0x0b, 0x4b, 0x8f, 0x59, 0x29, 0x99, 0x8a, 0x4f, 0x16, 0x5d, 0x27, 0x3e, 0x3f, 0x85, - 0x1b, 0x03, 0x27, 0x8e, 0xae, 0xa3, 0xab, 0xf9, 0x19, 0x8f, 0xed, 0x28, 0xf6, 0xaf, 0x35, 0xf9, - 0xbf, 0x0b, 0x50, 0xdd, 0x0d, 0xe3, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0x75, 0xa8, 0x33, 0xc2, 0x9c, - 0x89, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, 0x49, 0xf0, 0x06, 0x34, 0x42, 0x44, 0xdd, - 0x30, 0x56, 0x14, 0xc5, 0x3b, 0xa5, 0x8d, 0xb2, 0x55, 0x97, 0x38, 0x49, 0xb2, 0x09, 0x5d, 0x31, - 0x66, 0xe3, 0xc0, 0x3e, 0x43, 0x34, 0x40, 0x13, 0x9f, 0x78, 0x48, 0x04, 0x47, 0xd9, 0xea, 0x88, - 0xa1, 0xc3, 0xe0, 0xab, 0x64, 0xc0, 0x78, 0x07, 0x3a, 0x09, 0x3d, 0x8f, 0x78, 0x41, 0x5d, 0x16, - 0xd4, 0x6d, 0x45, 0x7d, 0xac, 0xd0, 0xe6, 0xbf, 0x42, 0xeb, 0xd9, 0x98, 0x12, 0xc6, 0x26, 0x38, - 0x18, 0x3d, 0x71, 0x98, 0xc3, 0xb7, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, - 0x2e, 0x74, 0x98, 0xa4, 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x25, 0x19, 0x18, 0x28, 0xe2, - 0xb7, 0xa0, 0x95, 0x12, 0xf3, 0xcd, 0xad, 0xf4, 0x6d, 0x26, 0xd8, 0x67, 0xd8, 0x47, 0xe6, 0xb9, - 0xf0, 0x95, 0x58, 0x64, 0xe3, 0x5d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x08, 0x69, 0xc9, 0x08, 0xd1, - 0xee, 0xb4, 0xaa, 0x89, 0x53, 0x3e, 0x87, 0x36, 0x4b, 0x14, 0xb7, 0x3d, 0x87, 0x39, 0xf9, 0xa0, - 0xca, 0x5b, 0x65, 0xb5, 0x58, 0x0e, 0x36, 0x3f, 0x83, 0xda, 0x00, 0x7b, 0x91, 0x14, 0xdc, 0x83, - 0x8a, 0x1b, 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x0a, 0x4b, 0x13, 0xec, 0x63, 0xa6, - 0xcc, 0x94, 0x80, 0x49, 0x00, 0x8e, 0x90, 0x4f, 0xe8, 0xa5, 0x70, 0xd8, 0x2a, 0x2c, 0x65, 0x17, - 0x57, 0x02, 0xc6, 0xab, 0x50, 0xf3, 0x9d, 0x8b, 0x64, 0x51, 0xf9, 0x48, 0xd5, 0x77, 0x2e, 0xa4, - 0xf2, 0x3d, 0xa8, 0x9c, 0x3a, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x72, 0x56, - 0xe0, 0xff, 0x15, 0xa1, 0x2e, 0x25, 0x4a, 0x85, 0x57, 0x61, 0xc9, 0x75, 0xdc, 0x71, 0x22, 0x52, - 0x00, 0xc6, 0x3d, 0xad, 0x48, 0x31, 0x9b, 0xe1, 0x52, 0x4d, 0xb5, 0x6a, 0x5b, 0x00, 0xd1, 0x73, - 0x27, 0x54, 0xba, 0x95, 0x16, 0x10, 0xd7, 0x38, 0x8d, 0x54, 0xf7, 0x03, 0x68, 0xc8, 0xb8, 0x53, - 0x53, 0xca, 0x0b, 0xa6, 0xd4, 0x25, 0x95, 0x9c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x31, 0x46, - 0xd4, 0xa1, 0xee, 0xf8, 0xb2, 0xb7, 0x24, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0x3c, 0x84, - 0x25, 0x9e, 0x5b, 0xa2, 0xde, 0xb2, 0x38, 0xeb, 0x6e, 0x67, 0x59, 0x0a, 0x53, 0x37, 0xc5, 0xef, - 0x5e, 0xc0, 0xe8, 0xa5, 0x25, 0x49, 0xfb, 0x1f, 0x03, 0xa4, 0x48, 0x63, 0x05, 0x4a, 0x67, 0xe8, - 0x52, 0xed, 0x43, 0xfe, 0xc9, 0x9d, 0x73, 0xee, 0x4c, 0x62, 0xed, 0x75, 0x09, 0x7c, 0x5a, 0xfc, - 0xb8, 0x60, 0xba, 0xd0, 0xde, 0x99, 0x9c, 0x61, 0x92, 0x99, 0xbe, 0x0a, 0x4b, 0xbe, 0xf3, 0x1d, - 0xa1, 0xda, 0x93, 0x02, 0x10, 0x58, 0x1c, 0x10, 0xaa, 0x59, 0x08, 0xc0, 0x68, 0x41, 0x91, 0x84, - 0xc2, 0x5f, 0x35, 0xab, 0x48, 0xc2, 0x54, 0x50, 0x39, 0x23, 0xc8, 0xfc, 0x7d, 0x19, 0x20, 0x95, - 0x62, 0x58, 0xd0, 0xc7, 0xc4, 0x8e, 0x10, 0xe5, 0xe7, 0xbb, 0x7d, 0x72, 0xc9, 0x50, 0x64, 0x53, - 0xe4, 0xc6, 0x34, 0xc2, 0xe7, 0x7c, 0xfd, 0xb8, 0xd9, 0x37, 0xa4, 0xd9, 0x53, 0xba, 0x59, 0x37, - 0x31, 0x19, 0xca, 0x79, 0x3b, 0x7c, 0x9a, 0xa5, 0x67, 0x19, 0x87, 0x70, 0x23, 0xe5, 0xe9, 0x65, - 0xd8, 0x15, 0xaf, 0x62, 0xd7, 0x4d, 0xd8, 0x79, 0x29, 0xab, 0x3d, 0xe8, 0x62, 0x62, 0x7f, 0x1f, - 0xa3, 0x38, 0xc7, 0xa8, 0x74, 0x15, 0xa3, 0x0e, 0x26, 0x7f, 0x2b, 0x26, 0xa4, 0x6c, 0x06, 0x70, - 0x2b, 0x63, 0x25, 0xdf, 0xee, 0x19, 0x66, 0xe5, 0xab, 0x98, 0xad, 0x25, 0x5a, 0xf1, 0x7c, 0x90, - 0x72, 0xfc, 0x12, 0xd6, 0x30, 0xb1, 0x9f, 0x3b, 0x98, 0x4d, 0xb3, 0x5b, 0xfa, 0x05, 0x23, 0xf9, - 0x89, 0x96, 0xe7, 0x25, 0x8d, 0xf4, 0x11, 0x1d, 0xe5, 0x8c, 0x5c, 0xfe, 0x05, 0x23, 0x8f, 0xc4, - 0x84, 0x94, 0xcd, 0x36, 0x74, 0x30, 0x99, 0xd6, 0xa6, 0x72, 0x15, 0x93, 0x36, 0x26, 0x79, 0x4d, - 0x76, 0xa0, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x0d, 0x82, 0xea, 0x55, 0x2c, 0x56, 0x14, 0x7d, 0xc2, - 0xc3, 0xfc, 0x47, 0x68, 0x1c, 0xc4, 0x23, 0xc4, 0x26, 0x27, 0x49, 0x32, 0x78, 0x69, 0xf9, 0xc7, - 0xfc, 0x63, 0x11, 0xea, 0xbb, 0x23, 0x4a, 0xe2, 0x30, 0x97, 0x93, 0xe5, 0x26, 0x9d, 0xce, 0xc9, - 0x82, 0x44, 0xe4, 0x64, 0x49, 0xfc, 0x21, 0x34, 0x7c, 0xb1, 0x75, 0x15, 0xbd, 0xcc, 0x43, 0x9d, - 0x99, 0x4d, 0x6d, 0xd5, 0xfd, 0x4c, 0x32, 0xdb, 0x04, 0x08, 0xb1, 0x17, 0xa9, 0x39, 0x32, 0x1d, - 0xb5, 0x55, 0xb9, 0xa5, 0x53, 0xb4, 0x55, 0x0b, 0x93, 0x6c, 0xfd, 0x3e, 0xd4, 0x4f, 0xb8, 0x93, - 0xd4, 0x84, 0x5c, 0x32, 0x4a, 0xbd, 0x67, 0xc1, 0x49, 0xba, 0x09, 0x0f, 0xa0, 0x39, 0x96, 0x2e, - 0x53, 0x93, 0x64, 0x0c, 0xdd, 0x55, 0x96, 0xa4, 0xf6, 0x6e, 0x66, 0x3d, 0x2b, 0x17, 0xa0, 0x31, - 0xce, 0xa0, 0xfa, 0x43, 0xe8, 0xcc, 0x90, 0xcc, 0xc9, 0x41, 0x1b, 0xd9, 0x1c, 0x54, 0x7f, 0x68, - 0x48, 0x41, 0xd9, 0x99, 0xd9, 0xbc, 0xf4, 0x9f, 0x45, 0x68, 0x7c, 0x83, 0xd8, 0x73, 0x42, 0xcf, - 0xa4, 0xbe, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa3, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e, 0xc8, - 0x04, 0xa2, 0xd6, 0xb3, 0x42, 0x2f, 0x44, 0x62, 0x30, 0x5e, 0x03, 0xa0, 0x17, 0x76, 0xe8, 0xb8, - 0x67, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xc5, 0x40, 0x22, 0x78, 0x28, 0xd0, 0x0b, 0x1b, 0x51, - 0x4a, 0x68, 0xa4, 0x72, 0x55, 0x95, 0x5e, 0xec, 0x09, 0x58, 0xcd, 0xf5, 0x28, 0x09, 0x43, 0xe4, - 0x89, 0x1c, 0x2d, 0xe6, 0x3e, 0x91, 0x08, 0x2e, 0x95, 0x69, 0xa9, 0xcb, 0x52, 0x2a, 0x4b, 0xa5, - 0xb2, 0x54, 0x6a, 0x45, 0xce, 0x64, 0x59, 0xa9, 0x2c, 0x91, 0x5a, 0x95, 0x52, 0x59, 0x46, 0x2a, - 0x4b, 0xa5, 0xd6, 0xf4, 0x5c, 0x25, 0xd5, 0xfc, 0x8f, 0x02, 0xac, 0x4d, 0x17, 0x7e, 0xaa, 0x36, - 0xfd, 0x10, 0x1a, 0xae, 0x58, 0xaf, 0x5c, 0x4c, 0x76, 0x66, 0x56, 0xd2, 0xaa, 0xbb, 0x99, 0x30, - 0x7e, 0x04, 0xcd, 0x40, 0x3a, 0x38, 0x09, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, - 0x03, 0x99, 0x1e, 0x18, 0xdf, 0x52, 0xcc, 0xd0, 0x90, 0x51, 0xe4, 0xf8, 0x2f, 0xa3, 0xba, 0x37, - 0xa0, 0x2c, 0xaa, 0x15, 0xbe, 0x4c, 0x0d, 0x4b, 0x7c, 0x9b, 0x6f, 0x43, 0x37, 0x27, 0x45, 0xd9, - 0xba, 0x02, 0xa5, 0x09, 0x0a, 0x04, 0xf7, 0xa6, 0xc5, 0x3f, 0x4d, 0x07, 0x3a, 0x16, 0x72, 0xbc, - 0x97, 0xa7, 0x8d, 0x12, 0x51, 0x4a, 0x45, 0x6c, 0x80, 0x91, 0x15, 0xa1, 0x54, 0xd1, 0x5a, 0x17, - 0x32, 0x5a, 0x3f, 0x85, 0xce, 0xee, 0x84, 0x44, 0x68, 0xc8, 0x3c, 0x1c, 0xbc, 0x8c, 0x76, 0xe4, - 0x5f, 0xa0, 0xfb, 0x8c, 0x5d, 0x7e, 0xcb, 0x99, 0x45, 0xf8, 0x07, 0xf4, 0x92, 0xec, 0xa3, 0xe4, - 0xb9, 0xb6, 0x8f, 0x92, 0xe7, 0xbc, 0xb9, 0x71, 0xc9, 0x24, 0xf6, 0x03, 0xb1, 0x15, 0x9a, 0x96, - 0x82, 0xcc, 0x1d, 0x68, 0xc8, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0xcd, 0xdd, 0x83, 0xeb, 0x00, - 0xa1, 0x43, 0x1d, 0x1f, 0x31, 0x44, 0x65, 0x0c, 0xd5, 0xac, 0x0c, 0xc6, 0xfc, 0xaf, 0x22, 0xac, - 0xca, 0xfb, 0x86, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x7d, 0xa8, 0x8e, 0x49, 0xc4, 0x32, 0x0c, 0x13, - 0x98, 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0xe6, 0x2e, 0x01, 0x4a, 0x57, 0x5f, 0x02, 0xcc, - 0xb4, 0xf9, 0xe5, 0xd9, 0x36, 0x9f, 0xef, 0x36, 0x4d, 0x84, 0xe5, 0x1e, 0xaf, 0x59, 0x35, 0x85, - 0x39, 0xf4, 0x8c, 0x7b, 0xd0, 0x1e, 0x71, 0x2d, 0xed, 0x31, 0x21, 0x67, 0x76, 0xe8, 0xb0, 0xb1, - 0xd8, 0xea, 0x35, 0xab, 0x29, 0xd0, 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0x3e, 0x81, 0x96, - 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, 0x1d, 0x7e, 0x6a, 0x17, 0x65, 0xbd, 0x67, 0x35, 0xcf, 0x32, - 0x50, 0x64, 0xde, 0x84, 0x1b, 0x4f, 0x50, 0xc4, 0x28, 0xb9, 0xcc, 0x3b, 0xc6, 0xfc, 0x6b, 0x80, - 0xc3, 0x80, 0x21, 0x7a, 0xea, 0xb8, 0x28, 0x32, 0xde, 0xcb, 0x42, 0xaa, 0x38, 0x5a, 0xd9, 0x94, - 0xd7, 0x3d, 0xc9, 0x80, 0x95, 0xa1, 0x31, 0x37, 0x61, 0xd9, 0x22, 0x31, 0x4f, 0x47, 0x6f, 0xea, - 0x2f, 0x35, 0xaf, 0xa1, 0xe6, 0x09, 0xa4, 0xa5, 0xc6, 0xcc, 0x03, 0xdd, 0xc2, 0xa6, 0xec, 0xd4, - 0x12, 0x6d, 0x42, 0x0d, 0x6b, 0x9c, 0xca, 0x2a, 0xb3, 0xa2, 0x53, 0x12, 0xf3, 0x33, 0xe8, 0x4a, - 0x4e, 0x92, 0xb3, 0x66, 0xf3, 0x26, 0x2c, 0x53, 0xad, 0x46, 0x21, 0xbd, 0xe7, 0x51, 0x44, 0x6a, - 0xcc, 0x3c, 0x84, 0xdb, 0x72, 0xf2, 0x5e, 0x38, 0x46, 0x3e, 0xa2, 0xce, 0xe4, 0x88, 0xc4, 0x01, - 0x4b, 0xb8, 0x64, 0x23, 0xa0, 0x70, 0x65, 0x04, 0x70, 0xd7, 0x7e, 0x8d, 0x23, 0x96, 0xfa, 0x44, - 0xbb, 0xb6, 0x0b, 0x1d, 0x3e, 0x90, 0x53, 0xcf, 0xfc, 0x02, 0x1a, 0xdb, 0xd6, 0xe0, 0x1b, 0x84, - 0x47, 0xe3, 0x13, 0x9e, 0x88, 0x3f, 0xca, 0xc3, 0x4a, 0x98, 0xa1, 0x0c, 0xcf, 0x0c, 0x59, 0x39, - 0x3a, 0xf3, 0x4b, 0x58, 0xdb, 0xf6, 0xbc, 0x2c, 0x4a, 0xab, 0xfe, 0x1e, 0xd4, 0x82, 0x0c, 0xbb, - 0xcc, 0xf1, 0x97, 0xa3, 0x4e, 0x89, 0xcc, 0x07, 0x60, 0xec, 0x23, 0x76, 0x38, 0x78, 0xe6, 0x9c, - 0x4c, 0x52, 0x47, 0xde, 0x84, 0x0a, 0x8e, 0x6c, 0x1c, 0x9e, 0x7f, 0x24, 0xb8, 0x54, 0xad, 0x65, - 0x1c, 0x1d, 0x86, 0xe7, 0x1f, 0x99, 0xf7, 0xa1, 0x9b, 0x23, 0xbf, 0x22, 0x43, 0x6d, 0x83, 0x31, - 0xfc, 0xf5, 0x9c, 0x13, 0x16, 0xc5, 0x0c, 0x8b, 0xfb, 0xd0, 0x1d, 0xfe, 0x4a, 0x69, 0xff, 0x04, - 0xdd, 0xa7, 0xc1, 0x04, 0x07, 0x68, 0x77, 0x70, 0x7c, 0x84, 0x92, 0xf4, 0x6c, 0x40, 0x99, 0x97, - 0xb1, 0x4a, 0x96, 0xf8, 0xe6, 0x2a, 0x04, 0x27, 0xb6, 0x1b, 0xc6, 0x91, 0xba, 0xff, 0x5a, 0x0e, - 0x4e, 0x76, 0xc3, 0x38, 0xe2, 0xe7, 0x2d, 0xaf, 0xb7, 0x48, 0x30, 0xb9, 0x14, 0x49, 0xab, 0x6a, - 0x55, 0xdc, 0x30, 0x7e, 0x1a, 0x4c, 0x2e, 0xcd, 0xbf, 0x12, 0x97, 0x12, 0x08, 0x79, 0x96, 0x13, - 0x78, 0xc4, 0x7f, 0x82, 0xce, 0x33, 0x12, 0x66, 0xf4, 0xfe, 0xa9, 0x00, 0x8d, 0xed, 0x11, 0x0a, - 0xd8, 0x13, 0xc4, 0x1c, 0x3c, 0x11, 0x4d, 0xee, 0x39, 0xa2, 0x11, 0x26, 0x81, 0xca, 0x40, 0x1a, - 0x34, 0x5e, 0x87, 0x3a, 0x0e, 0x30, 0xb3, 0x3d, 0x07, 0xf9, 0x24, 0x10, 0x5c, 0xaa, 0x16, 0x70, - 0xd4, 0x13, 0x81, 0x31, 0xde, 0x86, 0xb6, 0xbc, 0x9f, 0xb4, 0xc7, 0x4e, 0xe0, 0x4d, 0x78, 0xee, - 0x2b, 0x89, 0x6c, 0xd5, 0x92, 0xe8, 0x03, 0x85, 0x35, 0xee, 0xc3, 0x8a, 0x8a, 0xcb, 0x94, 0xb2, - 0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2, - 0x87, 0xaa, 0x43, 0x6c, 0x6b, 0xfc, 0x50, 0xa2, 0xcd, 0x11, 0x74, 0xf7, 0xb9, 0x9d, 0xca, 0x92, - 0x74, 0xa7, 0xb5, 0x7c, 0xe4, 0xdb, 0x27, 0x13, 0xe2, 0x9e, 0xd9, 0xfc, 0xbc, 0x50, 0x1e, 0xe6, - 0x35, 0xe8, 0x0e, 0x47, 0x0e, 0xf1, 0x0f, 0xe2, 0x32, 0x84, 0x53, 0x8d, 0x09, 0x0b, 0x27, 0xf1, - 0xc8, 0x0e, 0x29, 0x39, 0x41, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, - 0xbf, 0x05, 0x58, 0xcd, 0x4b, 0x52, 0xab, 0xbd, 0x05, 0xab, 0x79, 0x51, 0xaa, 0x22, 0x92, 0x15, - 0x77, 0x27, 0x2b, 0x50, 0xd6, 0x46, 0x8f, 0xa0, 0x29, 0x6e, 0xb3, 0x6d, 0x4f, 0x72, 0xca, 0xd7, - 0x81, 0xd9, 0x75, 0xb1, 0x1a, 0x4e, 0x76, 0x95, 0x3e, 0x81, 0x5b, 0xca, 0x7c, 0x7b, 0x56, 0x6d, - 0x19, 0x10, 0x6b, 0x8a, 0xe0, 0x68, 0x4a, 0xfb, 0xaf, 0xa1, 0x97, 0xa2, 0x76, 0x2e, 0x05, 0x32, - 0xdd, 0x94, 0xdd, 0x29, 0x63, 0xb7, 0x3d, 0x8f, 0x8a, 0xdd, 0x5e, 0xb6, 0xe6, 0x0d, 0x99, 0x8f, - 0xe1, 0xe6, 0x10, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xe6, 0x4c, 0x32, 0x5b, 0x81, 0xd2, 0x10, 0xb9, - 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0x79, 0x00, 0x1e, 0x47, 0xc8, 0x15, 0x56, 0x96, 0x2c, 0xf1, 0x6d, - 0x86, 0x50, 0xf9, 0x62, 0xb8, 0xcf, 0x4b, 0x30, 0x1e, 0xd4, 0xb2, 0x64, 0x53, 0xc7, 0x73, 0xd3, - 0xaa, 0x08, 0xf8, 0xd0, 0x33, 0xbe, 0x84, 0xae, 0x1c, 0x72, 0xc7, 0x4e, 0x30, 0x42, 0x76, 0x48, - 0x26, 0xd8, 0x95, 0xa1, 0xdf, 0x7a, 0xd8, 0x57, 0x69, 0x48, 0xf1, 0xd9, 0x15, 0x24, 0x03, 0x41, - 0x61, 0x75, 0x46, 0xd3, 0x28, 0xf3, 0x77, 0x05, 0xa8, 0xa8, 0xfc, 0xc8, 0x4f, 0x79, 0x8f, 0xe2, - 0x73, 0x44, 0x55, 0xb0, 0x2b, 0xc8, 0x78, 0x0b, 0x5a, 0xf2, 0xcb, 0x26, 0x21, 0xc3, 0x24, 0x39, - 0x77, 0x9b, 0x12, 0xfb, 0x54, 0x22, 0xc5, 0x0d, 0xa8, 0xb8, 0x83, 0x54, 0xed, 0xbe, 0x82, 0x38, - 0xfe, 0x34, 0xe2, 0x4a, 0x89, 0x73, 0xb6, 0x66, 0x29, 0x88, 0x6f, 0x2e, 0xcd, 0x6f, 0x49, 0xf0, - 0xd3, 0x20, 0xdf, 0x5c, 0x3e, 0x4f, 0xed, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0xc1, 0x0a, 0x02, 0x35, - 0xe0, 0x18, 0x63, 0x03, 0xaa, 0xa7, 0x91, 0x2d, 0xac, 0x11, 0x45, 0x74, 0x92, 0xea, 0x95, 0xd5, - 0x56, 0xe5, 0x34, 0x12, 0x1f, 0xe6, 0xbf, 0x17, 0x60, 0x59, 0xbe, 0x17, 0x18, 0x2d, 0x28, 0x26, - 0x85, 0x50, 0x11, 0x8b, 0xa2, 0x52, 0x68, 0x25, 0x8b, 0x1f, 0xf1, 0xcd, 0x73, 0xcc, 0xb9, 0x2f, - 0x8f, 0x73, 0x65, 0xc4, 0xb9, 0x2f, 0xce, 0xf1, 0xb7, 0xa0, 0x95, 0xd6, 0x53, 0x62, 0x5c, 0x1a, - 0xd3, 0x4c, 0xb0, 0x82, 0x6c, 0xa1, 0x4d, 0xe6, 0xdf, 0x03, 0xa4, 0xf7, 0xe6, 0x3c, 0x1c, 0xe2, - 0x44, 0x19, 0xfe, 0xc9, 0x31, 0xa3, 0xa4, 0x12, 0xe3, 0x9f, 0xc6, 0x3d, 0x68, 0x39, 0x9e, 0x87, - 0xf9, 0x74, 0x67, 0xb2, 0x8f, 0xbd, 0x24, 0x81, 0xe4, 0xb1, 0xe6, 0xff, 0x17, 0xa0, 0xbd, 0x4b, - 0xc2, 0xcb, 0x2f, 0xf0, 0x04, 0x65, 0xb2, 0x9b, 0x50, 0x52, 0x15, 0x62, 0xfc, 0x9b, 0x37, 0x17, - 0xa7, 0x78, 0x82, 0xe4, 0xb6, 0x97, 0x51, 0x57, 0xe5, 0x08, 0xb1, 0xe5, 0xf5, 0x60, 0x72, 0x4b, - 0xda, 0x94, 0x83, 0x47, 0xc4, 0x13, 0x6d, 0x94, 0x87, 0xa9, 0x9d, 0xdc, 0x89, 0x36, 0xad, 0x8a, - 0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x3b, 0xef, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59, - 0x83, 0x65, 0x72, 0x7a, 0x1a, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c, - 0x0a, 0x5e, 0x15, 0xe7, 0xda, 0xd3, 0xa7, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0xfa, 0x04, 0x7e, 0x00, - 0x55, 0x8d, 0xfa, 0x35, 0xb7, 0xc9, 0xef, 0x40, 0x6b, 0xdb, 0xf3, 0x86, 0xcf, 0x9d, 0x50, 0xfb, - 0xa3, 0x07, 0x95, 0xc1, 0xee, 0xe1, 0x40, 0xba, 0xa4, 0xc4, 0x0d, 0x50, 0x20, 0x3f, 0xf1, 0xf7, - 0x11, 0x3b, 0x42, 0x8c, 0x62, 0x37, 0x39, 0xf1, 0xef, 0x42, 0x45, 0x61, 0xf8, 0x4c, 0x5f, 0x7e, - 0xea, 0x23, 0x40, 0x81, 0xe6, 0xdf, 0x80, 0xf1, 0x77, 0xbc, 0x0c, 0x46, 0xb2, 0x07, 0x52, 0x92, - 0xde, 0x81, 0xce, 0xb9, 0xc0, 0xda, 0xb2, 0x3e, 0xcc, 0x2c, 0x43, 0x5b, 0x0e, 0x88, 0xfc, 0x20, - 0x64, 0x1f, 0x43, 0x57, 0x56, 0xed, 0x92, 0xcf, 0x35, 0x58, 0x70, 0x1f, 0x26, 0xeb, 0x59, 0xb6, - 0xc4, 0xf7, 0xc3, 0x1f, 0xbb, 0xea, 0x18, 0x53, 0x97, 0x44, 0xc6, 0x3e, 0xb4, 0xa7, 0x5e, 0xf4, - 0x0c, 0x75, 0x6b, 0x38, 0xff, 0xa1, 0xaf, 0xbf, 0xb6, 0x29, 0x5f, 0x08, 0x37, 0xf5, 0x0b, 0xe1, - 0xe6, 0x9e, 0x1f, 0xb2, 0x4b, 0x63, 0x0f, 0x5a, 0xf9, 0xb7, 0x2f, 0xe3, 0x55, 0x5d, 0x62, 0xcd, - 0x79, 0x11, 0x5b, 0xc8, 0x66, 0x1f, 0xda, 0x53, 0xcf, 0x60, 0x5a, 0x9f, 0xf9, 0xaf, 0x63, 0x0b, - 0x19, 0x3d, 0x86, 0x7a, 0xe6, 0xdd, 0xcb, 0xe8, 0x49, 0x26, 0xb3, 0x4f, 0x61, 0x0b, 0x19, 0xec, - 0x42, 0x33, 0xf7, 0x14, 0x65, 0xf4, 0x95, 0x3d, 0x73, 0xde, 0xa7, 0x16, 0x32, 0xd9, 0x81, 0x7a, - 0xe6, 0x45, 0x48, 0x6b, 0x31, 0xfb, 0xec, 0xd4, 0xbf, 0x35, 0x67, 0x44, 0x9d, 0x96, 0xfb, 0xd0, - 0x9e, 0x7a, 0x26, 0xd2, 0x2e, 0x99, 0xff, 0x7a, 0xb4, 0x50, 0x99, 0x21, 0xdc, 0x98, 0x5b, 0x25, - 0x1b, 0x66, 0x96, 0xdd, 0xfc, 0x12, 0x7a, 0x21, 0xd3, 0xaf, 0xc4, 0xba, 0x67, 0xae, 0x16, 0x32, - 0xeb, 0x3e, 0xfb, 0xd2, 0xd4, 0xbf, 0x3d, 0x7f, 0x50, 0x99, 0xba, 0x07, 0xad, 0xfc, 0x23, 0x93, - 0x66, 0x36, 0xf7, 0xe9, 0xe9, 0xea, 0x20, 0xca, 0xbd, 0x37, 0xa5, 0x41, 0x34, 0xef, 0x19, 0x6a, - 0x21, 0xa3, 0x6d, 0x00, 0x75, 0x91, 0xe0, 0xe1, 0x20, 0x59, 0xbd, 0x99, 0x0b, 0x8c, 0x64, 0xf5, - 0xe6, 0x5c, 0x3a, 0x3c, 0x06, 0x90, 0xfd, 0xbf, 0x47, 0x62, 0x66, 0xdc, 0xd4, 0x6a, 0x4c, 0x5d, - 0x3a, 0xf4, 0x7b, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x39, 0x40, 0x7a, 0xaf, - 0xa0, 0x19, 0xcc, 0xdc, 0x34, 0x5c, 0xe1, 0x83, 0x46, 0xf6, 0x16, 0xc1, 0x50, 0xb6, 0xce, 0xb9, - 0x59, 0xb8, 0x82, 0x45, 0x7b, 0xaa, 0x4b, 0xcc, 0x47, 0xf0, 0x74, 0xf3, 0xd8, 0x9f, 0xe9, 0x14, - 0x8d, 0x47, 0xd0, 0xc8, 0xb6, 0x87, 0x5a, 0x8b, 0x39, 0x2d, 0x63, 0x3f, 0xd7, 0x22, 0x1a, 0x8f, - 0xa1, 0x95, 0xef, 0xe7, 0x74, 0x48, 0xcd, 0xed, 0xf2, 0xfa, 0xea, 0xe2, 0x33, 0x43, 0xfe, 0x01, - 0x40, 0xda, 0xf7, 0x69, 0xf7, 0xcd, 0x74, 0x82, 0x53, 0x52, 0xf7, 0xa1, 0x3d, 0xd5, 0xcf, 0x69, - 0x8b, 0xe7, 0xb7, 0x79, 0x57, 0x25, 0x90, 0x4c, 0x77, 0xa6, 0x43, 0x70, 0xb6, 0xbf, 0xd3, 0x21, - 0x38, 0xaf, 0x95, 0xdb, 0x81, 0xfa, 0x70, 0x96, 0xc7, 0x70, 0x21, 0x8f, 0x79, 0x0d, 0xda, 0x87, - 0x00, 0xe9, 0x59, 0xa8, 0xbd, 0x30, 0x73, 0x3a, 0xf6, 0x9b, 0xfa, 0x72, 0x5a, 0xd2, 0xed, 0x42, - 0x33, 0x77, 0x7f, 0xa3, 0x73, 0xe8, 0xbc, 0x4b, 0x9d, 0xab, 0x4e, 0x96, 0xfc, 0x65, 0x87, 0x5e, - 0xc1, 0xb9, 0x57, 0x20, 0x57, 0xc5, 0x71, 0xb6, 0x9d, 0xd4, 0x11, 0x34, 0xa7, 0xc5, 0xfc, 0x85, - 0xbc, 0x92, 0x6d, 0x19, 0x33, 0x79, 0x65, 0x4e, 0x27, 0xb9, 0x90, 0xd1, 0x01, 0xb4, 0xf7, 0x75, - 0x37, 0xa0, 0x3a, 0x15, 0xbd, 0x7e, 0xb3, 0x9d, 0x59, 0xbf, 0x3f, 0x6f, 0x48, 0xad, 0xcb, 0x57, - 0xd0, 0x99, 0xe9, 0x52, 0x8c, 0xf5, 0xe4, 0x89, 0x60, 0x6e, 0xfb, 0xb2, 0x50, 0xad, 0x43, 0x58, - 0x99, 0x6e, 0x52, 0x8c, 0xd7, 0x92, 0x98, 0x98, 0xd7, 0xbc, 0x2c, 0x64, 0xf5, 0x09, 0x54, 0x75, - 0xe1, 0x69, 0xa8, 0xa7, 0x98, 0xa9, 0x42, 0x74, 0xe1, 0xd4, 0x47, 0x22, 0xe4, 0x93, 0xa2, 0x2e, - 0x0d, 0xf9, 0xa9, 0xd2, 0xaf, 0xaf, 0x5e, 0x4e, 0x12, 0xca, 0x47, 0x50, 0x51, 0xb5, 0x9d, 0xb1, - 0x9a, 0x6c, 0xb6, 0x4c, 0xa9, 0x77, 0x55, 0x84, 0xed, 0x23, 0x96, 0xa9, 0xd8, 0xb4, 0xd0, 0xd9, - 0x22, 0x4e, 0xef, 0x91, 0xdc, 0x88, 0x5a, 0x8b, 0x6d, 0x68, 0x64, 0x6b, 0x36, 0xbd, 0xa4, 0x73, - 0xea, 0xb8, 0x45, 0x9a, 0xec, 0x5c, 0xfc, 0xf4, 0xf3, 0xfa, 0x2b, 0xbf, 0xfd, 0x79, 0xfd, 0x95, - 0x7f, 0x7b, 0xb1, 0x5e, 0xf8, 0xe9, 0xc5, 0x7a, 0xe1, 0x37, 0x2f, 0xd6, 0x0b, 0x7f, 0x78, 0xb1, - 0x5e, 0xf8, 0x87, 0x7f, 0xfe, 0x33, 0xff, 0x13, 0x46, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x9d, 0x63, - 0xca, 0x32, 0x43, 0xe1, 0xd9, 0x48, 0xfe, 0x31, 0x2c, 0xf3, 0x7f, 0x31, 0xae, 0xe5, 0xc9, 0xb2, - 0x80, 0x3f, 0xf8, 0x53, 0x00, 0x00, 0x00, 0xff, 0xff, 0xeb, 0xb0, 0x39, 0x1d, 0x7c, 0x26, 0x00, - 0x00, +var fileDescriptor_56ede974c0020f77 = []byte{ + // 3275 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x1c, 0x47, + 0x73, 0xdf, 0x3e, 0xc8, 0xdd, 0xad, 0x7d, 0x71, 0x87, 0x14, 0xb5, 0x5a, 0xeb, 0xa3, 0xe5, 0x91, + 0x25, 0x51, 0x76, 0x44, 0xda, 0xb2, 0x61, 0xf9, 0x01, 0x47, 0x21, 0x29, 0x9a, 0xa4, 0x6d, 0x5a, + 0x9b, 0x59, 0xd1, 0x0e, 0x12, 0x24, 0x83, 0xe1, 0x4c, 0x73, 0xb7, 0xcd, 0x9d, 0xe9, 0x71, 0x4f, + 0x0f, 0x45, 0x3a, 0x40, 0x90, 0x43, 0x90, 0xdc, 0x72, 0xcc, 0x2d, 0x7f, 0x20, 0xc8, 0x3f, 0xc8, + 0x35, 0x07, 0x23, 0xa7, 0x1c, 0x73, 0x49, 0x10, 0xeb, 0x27, 0xe4, 0x17, 0x04, 0xfd, 0x9a, 0xc7, + 0xbe, 0x64, 0x10, 0x02, 0xbe, 0xcb, 0x62, 0xba, 0xba, 0xba, 0x5e, 0x5d, 0x5d, 0x5d, 0x55, 0xbd, + 0x50, 0x77, 0x86, 0x28, 0x60, 0x5b, 0x21, 0x25, 0x8c, 0x18, 0xe5, 0x21, 0x0d, 0xdd, 0x5e, 0x8d, + 0xb8, 0x58, 0x02, 0x7a, 0x35, 0x37, 0xd2, 0x9f, 0x75, 0x76, 0x15, 0xa2, 0x48, 0x0d, 0xde, 0x1a, + 0x12, 0x32, 0x1c, 0xa3, 0x6d, 0x31, 0x3a, 0x8d, 0xcf, 0xb6, 0x91, 0x1f, 0xb2, 0x2b, 0x39, 0x69, + 0xfe, 0x73, 0x11, 0xd6, 0xf7, 0x28, 0x72, 0x18, 0xda, 0x23, 0x01, 0x73, 0x70, 0x80, 0xa8, 0x85, + 0x7e, 0x8a, 0x51, 0xc4, 0x8c, 0x77, 0xa0, 0xe1, 0x6a, 0x98, 0x8d, 0xbd, 0x6e, 0xe1, 0x4e, 0x61, + 0xb3, 0x66, 0xd5, 0x13, 0xd8, 0x91, 0x67, 0xdc, 0x84, 0x0a, 0xba, 0x44, 0x2e, 0x9f, 0x2d, 0x8a, + 0xd9, 0x65, 0x3e, 0x3c, 0xf2, 0x8c, 0x0f, 0xa1, 0x1e, 0x31, 0x8a, 0x83, 0xa1, 0x1d, 0x47, 0x88, + 0x76, 0x4b, 0x77, 0x0a, 0x9b, 0xf5, 0xc7, 0x2b, 0x5b, 0x5c, 0xe4, 0xad, 0x81, 0x98, 0x38, 0x89, + 0x10, 0xb5, 0x20, 0x4a, 0xbe, 0x8d, 0xfb, 0x50, 0xf1, 0xd0, 0x05, 0x76, 0x51, 0xd4, 0x2d, 0xdf, + 0x29, 0x6d, 0xd6, 0x1f, 0x37, 0x24, 0xfa, 0x33, 0x01, 0xb4, 0xf4, 0xa4, 0xf1, 0x10, 0xaa, 0x11, + 0x23, 0xd4, 0x19, 0xa2, 0xa8, 0xbb, 0x24, 0x10, 0x9b, 0x9a, 0xae, 0x80, 0x5a, 0xc9, 0xb4, 0x71, + 0x1b, 0x4a, 0xcf, 0xf7, 0x8e, 0xba, 0xcb, 0x82, 0x3b, 0x28, 0xac, 0x10, 0xb9, 0x16, 0x07, 0x1b, + 0x77, 0xa1, 0x19, 0x39, 0x81, 0x77, 0x4a, 0x2e, 0xed, 0x10, 0x7b, 0x41, 0xd4, 0xad, 0xdc, 0x29, + 0x6c, 0x56, 0xad, 0x86, 0x02, 0xf6, 0x39, 0xcc, 0xfc, 0x1c, 0x6e, 0x0c, 0x98, 0x43, 0xd9, 0x35, + 0xac, 0x63, 0x9e, 0xc0, 0xba, 0x85, 0x7c, 0x72, 0x71, 0x2d, 0xd3, 0x76, 0xa1, 0xc2, 0xb0, 0x8f, + 0x48, 0xcc, 0x84, 0x69, 0x9b, 0x96, 0x1e, 0x9a, 0xff, 0x5a, 0x00, 0x63, 0xff, 0x12, 0xb9, 0x7d, + 0x4a, 0x5c, 0x14, 0x45, 0x7f, 0xa0, 0xed, 0x7a, 0x00, 0x95, 0x50, 0x0a, 0xd0, 0x2d, 0x0b, 0x74, + 0xb5, 0x0b, 0x5a, 0x2a, 0x3d, 0x6b, 0xfe, 0x08, 0x6b, 0x03, 0x3c, 0x0c, 0x9c, 0xf1, 0x1b, 0x94, + 0x77, 0x1d, 0x96, 0x23, 0x41, 0x53, 0x88, 0xda, 0xb4, 0xd4, 0xc8, 0xec, 0x83, 0xf1, 0x83, 0x83, + 0xd9, 0x9b, 0xe3, 0x64, 0x3e, 0x82, 0xd5, 0x1c, 0xc5, 0x28, 0x24, 0x41, 0x84, 0x84, 0x00, 0xcc, + 0x61, 0x71, 0x24, 0x88, 0x2d, 0x59, 0x6a, 0x64, 0x12, 0x58, 0x3f, 0x09, 0xbd, 0x6b, 0x9e, 0xa6, + 0xc7, 0x50, 0xa3, 0x28, 0x22, 0x31, 0xe5, 0x67, 0xa0, 0x28, 0x8c, 0xba, 0x26, 0x8d, 0xfa, 0x2d, + 0x0e, 0xe2, 0x4b, 0x4b, 0xcf, 0x59, 0x29, 0x9a, 0xf2, 0x4f, 0x16, 0x5d, 0xc7, 0x3f, 0x3f, 0x87, + 0x1b, 0x7d, 0x27, 0x8e, 0xae, 0x23, 0xab, 0xf9, 0x05, 0xf7, 0xed, 0x28, 0xf6, 0xaf, 0xb5, 0xf8, + 0x5f, 0x0a, 0x50, 0xdd, 0x0b, 0xe3, 0x93, 0xc8, 0x19, 0x22, 0xe3, 0x6d, 0xa8, 0x33, 0xc2, 0x9c, + 0xb1, 0x1d, 0xf3, 0xa1, 0x40, 0x2f, 0x5b, 0x20, 0x40, 0x12, 0xe1, 0x1d, 0x68, 0x84, 0x88, 0xba, + 0x61, 0xac, 0x30, 0x8a, 0x77, 0x4a, 0x9b, 0x65, 0xab, 0x2e, 0x61, 0x12, 0x65, 0x0b, 0x56, 0xc5, + 0x9c, 0x8d, 0x03, 0xfb, 0x1c, 0xd1, 0x00, 0x8d, 0x7d, 0xe2, 0x21, 0xe1, 0x1c, 0x65, 0xab, 0x23, + 0xa6, 0x8e, 0x82, 0x6f, 0x92, 0x09, 0xe3, 0x3d, 0xe8, 0x24, 0xf8, 0xdc, 0xe3, 0x05, 0x76, 0x59, + 0x60, 0xb7, 0x15, 0xf6, 0x89, 0x02, 0x9b, 0x7f, 0x03, 0xad, 0x17, 0x23, 0x4a, 0x18, 0x1b, 0xe3, + 0x60, 0xf8, 0xcc, 0x61, 0x0e, 0x3f, 0x9a, 0x21, 0xa2, 0x98, 0x78, 0x91, 0x92, 0x56, 0x0f, 0x8d, + 0xf7, 0xa1, 0xc3, 0x24, 0x2e, 0xf2, 0x6c, 0x8d, 0x53, 0x14, 0x38, 0x2b, 0xc9, 0x44, 0x5f, 0x21, + 0xdf, 0x83, 0x56, 0x8a, 0xcc, 0x0f, 0xb7, 0x92, 0xb7, 0x99, 0x40, 0x5f, 0x60, 0x1f, 0x99, 0x17, + 0xc2, 0x56, 0x62, 0x93, 0x8d, 0xf7, 0xa1, 0x96, 0xda, 0xa1, 0x20, 0x3c, 0xa4, 0x25, 0x3d, 0x44, + 0x9b, 0xd3, 0xaa, 0x26, 0x46, 0xf9, 0x12, 0xda, 0x2c, 0x11, 0xdc, 0xf6, 0x1c, 0xe6, 0xe4, 0x9d, + 0x2a, 0xaf, 0x95, 0xd5, 0x62, 0xb9, 0xb1, 0xf9, 0x05, 0xd4, 0xfa, 0xd8, 0x8b, 0x24, 0xe3, 0x2e, + 0x54, 0xdc, 0x98, 0x52, 0x14, 0x30, 0xad, 0xb2, 0x1a, 0x1a, 0x6b, 0xb0, 0x34, 0xc6, 0x3e, 0x66, + 0x4a, 0x4d, 0x39, 0x30, 0x09, 0xc0, 0x31, 0xf2, 0x09, 0xbd, 0x12, 0x06, 0x5b, 0x83, 0xa5, 0xec, + 0xe6, 0xca, 0x81, 0xf1, 0x16, 0xd4, 0x7c, 0xe7, 0x32, 0xd9, 0x54, 0x3e, 0x53, 0xf5, 0x9d, 0x4b, + 0x29, 0x7c, 0x17, 0x2a, 0x67, 0x0e, 0x1e, 0xbb, 0x01, 0x53, 0x56, 0xd1, 0xc3, 0x94, 0x61, 0x39, + 0xcb, 0xf0, 0xdf, 0x8b, 0x50, 0x97, 0x1c, 0xa5, 0xc0, 0x6b, 0xb0, 0xe4, 0x3a, 0xee, 0x28, 0x61, + 0x29, 0x06, 0xc6, 0x7d, 0x2d, 0x48, 0x31, 0x1b, 0xe1, 0x52, 0x49, 0xb5, 0x68, 0xdb, 0x00, 0xd1, + 0x4b, 0x27, 0x54, 0xb2, 0x95, 0xe6, 0x20, 0xd7, 0x38, 0x8e, 0x14, 0xf7, 0x23, 0x68, 0x48, 0xbf, + 0x53, 0x4b, 0xca, 0x73, 0x96, 0xd4, 0x25, 0x96, 0x5c, 0x74, 0x17, 0x9a, 0x71, 0x84, 0xec, 0x11, + 0x46, 0xd4, 0xa1, 0xee, 0xe8, 0xaa, 0xbb, 0x24, 0x2f, 0xa0, 0x38, 0x42, 0x87, 0x1a, 0x66, 0x3c, + 0x86, 0x25, 0x1e, 0x5b, 0xa2, 0xee, 0xb2, 0xb8, 0xeb, 0x6e, 0x67, 0x49, 0x0a, 0x55, 0xb7, 0xc4, + 0xef, 0x7e, 0xc0, 0xe8, 0x95, 0x25, 0x51, 0x7b, 0x9f, 0x02, 0xa4, 0x40, 0x63, 0x05, 0x4a, 0xe7, + 0xe8, 0x4a, 0x9d, 0x43, 0xfe, 0xc9, 0x8d, 0x73, 0xe1, 0x8c, 0x63, 0x6d, 0x75, 0x39, 0xf8, 0xbc, + 0xf8, 0x69, 0xc1, 0x74, 0xa1, 0xbd, 0x3b, 0x3e, 0xc7, 0x24, 0xb3, 0x7c, 0x0d, 0x96, 0x7c, 0xe7, + 0x47, 0x42, 0xb5, 0x25, 0xc5, 0x40, 0x40, 0x71, 0x40, 0xa8, 0x26, 0x21, 0x06, 0x46, 0x0b, 0x8a, + 0x24, 0x14, 0xf6, 0xaa, 0x59, 0x45, 0x12, 0xa6, 0x8c, 0xca, 0x19, 0x46, 0xe6, 0xff, 0x94, 0x01, + 0x52, 0x2e, 0x86, 0x05, 0x3d, 0x4c, 0xec, 0x08, 0x51, 0x7e, 0xbf, 0xdb, 0xa7, 0x57, 0x0c, 0x45, + 0x36, 0x45, 0x6e, 0x4c, 0x23, 0x7c, 0xc1, 0xf7, 0x8f, 0xab, 0x7d, 0x43, 0xaa, 0x3d, 0x21, 0x9b, + 0x75, 0x13, 0x93, 0x81, 0x5c, 0xb7, 0xcb, 0x97, 0x59, 0x7a, 0x95, 0x71, 0x04, 0x37, 0x52, 0x9a, + 0x5e, 0x86, 0x5c, 0x71, 0x11, 0xb9, 0xd5, 0x84, 0x9c, 0x97, 0x92, 0xda, 0x87, 0x55, 0x4c, 0xec, + 0x9f, 0x62, 0x14, 0xe7, 0x08, 0x95, 0x16, 0x11, 0xea, 0x60, 0xf2, 0xa7, 0x62, 0x41, 0x4a, 0xa6, + 0x0f, 0xb7, 0x32, 0x5a, 0xf2, 0xe3, 0x9e, 0x21, 0x56, 0x5e, 0x44, 0x6c, 0x3d, 0x91, 0x8a, 0xc7, + 0x83, 0x94, 0xe2, 0xd7, 0xb0, 0x8e, 0x89, 0xfd, 0xd2, 0xc1, 0x6c, 0x92, 0xdc, 0xd2, 0x6b, 0x94, + 0xe4, 0x37, 0x5a, 0x9e, 0x96, 0x54, 0xd2, 0x47, 0x74, 0x98, 0x53, 0x72, 0xf9, 0x35, 0x4a, 0x1e, + 0x8b, 0x05, 0x29, 0x99, 0x1d, 0xe8, 0x60, 0x32, 0x29, 0x4d, 0x65, 0x11, 0x91, 0x36, 0x26, 0x79, + 0x49, 0x76, 0xa1, 0x13, 0x21, 0x97, 0x11, 0x9a, 0x75, 0x82, 0xea, 0x22, 0x12, 0x2b, 0x0a, 0x3f, + 0xa1, 0x61, 0xfe, 0x05, 0x34, 0x0e, 0xe3, 0x21, 0x62, 0xe3, 0xd3, 0x24, 0x18, 0xbc, 0xb1, 0xf8, + 0x63, 0xfe, 0x5f, 0x11, 0xea, 0x7b, 0x43, 0x4a, 0xe2, 0x30, 0x17, 0x93, 0xe5, 0x21, 0x9d, 0x8c, + 0xc9, 0x02, 0x45, 0xc4, 0x64, 0x89, 0xfc, 0x31, 0x34, 0x7c, 0x71, 0x74, 0x15, 0xbe, 0x8c, 0x43, + 0x9d, 0xa9, 0x43, 0x6d, 0xd5, 0xfd, 0x4c, 0x30, 0xdb, 0x02, 0x08, 0xb1, 0x17, 0xa9, 0x35, 0x32, + 0x1c, 0xb5, 0x55, 0xba, 0xa5, 0x43, 0xb4, 0x55, 0x0b, 0x93, 0x68, 0xfd, 0x21, 0xd4, 0x4f, 0xb9, + 0x91, 0xd4, 0x82, 0x5c, 0x30, 0x4a, 0xad, 0x67, 0xc1, 0x69, 0x7a, 0x08, 0x0f, 0xa1, 0x39, 0x92, + 0x26, 0x53, 0x8b, 0xa4, 0x0f, 0xdd, 0x55, 0x9a, 0xa4, 0xfa, 0x6e, 0x65, 0x2d, 0x2b, 0x37, 0xa0, + 0x31, 0xca, 0x80, 0x7a, 0x03, 0xe8, 0x4c, 0xa1, 0xcc, 0x88, 0x41, 0x9b, 0xd9, 0x18, 0x54, 0x7f, + 0x6c, 0x48, 0x46, 0xd9, 0x95, 0xd9, 0xb8, 0xf4, 0x8f, 0x45, 0x68, 0x7c, 0x87, 0xd8, 0x4b, 0x42, + 0xcf, 0xa5, 0xbc, 0x06, 0x94, 0x03, 0xc7, 0x47, 0x8a, 0xa2, 0xf8, 0x36, 0x6e, 0x41, 0x95, 0x5e, + 0xca, 0x00, 0xa2, 0xf6, 0xb3, 0x42, 0x2f, 0x45, 0x60, 0x30, 0x7e, 0x0f, 0x40, 0x2f, 0xed, 0xd0, + 0x71, 0xcf, 0x91, 0xb2, 0x60, 0xd9, 0xaa, 0xd1, 0xcb, 0xbe, 0x04, 0x70, 0x57, 0xa0, 0x97, 0x36, + 0xa2, 0x94, 0xd0, 0x48, 0xc5, 0xaa, 0x2a, 0xbd, 0xdc, 0x17, 0x63, 0xb5, 0xd6, 0xa3, 0x24, 0x0c, + 0x91, 0x27, 0x62, 0xb4, 0x58, 0xfb, 0x4c, 0x02, 0x38, 0x57, 0xa6, 0xb9, 0x2e, 0x4b, 0xae, 0x2c, + 0xe5, 0xca, 0x52, 0xae, 0x15, 0xb9, 0x92, 0x65, 0xb9, 0xb2, 0x84, 0x6b, 0x55, 0x72, 0x65, 0x19, + 0xae, 0x2c, 0xe5, 0x5a, 0xd3, 0x6b, 0x15, 0x57, 0xf3, 0x1f, 0x0a, 0xb0, 0x3e, 0x99, 0xf8, 0xa9, + 0xdc, 0xf4, 0x63, 0x68, 0xb8, 0x62, 0xbf, 0x72, 0x3e, 0xd9, 0x99, 0xda, 0x49, 0xab, 0xee, 0x66, + 0xdc, 0xf8, 0x09, 0x34, 0x03, 0x69, 0xe0, 0xc4, 0x35, 0x4b, 0xe9, 0xbe, 0x64, 0x6d, 0x6f, 0x35, + 0x82, 0xcc, 0xc8, 0xf4, 0xc0, 0xf8, 0x81, 0x62, 0x86, 0x06, 0x8c, 0x22, 0xc7, 0x7f, 0x13, 0xd9, + 0xbd, 0x01, 0x65, 0x91, 0xad, 0xf0, 0x6d, 0x6a, 0x58, 0xe2, 0xdb, 0x7c, 0x00, 0xab, 0x39, 0x2e, + 0x4a, 0xd7, 0x15, 0x28, 0x8d, 0x51, 0x20, 0xa8, 0x37, 0x2d, 0xfe, 0x69, 0x3a, 0xd0, 0xb1, 0x90, + 0xe3, 0xbd, 0x39, 0x69, 0x14, 0x8b, 0x52, 0xca, 0x62, 0x13, 0x8c, 0x2c, 0x0b, 0x25, 0x8a, 0x96, + 0xba, 0x90, 0x91, 0xfa, 0x39, 0x74, 0xf6, 0xc6, 0x24, 0x42, 0x03, 0xe6, 0xe1, 0xe0, 0x4d, 0x94, + 0x23, 0x7f, 0x0d, 0xab, 0x2f, 0xd8, 0xd5, 0x0f, 0x9c, 0x58, 0x84, 0x7f, 0x46, 0x6f, 0x48, 0x3f, + 0x4a, 0x5e, 0x6a, 0xfd, 0x28, 0x79, 0xc9, 0x8b, 0x1b, 0x97, 0x8c, 0x63, 0x3f, 0x10, 0x47, 0xa1, + 0x69, 0xa9, 0x91, 0xb9, 0x0b, 0x0d, 0x99, 0x43, 0x1f, 0x13, 0x2f, 0x1e, 0xa3, 0x99, 0x67, 0x70, + 0x03, 0x20, 0x74, 0xa8, 0xe3, 0x23, 0x86, 0xa8, 0xf4, 0xa1, 0x9a, 0x95, 0x81, 0x98, 0xff, 0x54, + 0x84, 0x35, 0xd9, 0x6f, 0x18, 0xc8, 0x32, 0x5b, 0xab, 0xd0, 0x83, 0xea, 0x88, 0x44, 0x2c, 0x43, + 0x30, 0x19, 0x73, 0x11, 0x79, 0x7d, 0x2e, 0xa9, 0xf1, 0xcf, 0x5c, 0x13, 0xa0, 0xb4, 0xb8, 0x09, + 0x30, 0x55, 0xe6, 0x97, 0xa7, 0xcb, 0x7c, 0x7e, 0xda, 0x34, 0x12, 0x96, 0x67, 0xbc, 0x66, 0xd5, + 0x14, 0xe4, 0xc8, 0x33, 0xee, 0x43, 0x7b, 0xc8, 0xa5, 0xb4, 0x47, 0x84, 0x9c, 0xdb, 0xa1, 0xc3, + 0x46, 0xe2, 0xa8, 0xd7, 0xac, 0xa6, 0x00, 0x1f, 0x12, 0x72, 0xde, 0x77, 0xd8, 0xc8, 0xf8, 0x0c, + 0x5a, 0x2a, 0x0d, 0xf4, 0x85, 0x89, 0x22, 0x75, 0xf9, 0xa9, 0x53, 0x94, 0xb5, 0x9e, 0xd5, 0x3c, + 0xcf, 0x8c, 0x22, 0xf3, 0x26, 0xdc, 0x78, 0x86, 0x22, 0x46, 0xc9, 0x55, 0xde, 0x30, 0xe6, 0x03, + 0xb8, 0x27, 0xbb, 0x08, 0x03, 0xe6, 0x8c, 0xd1, 0xf7, 0x98, 0x32, 0x4c, 0xce, 0xa2, 0xc1, 0xc8, + 0xa1, 0xe8, 0x98, 0xc4, 0x01, 0xd3, 0x65, 0xae, 0xf9, 0xc7, 0x00, 0x47, 0x01, 0x43, 0xf4, 0xcc, + 0x71, 0x51, 0x64, 0x7c, 0x90, 0x1d, 0xa9, 0x2c, 0x6a, 0x65, 0x4b, 0xf6, 0x85, 0x92, 0x09, 0x2b, + 0x83, 0x63, 0x6e, 0xc1, 0xb2, 0x45, 0x62, 0x1e, 0xb7, 0xde, 0xd5, 0x5f, 0x6a, 0x5d, 0x43, 0xad, + 0x13, 0x40, 0x4b, 0xcd, 0x99, 0x87, 0xba, 0xd6, 0x4d, 0xc9, 0xa9, 0xbd, 0xdc, 0x82, 0x1a, 0xd6, + 0x30, 0x15, 0x7e, 0xa6, 0x59, 0xa7, 0x28, 0xe6, 0x17, 0xb0, 0x2a, 0x29, 0x49, 0xca, 0x9a, 0xcc, + 0xbb, 0xb0, 0x4c, 0xb5, 0x18, 0x85, 0xb4, 0x21, 0xa4, 0x90, 0xd4, 0x9c, 0x79, 0x04, 0xb7, 0xe5, + 0xe2, 0xfd, 0x70, 0x84, 0x7c, 0x44, 0x9d, 0x71, 0xce, 0x2c, 0x39, 0x57, 0x29, 0x2c, 0x74, 0x15, + 0xbe, 0x07, 0xdf, 0xe2, 0x88, 0xa5, 0x36, 0xd1, 0xa6, 0x5d, 0x85, 0x0e, 0x9f, 0xc8, 0x89, 0x67, + 0x7e, 0x05, 0x8d, 0x1d, 0xab, 0xff, 0x1d, 0xc2, 0xc3, 0xd1, 0x29, 0x8f, 0xd8, 0x9f, 0xe4, 0xc7, + 0x8a, 0x99, 0xa1, 0x14, 0xcf, 0x4c, 0x59, 0x39, 0x3c, 0xf3, 0x6b, 0x58, 0xdf, 0xf1, 0xbc, 0x2c, + 0x48, 0x8b, 0xfe, 0x01, 0xd4, 0x82, 0x0c, 0xb9, 0xcc, 0x3d, 0x99, 0xc3, 0x4e, 0x91, 0xcc, 0x47, + 0x60, 0x1c, 0x20, 0x76, 0xd4, 0x7f, 0xe1, 0x9c, 0x8e, 0x53, 0x43, 0xde, 0x84, 0x0a, 0x8e, 0x6c, + 0x1c, 0x5e, 0x7c, 0x22, 0xa8, 0x54, 0xad, 0x65, 0x1c, 0x1d, 0x85, 0x17, 0x9f, 0x98, 0x0f, 0x61, + 0x35, 0x87, 0xbe, 0x20, 0x94, 0xed, 0x80, 0x31, 0xf8, 0xed, 0x94, 0x13, 0x12, 0xc5, 0x0c, 0x89, + 0x87, 0xb0, 0x3a, 0xf8, 0x8d, 0xdc, 0xfe, 0x12, 0x56, 0x9f, 0x07, 0x63, 0x1c, 0xa0, 0xbd, 0xfe, + 0xc9, 0x31, 0x4a, 0xe2, 0xb8, 0x01, 0x65, 0x9e, 0xef, 0x2a, 0x5e, 0xe2, 0x9b, 0x8b, 0x10, 0x9c, + 0xda, 0x6e, 0x18, 0x47, 0xaa, 0x51, 0xb6, 0x1c, 0x9c, 0xee, 0x85, 0x71, 0xc4, 0x2f, 0x66, 0x9e, + 0x98, 0x91, 0x60, 0x7c, 0x25, 0xa2, 0x5b, 0xd5, 0xaa, 0xb8, 0x61, 0xfc, 0x3c, 0x18, 0x5f, 0x99, + 0x7f, 0x24, 0xba, 0x17, 0x08, 0x79, 0x96, 0x13, 0x78, 0xc4, 0x7f, 0x86, 0x2e, 0x32, 0x1c, 0xa6, + 0xe4, 0xfe, 0xa5, 0x00, 0x8d, 0x9d, 0x21, 0x0a, 0xd8, 0x33, 0xc4, 0x1c, 0x3c, 0x16, 0xd5, 0xf0, + 0x05, 0xa2, 0x11, 0x26, 0x81, 0x0a, 0x55, 0x7a, 0x68, 0xbc, 0x0d, 0x75, 0x1c, 0x60, 0x66, 0x7b, + 0x0e, 0xf2, 0x49, 0x20, 0xa8, 0x54, 0x2d, 0xe0, 0xa0, 0x67, 0x02, 0x62, 0x3c, 0x80, 0xb6, 0x6c, + 0x64, 0xda, 0x23, 0x27, 0xf0, 0xc6, 0x3c, 0x48, 0x96, 0x44, 0x58, 0x6b, 0x49, 0xf0, 0xa1, 0x82, + 0x1a, 0x0f, 0x61, 0x45, 0xf9, 0x65, 0x8a, 0x59, 0x16, 0x98, 0x6d, 0x05, 0xcf, 0xa1, 0xc6, 0x61, + 0x48, 0x28, 0x8b, 0xec, 0x08, 0xb9, 0x2e, 0xf1, 0x43, 0x55, 0x4a, 0xb6, 0x35, 0x7c, 0x20, 0xc1, + 0xe6, 0x10, 0x56, 0x0f, 0xb8, 0x9e, 0x4a, 0x93, 0xf4, 0xa4, 0xb5, 0x7c, 0xe4, 0xdb, 0xa7, 0x63, + 0xe2, 0x9e, 0xdb, 0xfc, 0x62, 0x51, 0x16, 0xe6, 0xc9, 0xea, 0x2e, 0x07, 0x0e, 0xf0, 0xcf, 0xa2, + 0x6b, 0xc2, 0xb1, 0x46, 0x84, 0x85, 0xe3, 0x78, 0x68, 0x87, 0x94, 0x9c, 0x22, 0xa5, 0x62, 0xdb, + 0x47, 0xfe, 0xa1, 0x84, 0xf7, 0x39, 0xd8, 0xfc, 0xb7, 0x02, 0xac, 0xe5, 0x39, 0xa9, 0xdd, 0xde, + 0x86, 0xb5, 0x3c, 0x2b, 0x95, 0x3a, 0xc9, 0xd4, 0xbc, 0x93, 0x65, 0x28, 0x93, 0xa8, 0x27, 0xd0, + 0x14, 0x6d, 0x6f, 0xdb, 0x93, 0x94, 0xf2, 0x09, 0x63, 0x76, 0x5f, 0xac, 0x86, 0x93, 0xdd, 0xa5, + 0xcf, 0xe0, 0x96, 0x52, 0xdf, 0x9e, 0x16, 0x5b, 0x3a, 0xc4, 0xba, 0x42, 0x38, 0x9e, 0x90, 0xfe, + 0x5b, 0xe8, 0xa6, 0xa0, 0xdd, 0x2b, 0x01, 0x4c, 0x0f, 0xe5, 0xea, 0x84, 0xb2, 0x3b, 0x9e, 0x47, + 0xc5, 0x69, 0x2f, 0x5b, 0xb3, 0xa6, 0xcc, 0xa7, 0x70, 0x73, 0x80, 0x98, 0xb4, 0x86, 0xc3, 0x54, + 0x15, 0x27, 0x89, 0xad, 0x40, 0x69, 0x80, 0x5c, 0xa1, 0x7c, 0xc9, 0xe2, 0x9f, 0xdc, 0x01, 0x4f, + 0x22, 0xe4, 0x0a, 0x2d, 0x4b, 0x96, 0xf8, 0x36, 0x43, 0xa8, 0x7c, 0x35, 0x38, 0xe0, 0xb9, 0x1a, + 0x77, 0x6a, 0x99, 0xdb, 0xa9, 0x7b, 0xbc, 0x69, 0x55, 0xc4, 0xf8, 0xc8, 0x33, 0xbe, 0x86, 0x55, + 0x39, 0xe5, 0x8e, 0x9c, 0x60, 0x88, 0xec, 0x90, 0x8c, 0xb1, 0x2b, 0x5d, 0xbf, 0xf5, 0xb8, 0xa7, + 0xc2, 0x90, 0xa2, 0xb3, 0x27, 0x50, 0xfa, 0x02, 0xc3, 0xea, 0x0c, 0x27, 0x41, 0xe6, 0x7f, 0x17, + 0xa0, 0xa2, 0xe2, 0x23, 0x4f, 0x07, 0x3c, 0x8a, 0x2f, 0x10, 0x55, 0xce, 0xae, 0x46, 0xc6, 0x3d, + 0x68, 0xc9, 0x2f, 0x9b, 0x84, 0x0c, 0x93, 0xe4, 0x82, 0x6e, 0x4a, 0xe8, 0x73, 0x09, 0x14, 0xad, + 0x52, 0xd1, 0xac, 0x54, 0x7d, 0x01, 0x35, 0xe2, 0xf0, 0xb3, 0x88, 0x0b, 0x25, 0x2e, 0xe4, 0x9a, + 0xa5, 0x46, 0xfc, 0x70, 0x69, 0x7a, 0x4b, 0x82, 0x9e, 0x1e, 0xf2, 0xc3, 0xe5, 0xf3, 0xd0, 0x6e, + 0x87, 0x04, 0x07, 0x4c, 0xdd, 0xc0, 0x20, 0x40, 0x7d, 0x0e, 0x31, 0x36, 0xa1, 0x7a, 0x16, 0xd9, + 0x42, 0x1b, 0x91, 0x6d, 0x27, 0xa1, 0x5e, 0x69, 0x6d, 0x55, 0xce, 0x22, 0xf1, 0x61, 0xfe, 0x7d, + 0x01, 0x96, 0xe5, 0xc3, 0x82, 0xd1, 0x82, 0x62, 0x92, 0x31, 0x15, 0xb1, 0xc8, 0x3e, 0x85, 0x54, + 0x32, 0x4b, 0x12, 0xdf, 0x3c, 0xc6, 0x5c, 0xf8, 0xf2, 0xde, 0x57, 0x4a, 0x5c, 0xf8, 0xe2, 0xc2, + 0xbf, 0x07, 0xad, 0x34, 0xf1, 0x12, 0xf3, 0x52, 0x99, 0x66, 0x02, 0x15, 0x68, 0x73, 0x75, 0x32, + 0xff, 0x0c, 0x20, 0x6d, 0xb0, 0x73, 0x77, 0x88, 0x13, 0x61, 0xf8, 0x27, 0x87, 0x0c, 0x93, 0x94, + 0x8d, 0x7f, 0x1a, 0xf7, 0xa1, 0xe5, 0x78, 0x1e, 0xe6, 0xcb, 0x9d, 0xf1, 0x01, 0xf6, 0x92, 0x00, + 0x92, 0x87, 0x9a, 0xff, 0x51, 0x80, 0xf6, 0x1e, 0x09, 0xaf, 0xbe, 0xc2, 0x63, 0x94, 0x89, 0x6e, + 0x42, 0x48, 0x95, 0xb1, 0xf1, 0x6f, 0x5e, 0x85, 0x9c, 0xe1, 0x31, 0x92, 0xc7, 0x5e, 0x7a, 0x5d, + 0x95, 0x03, 0xc4, 0x91, 0xd7, 0x93, 0x49, 0x3b, 0xb5, 0x29, 0x27, 0x8f, 0x89, 0x27, 0xea, 0x2d, + 0x0f, 0x53, 0x3b, 0x69, 0x9e, 0x36, 0xad, 0x8a, 0x87, 0xa9, 0x98, 0x52, 0x8a, 0x2c, 0x89, 0xe6, + 0x78, 0x56, 0x91, 0x65, 0x09, 0xe1, 0x8a, 0xac, 0xc3, 0x32, 0x39, 0x3b, 0x8b, 0x10, 0x13, 0x7b, + 0x55, 0xb2, 0xd4, 0x28, 0x09, 0xc1, 0xd5, 0x4c, 0x08, 0x5e, 0x13, 0xf7, 0xda, 0xf3, 0xe7, 0xc7, + 0xfb, 0x17, 0x28, 0x60, 0xfa, 0x06, 0x7e, 0x04, 0x55, 0x0d, 0xfa, 0x2d, 0x6d, 0xe7, 0xf7, 0xa0, + 0xb5, 0xe3, 0x79, 0x83, 0x97, 0x4e, 0xa8, 0xed, 0xd1, 0x85, 0x4a, 0x7f, 0xef, 0xa8, 0x2f, 0x4d, + 0x52, 0xe2, 0x0a, 0xa8, 0x21, 0xbf, 0xf1, 0x0f, 0x10, 0x3b, 0x46, 0x8c, 0x62, 0x37, 0xb9, 0xf1, + 0xef, 0x42, 0x45, 0x41, 0xf8, 0x4a, 0x5f, 0x7e, 0xea, 0x2b, 0x40, 0x0d, 0xcd, 0x3f, 0x01, 0xe3, + 0x7b, 0x9e, 0x2f, 0x23, 0x59, 0x2c, 0x29, 0x4e, 0xef, 0x41, 0xe7, 0x42, 0x40, 0x6d, 0x99, 0x48, + 0x66, 0xb6, 0xa1, 0x2d, 0x27, 0x44, 0x7c, 0x10, 0xbc, 0x4f, 0x60, 0x55, 0xa6, 0xf7, 0x92, 0xce, + 0x35, 0x48, 0x70, 0x1b, 0x26, 0xfb, 0x59, 0xb6, 0xc4, 0xf7, 0xe3, 0xbf, 0x5b, 0x53, 0xd7, 0x98, + 0xea, 0x26, 0x19, 0x07, 0xd0, 0x9e, 0x78, 0xfa, 0x33, 0x54, 0x7b, 0x71, 0xf6, 0x8b, 0x60, 0x6f, + 0x7d, 0x4b, 0x3e, 0x25, 0x6e, 0xe9, 0xa7, 0xc4, 0xad, 0x7d, 0x3f, 0x64, 0x57, 0xc6, 0x3e, 0xb4, + 0xf2, 0x8f, 0x64, 0xc6, 0x5b, 0x3a, 0xc5, 0x9a, 0xf1, 0x74, 0x36, 0x97, 0xcc, 0x01, 0xb4, 0x27, + 0xde, 0xcb, 0xb4, 0x3c, 0xb3, 0x9f, 0xd1, 0xe6, 0x12, 0x7a, 0x0a, 0xf5, 0xcc, 0x03, 0x99, 0xd1, + 0x95, 0x44, 0xa6, 0xdf, 0xcc, 0xe6, 0x12, 0xd8, 0x83, 0x66, 0xee, 0xcd, 0xca, 0xe8, 0x29, 0x7d, + 0x66, 0x3c, 0x64, 0xcd, 0x25, 0xb2, 0x0b, 0xf5, 0xcc, 0xd3, 0x91, 0x96, 0x62, 0xfa, 0x7d, 0xaa, + 0x77, 0x6b, 0xc6, 0x8c, 0xba, 0x2d, 0x0f, 0xa0, 0x3d, 0xf1, 0x9e, 0xa4, 0x4d, 0x32, 0xfb, 0x99, + 0x69, 0xae, 0x30, 0x03, 0xb8, 0x31, 0x33, 0x4b, 0x36, 0xcc, 0x2c, 0xb9, 0xd9, 0x29, 0xf4, 0x5c, + 0xa2, 0xdf, 0x88, 0x7d, 0xcf, 0xf4, 0x20, 0x32, 0xfb, 0x3e, 0xfd, 0x24, 0xd5, 0xbb, 0x3d, 0x7b, + 0x52, 0xa9, 0xba, 0x0f, 0xad, 0xfc, 0x6b, 0x94, 0x26, 0x36, 0xf3, 0x8d, 0x6a, 0xb1, 0x13, 0xe5, + 0x1e, 0xa6, 0x52, 0x27, 0x9a, 0xf5, 0x5e, 0x35, 0x97, 0x10, 0x82, 0x8d, 0xc5, 0x75, 0x97, 0xf1, + 0x7e, 0xd6, 0x39, 0x5f, 0x53, 0x9d, 0xcd, 0x65, 0xb3, 0x03, 0xa0, 0x1a, 0x1b, 0x1e, 0x0e, 0x12, + 0x27, 0x99, 0x6a, 0xa8, 0x24, 0x4e, 0x32, 0xa3, 0x09, 0xf2, 0x14, 0x40, 0xf6, 0x23, 0x3c, 0x12, + 0x33, 0xe3, 0xa6, 0x96, 0x6a, 0xa2, 0x09, 0xd2, 0xeb, 0x4e, 0x4f, 0x4c, 0x11, 0x40, 0x94, 0x5e, + 0x87, 0xc0, 0x97, 0x00, 0x69, 0x9f, 0x43, 0x13, 0x98, 0xea, 0x7c, 0x2c, 0xb0, 0x41, 0x23, 0xdb, + 0xd5, 0x30, 0x94, 0xae, 0x33, 0x3a, 0x1d, 0x0b, 0x48, 0xb4, 0x27, 0x8a, 0xd1, 0xfc, 0x41, 0x99, + 0xac, 0x51, 0x7b, 0x53, 0x05, 0xa9, 0xf1, 0x04, 0x1a, 0xd9, 0x2a, 0x54, 0x4b, 0x31, 0xa3, 0x32, + 0xed, 0xe5, 0x2a, 0x51, 0xe3, 0x29, 0xb4, 0xf2, 0x65, 0xa3, 0xf6, 0xdc, 0x99, 0xc5, 0x64, 0x4f, + 0x35, 0x62, 0x33, 0xe8, 0x1f, 0x01, 0xa4, 0xe5, 0xa5, 0x36, 0xdf, 0x54, 0xc1, 0x39, 0xc1, 0xf5, + 0x00, 0xda, 0x13, 0x65, 0xa3, 0xd6, 0x78, 0x76, 0x35, 0xb9, 0x28, 0x4e, 0x65, 0x8a, 0x40, 0xed, + 0x82, 0xd3, 0x65, 0xa4, 0x76, 0xc1, 0x59, 0x15, 0xe3, 0x2e, 0xd4, 0x07, 0xd3, 0x34, 0x06, 0x73, + 0x69, 0xcc, 0xaa, 0x03, 0x3f, 0x06, 0x48, 0xaf, 0x5c, 0x6d, 0x85, 0xa9, 0x4b, 0xb8, 0xd7, 0xd4, + 0xcd, 0x72, 0x89, 0xb7, 0x07, 0xcd, 0x5c, 0x3f, 0x49, 0x87, 0xea, 0x59, 0x4d, 0xa6, 0x45, 0x17, + 0x58, 0xbe, 0xf9, 0xa2, 0x77, 0x70, 0x66, 0x4b, 0x66, 0x91, 0x1f, 0x67, 0xab, 0x56, 0xed, 0x41, + 0x33, 0x2a, 0xd9, 0xd7, 0x84, 0xaf, 0x6c, 0x65, 0x9a, 0x09, 0x5f, 0x33, 0x0a, 0xd6, 0xb9, 0x84, + 0x0e, 0xa1, 0x7d, 0xa0, 0x8b, 0x0e, 0x55, 0x10, 0xe9, 0xfd, 0x9b, 0x2e, 0x00, 0x7b, 0xbd, 0x59, + 0x53, 0x6a, 0x5f, 0xbe, 0x81, 0xce, 0x54, 0x31, 0x64, 0x6c, 0x24, 0x4f, 0x16, 0x33, 0xab, 0xa4, + 0xb9, 0x62, 0x1d, 0xc1, 0xca, 0x64, 0x2d, 0x64, 0xfc, 0x3e, 0xf1, 0x89, 0x59, 0x35, 0xd2, 0x5c, + 0x52, 0x9f, 0x41, 0x55, 0xe7, 0xb7, 0x86, 0x7a, 0x1a, 0x9a, 0xc8, 0x77, 0xe7, 0x2e, 0x7d, 0x22, + 0x5c, 0x3e, 0xc9, 0x1d, 0x53, 0x97, 0x9f, 0xc8, 0x30, 0x7b, 0xea, 0x25, 0x27, 0xc1, 0x7c, 0x02, + 0x15, 0x95, 0x42, 0x1a, 0x6b, 0xc9, 0x61, 0xcb, 0x64, 0x94, 0x8b, 0x3c, 0xec, 0x00, 0xb1, 0x4c, + 0x62, 0xa8, 0x99, 0x4e, 0xe7, 0x8a, 0xfa, 0x8c, 0xe4, 0x66, 0xd4, 0x5e, 0xec, 0x40, 0x23, 0x9b, + 0x1a, 0xea, 0x2d, 0x9d, 0x91, 0x2e, 0xce, 0x93, 0x64, 0xf7, 0xf2, 0x97, 0x5f, 0x37, 0x7e, 0xf7, + 0x5f, 0xbf, 0x6e, 0xfc, 0xee, 0x6f, 0x5f, 0x6d, 0x14, 0x7e, 0x79, 0xb5, 0x51, 0xf8, 0xcf, 0x57, + 0x1b, 0x85, 0xff, 0x7d, 0xb5, 0x51, 0xf8, 0xf3, 0xbf, 0x1a, 0x62, 0x36, 0x8a, 0x4f, 0xb7, 0x5c, + 0xe2, 0x6f, 0x9f, 0x3b, 0xcc, 0x79, 0x94, 0x24, 0xcf, 0xd1, 0xd4, 0x38, 0xa2, 0xee, 0x36, 0x8d, + 0x03, 0x86, 0x7d, 0xb4, 0x7d, 0x81, 0x29, 0xcb, 0x4c, 0x85, 0xe7, 0xc3, 0x6d, 0x51, 0x88, 0xcb, + 0x7f, 0x9c, 0xb9, 0x64, 0x1c, 0x6d, 0x73, 0x29, 0x4f, 0x97, 0xc5, 0xf8, 0xa3, 0xff, 0x0f, 0x00, + 0x00, 0xff, 0xff, 0x76, 0x47, 0x9c, 0x5b, 0xc7, 0x26, 0x00, 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -4741,6 +4779,33 @@ func (m *DestroySandboxRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *RemoveStaleVirtiofsShareMountsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RemoveStaleVirtiofsShareMountsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RemoveStaleVirtiofsShareMountsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + func (m *Interfaces) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6917,6 +6982,18 @@ func (m *DestroySandboxRequest) Size() (n int) { return n } +func (m *RemoveStaleVirtiofsShareMountsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *Interfaces) Size() (n int) { if m == nil { return 0 @@ -8056,6 +8133,16 @@ func (this *DestroySandboxRequest) String() string { }, "") return s } +func (this *RemoveStaleVirtiofsShareMountsRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RemoveStaleVirtiofsShareMountsRequest{`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func (this *Interfaces) String() string { if this == nil { return "nil" @@ -8477,6 +8564,7 @@ type AgentServiceService interface { StatsContainer(ctx context.Context, req *StatsContainerRequest) (*StatsContainerResponse, error) PauseContainer(ctx context.Context, req *PauseContainerRequest) (*types.Empty, error) ResumeContainer(ctx context.Context, req *ResumeContainerRequest) (*types.Empty, error) + RemoveStaleVirtiofsShareMounts(ctx context.Context, req *RemoveStaleVirtiofsShareMountsRequest) (*types.Empty, error) WriteStdin(ctx context.Context, req *WriteStreamRequest) (*WriteStreamResponse, error) ReadStdout(ctx context.Context, req *ReadStreamRequest) (*ReadStreamResponse, error) ReadStderr(ctx context.Context, req *ReadStreamRequest) (*ReadStreamResponse, error) @@ -8583,6 +8671,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag } return svc.ResumeContainer(ctx, &req) }, + "RemoveStaleVirtiofsShareMounts": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req RemoveStaleVirtiofsShareMountsRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.RemoveStaleVirtiofsShareMounts(ctx, &req) + }, "WriteStdin": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { var req WriteStreamRequest if err := unmarshal(&req); err != nil { @@ -8859,6 +8954,14 @@ func (c *agentServiceClient) ResumeContainer(ctx context.Context, req *ResumeCon return &resp, nil } +func (c *agentServiceClient) RemoveStaleVirtiofsShareMounts(ctx context.Context, req *RemoveStaleVirtiofsShareMountsRequest) (*types.Empty, error) { + var resp types.Empty + if err := c.client.Call(ctx, "grpc.AgentService", "RemoveStaleVirtiofsShareMounts", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + func (c *agentServiceClient) WriteStdin(ctx context.Context, req *WriteStreamRequest) (*WriteStreamResponse, error) { var resp WriteStreamResponse if err := c.client.Call(ctx, "grpc.AgentService", "WriteStdin", req, &resp); err != nil { @@ -13724,6 +13827,57 @@ func (m *DestroySandboxRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *RemoveStaleVirtiofsShareMountsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RemoveStaleVirtiofsShareMountsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RemoveStaleVirtiofsShareMountsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Interfaces) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index 750c1e725..14ed74f2f 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -415,10 +415,17 @@ components: VmmPingResponse: description: Virtual Machine Monitor information example: + build_version: build_version + pid: 0 version: version properties: + build_version: + type: string version: type: string + pid: + format: int64 + type: integer required: - version type: object diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md index f48f42989..29abc3c88 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmmPingResponse.md @@ -4,7 +4,9 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**BuildVersion** | Pointer to **string** | | [optional] **Version** | **string** | | +**Pid** | Pointer to **int64** | | [optional] ## Methods @@ -25,6 +27,31 @@ NewVmmPingResponseWithDefaults instantiates a new VmmPingResponse object This constructor will only assign default values to properties that have it defined, but it doesn't guarantee that properties required by API are set +### GetBuildVersion + +`func (o *VmmPingResponse) GetBuildVersion() string` + +GetBuildVersion returns the BuildVersion field if non-nil, zero value otherwise. + +### GetBuildVersionOk + +`func (o *VmmPingResponse) GetBuildVersionOk() (*string, bool)` + +GetBuildVersionOk returns a tuple with the BuildVersion field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetBuildVersion + +`func (o *VmmPingResponse) SetBuildVersion(v string)` + +SetBuildVersion sets BuildVersion field to given value. + +### HasBuildVersion + +`func (o *VmmPingResponse) HasBuildVersion() bool` + +HasBuildVersion returns a boolean if a field has been set. + ### GetVersion `func (o *VmmPingResponse) GetVersion() string` @@ -45,6 +72,31 @@ and a boolean to check if the value has been set. SetVersion sets Version field to given value. +### GetPid + +`func (o *VmmPingResponse) GetPid() int64` + +GetPid returns the Pid field if non-nil, zero value otherwise. + +### GetPidOk + +`func (o *VmmPingResponse) GetPidOk() (*int64, bool)` + +GetPidOk returns a tuple with the Pid field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetPid + +`func (o *VmmPingResponse) SetPid(v int64)` + +SetPid sets Pid field to given value. + +### HasPid + +`func (o *VmmPingResponse) HasPid() bool` + +HasPid returns a boolean if a field has been set. + [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go index 63c7b0f50..314d7da33 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vmm_ping_response.go @@ -16,7 +16,9 @@ import ( // VmmPingResponse Virtual Machine Monitor information type VmmPingResponse struct { - Version string `json:"version"` + BuildVersion *string `json:"build_version,omitempty"` + Version string `json:"version"` + Pid *int64 `json:"pid,omitempty"` } // NewVmmPingResponse instantiates a new VmmPingResponse object @@ -37,6 +39,38 @@ func NewVmmPingResponseWithDefaults() *VmmPingResponse { return &this } +// GetBuildVersion returns the BuildVersion field value if set, zero value otherwise. +func (o *VmmPingResponse) GetBuildVersion() string { + if o == nil || o.BuildVersion == nil { + var ret string + return ret + } + return *o.BuildVersion +} + +// GetBuildVersionOk returns a tuple with the BuildVersion field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *VmmPingResponse) GetBuildVersionOk() (*string, bool) { + if o == nil || o.BuildVersion == nil { + return nil, false + } + return o.BuildVersion, true +} + +// HasBuildVersion returns a boolean if a field has been set. +func (o *VmmPingResponse) HasBuildVersion() bool { + if o != nil && o.BuildVersion != nil { + return true + } + + return false +} + +// SetBuildVersion gets a reference to the given string and assigns it to the BuildVersion field. +func (o *VmmPingResponse) SetBuildVersion(v string) { + o.BuildVersion = &v +} + // GetVersion returns the Version field value func (o *VmmPingResponse) GetVersion() string { if o == nil { @@ -61,11 +95,49 @@ func (o *VmmPingResponse) SetVersion(v string) { o.Version = v } +// GetPid returns the Pid field value if set, zero value otherwise. +func (o *VmmPingResponse) GetPid() int64 { + if o == nil || o.Pid == nil { + var ret int64 + return ret + } + return *o.Pid +} + +// GetPidOk returns a tuple with the Pid field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *VmmPingResponse) GetPidOk() (*int64, bool) { + if o == nil || o.Pid == nil { + return nil, false + } + return o.Pid, true +} + +// HasPid returns a boolean if a field has been set. +func (o *VmmPingResponse) HasPid() bool { + if o != nil && o.Pid != nil { + return true + } + + return false +} + +// SetPid gets a reference to the given int64 and assigns it to the Pid field. +func (o *VmmPingResponse) SetPid(v int64) { + o.Pid = &v +} + func (o VmmPingResponse) MarshalJSON() ([]byte, error) { toSerialize := map[string]interface{}{} + if o.BuildVersion != nil { + toSerialize["build_version"] = o.BuildVersion + } if true { toSerialize["version"] = o.Version } + if o.Pid != nil { + toSerialize["pid"] = o.Pid + } return json.Marshal(toSerialize) } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index dee8bdbf0..57be27c82 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -438,8 +438,13 @@ components: - version type: object properties: + build_version: + type: string version: type: string + pid: + type: integer + format: int64 description: Virtual Machine Monitor information VmInfo: diff --git a/src/runtime/virtcontainers/pkg/compatoci/utils.go b/src/runtime/virtcontainers/pkg/compatoci/utils.go index 01568dd45..e8ea62911 100644 --- a/src/runtime/virtcontainers/pkg/compatoci/utils.go +++ b/src/runtime/virtcontainers/pkg/compatoci/utils.go @@ -74,7 +74,7 @@ func containerCapabilities(s compatOCISpec) (specs.LinuxCapabilities, error) { } default: - return c, fmt.Errorf("Unexpected format for capabilities: %v", caps) + return c, fmt.Errorf("unexpected format for capabilities: %v", caps) } } case []interface{}: @@ -94,7 +94,7 @@ func containerCapabilities(s compatOCISpec) (specs.LinuxCapabilities, error) { ociLog.Debug("Empty capabilities have been passed") return c, nil default: - return c, fmt.Errorf("Unexpected format for capabilities: %v", caps) + return c, fmt.Errorf("unexpected format for capabilities: %v", caps) } return c, nil @@ -154,5 +154,5 @@ func GetContainerSpec(annotations map[string]string) (specs.Spec, error) { ociLog.Errorf("Annotations[%s] not found, cannot find container spec", vcAnnotations.BundlePathKey) - return specs.Spec{}, fmt.Errorf("Could not find container spec") + return specs.Spec{}, fmt.Errorf("could not find container spec") } diff --git a/src/runtime/virtcontainers/pkg/compatoci/utils_test.go b/src/runtime/virtcontainers/pkg/compatoci/utils_test.go index ed8bdf864..aa44500bb 100644 --- a/src/runtime/virtcontainers/pkg/compatoci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/compatoci/utils_test.go @@ -7,6 +7,8 @@ package compatoci import ( "encoding/json" + "os" + "path" "path/filepath" "testing" @@ -78,6 +80,18 @@ func TestContainerCapabilities(t *testing.T) { assert.Equal(t, c.Inheritable, []string{"CAP_KILL", "CAP_LEASE", "CAP_SYS_ADMIN"}) assert.Equal(t, c.Ambient, []string{""}) + ociSpec.Process.Capabilities = map[string]interface{}{ + "unexpected": interface{}(""), + } + + c, err = ContainerCapabilities(ociSpec) + assert.NotNil(t, err) + assert.Equal(t, len(c.Bounding), 0) + assert.Equal(t, len(c.Effective), 0) + assert.Equal(t, len(c.Permitted), 0) + assert.Equal(t, len(c.Inheritable), 0) + assert.Equal(t, len(c.Ambient), 0) + ociSpec.Process.Capabilities = []interface{}{"CAP_LEASE", "CAP_SETUID"} c, err = ContainerCapabilities(ociSpec) @@ -92,11 +106,31 @@ func TestContainerCapabilities(t *testing.T) { c, err = ContainerCapabilities(ociSpec) assert.Nil(t, err) - assert.Equal(t, c.Bounding, []string(nil)) - assert.Equal(t, c.Effective, []string(nil)) - assert.Equal(t, c.Permitted, []string(nil)) - assert.Equal(t, c.Inheritable, []string(nil)) - assert.Equal(t, c.Ambient, []string(nil)) + assert.Equal(t, len(c.Bounding), 0) + assert.Equal(t, len(c.Effective), 0) + assert.Equal(t, len(c.Permitted), 0) + assert.Equal(t, len(c.Inheritable), 0) + assert.Equal(t, len(c.Ambient), 0) + + ociSpec.Process.Capabilities = interface{}("") + + c, err = ContainerCapabilities(ociSpec) + assert.NotNil(t, err) + assert.Equal(t, len(c.Bounding), 0) + assert.Equal(t, len(c.Effective), 0) + assert.Equal(t, len(c.Permitted), 0) + assert.Equal(t, len(c.Inheritable), 0) + assert.Equal(t, len(c.Ambient), 0) + + ociSpec.Process = nil + + c, err = ContainerCapabilities(ociSpec) + assert.NotNil(t, err) + assert.Equal(t, len(c.Bounding), 0) + assert.Equal(t, len(c.Effective), 0) + assert.Equal(t, len(c.Permitted), 0) + assert.Equal(t, len(c.Inheritable), 0) + assert.Equal(t, len(c.Ambient), 0) } // use specs.Spec to decode the spec, the content of capabilities is [] string @@ -145,3 +179,55 @@ func TestGetConfigPath(t *testing.T) { configPath := getConfigPath(tempBundlePath) assert.Equal(t, configPath, expected) } + +func TestParseConfigJSON(t *testing.T) { + tmpDir := t.TempDir() + + var ociSpec compatOCISpec + var configByte []byte + + ociSpec.Spec.Version = "1.0.0" + ociSpec.Process = &compatOCIProcess{} + ociSpec.Process.Capabilities = map[string]interface{}{ + "bounding": []interface{}{"CAP_KILL"}, + "effective": []interface{}{"CAP_KILL", "CAP_LEASE"}, + "permitted": []interface{}{"CAP_SETUID"}, + "inheritable": []interface{}{"CAP_KILL", "CAP_LEASE", "CAP_SYS_ADMIN"}, + "ambient": []interface{}{""}, + } + + file, err := os.Create(path.Join(tmpDir, "config.json")) + if err != nil { + t.Error("cannot create file") + } + defer file.Close() + + if configByte, err = json.Marshal(ociSpec); err != nil { + t.Error("cannot marshal compat oci spec") + } + + _, err = file.Write(configByte) + if err != nil { + t.Error("cannot write config data into file") + } + + spec, err := ParseConfigJSON(tmpDir) + assert.Nil(t, err) + assert.Equal(t, spec.Version, "1.0.0") +} + +func TestGetContainerSpec(t *testing.T) { + annotations := map[string]string{ + "io.katacontainers.pkg.oci.bundle_path": "", + } + + _, err := GetContainerSpec(annotations) + assert.NotNil(t, err) + + annotations = map[string]string{ + "io.katacontainers.pkg.oci.wrong_path": "", + } + + _, err = GetContainerSpec(annotations) + assert.NotNil(t, err) +} diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go index b56bfabb6..2e38cd65e 100644 --- a/src/runtime/virtcontainers/pkg/mock/mock.go +++ b/src/runtime/virtcontainers/pkg/mock/mock.go @@ -245,6 +245,10 @@ func (p *HybridVSockTTRPCMockImp) ResizeVolume(ctx context.Context, req *pb.Resi return &gpb.Empty{}, nil } +func (p *HybridVSockTTRPCMockImp) RemoveStaleVirtiofsShareMounts(ctx context.Context, req *pb.RemoveStaleVirtiofsShareMountsRequest) (*gpb.Empty, error) { + return &gpb.Empty{}, nil +} + func (p *HybridVSockTTRPCMockImp) PullImage(ctx context.Context, req *pb.PullImageRequest) (*gpb.Empty, error) { return &gpb.Empty{}, nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 744fa9716..4a23777dc 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -84,6 +84,7 @@ type QemuState struct { VirtiofsDaemonPid int PCIeRootPort int HotplugVFIOOnRootBus bool + ColdPlugVFIO hv.PCIePort } // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. @@ -278,6 +279,7 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso q.Logger().Debug("Creating UUID") q.state.UUID = uuid.Generate().String() + q.state.ColdPlugVFIO = q.config.ColdPlugVFIO q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus q.state.PCIeRootPort = int(q.config.PCIeRootPort) @@ -733,6 +735,18 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort, memSize32bit, memSize64bit) } + // The default OVMF MMIO aperture is too small for some PCIe devices + // with huge BARs so we need to increase it. + // memSize64bit is in bytes, convert to MB, OVMF expects MB as a string + if strings.Contains(strings.ToLower(hypervisorConfig.FirmwarePath), "ovmf") { + pciMmio64Mb := fmt.Sprintf("%d", (memSize64bit / 1024 / 1024)) + fwCfg := govmmQemu.FwCfg{ + Name: "opt/ovmf/X-PciMmio64Mb", + Str: pciMmio64Mb, + } + qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg) + } + q.qemuConfig = qemuConfig q.virtiofsDaemon, err = q.createVirtiofsDaemon(hypervisorConfig.SharedPath) diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index a8bc6a33d..bfa348145 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -76,24 +76,10 @@ func TestQemuKernelParameters(t *testing.T) { } func TestQemuCreateVM(t *testing.T) { - qemuConfig := newQemuConfig() assert := assert.New(t) store, err := persist.GetDriver() assert.NoError(err) - q := &qemu{ - config: HypervisorConfig{ - VMStorePath: store.RunVMStoragePath(), - RunStorePath: store.RunStoragePath(), - }, - } - sandbox := &Sandbox{ - ctx: context.Background(), - id: "testSandbox", - config: &SandboxConfig{ - HypervisorConfig: qemuConfig, - }, - } // Create the hypervisor fake binary testQemuPath := filepath.Join(testDir, testHypervisor) @@ -101,15 +87,142 @@ func TestQemuCreateVM(t *testing.T) { assert.NoError(err) // Create parent dir path for hypervisor.json - parentDir := filepath.Join(store.RunStoragePath(), sandbox.id) + parentDir := filepath.Join(store.RunStoragePath(), "testSandbox") assert.NoError(os.MkdirAll(parentDir, DirMode)) network, err := NewNetwork() assert.NoError(err) - err = q.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig) - assert.NoError(err) + + config0 := newQemuConfig() + + config1 := newQemuConfig() + config1.SeccompSandbox = "enable=1" + + config2 := newQemuConfig() + config2.InitrdPath = "" + config2.ImagePath = testQemuImagePath + + config3 := newQemuConfig() + config3.Debug = true + + config5 := newQemuConfig() + config5.GuestMemoryDumpPath = "/tmp" + + config6 := newQemuConfig() + config6.DisableGuestSeLinux = false + + config7 := newQemuConfig() + config7.PCIeRootPort = 1 + + config8 := newQemuConfig() + config8.EnableVhostUserStore = true + config8.HugePages = true + + config9 := newQemuConfig() + config9.EnableVhostUserStore = true + config9.HugePages = false + + config10 := newQemuConfig() + config10.BootToBeTemplate = true + + config11 := newQemuConfig() + config11.BootFromTemplate = true + + config12 := newQemuConfig() + config12.BootToBeTemplate = true + config12.SharedFS = config.VirtioFS + + config13 := newQemuConfig() + config13.FileBackedMemRootDir = "/tmp/xyzabc" + config13.HugePages = true + + config14 := newQemuConfig() + config14.SharedFS = config.VirtioFS + + config15 := newQemuConfig() + config15.BlockDeviceDriver = "" + + config16 := newQemuConfig() + config16.SharedFS = config.VirtioFSNydus + + config17 := newQemuConfig() + config17.VMid = "testSandbox" + + type testData struct { + config HypervisorConfig + expectError bool + configMatch bool + } + + data := []testData{ + {config0, false, true}, + {config1, false, true}, + {config2, false, true}, + {config3, false, true}, + {config5, false, true}, + {config6, false, false}, + {config7, false, true}, + {config8, false, true}, + {config9, true, false}, + {config10, false, true}, + {config11, false, true}, + {config12, true, false}, + {config13, false, true}, + {config14, false, true}, + {config15, false, true}, + {config16, false, true}, + {config17, false, true}, + } + + for i, d := range data { + msg := fmt.Sprintf("test[%d]", i) + + q := &qemu{ + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, + } + + err = q.CreateVM(context.Background(), "testSandbox", network, &d.config) + + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + + if d.configMatch { + assert.Exactly(d.config, q.config, msg) + } + + mem := q.GetTotalMemoryMB(context.Background()) + assert.True(mem > 0) + + err = q.canDumpGuestMemory("/tmp") + assert.NoError(err) + + err = q.dumpGuestMemory("") + assert.NoError(err) + + q.dumpSandboxMetaInfo("/tmp/") + + // now we exercise code that should fail since the VM isn't running + err = q.dumpGuestMemory("/tmp") + assert.Error(err) + + err = q.setupVirtioMem(context.Background()) + assert.Error(err) + + err = q.SaveVM() + assert.Error(err) + + err = q.StopVM(context.Background(), true) + assert.Error(err) + } + assert.NoError(os.RemoveAll(parentDir)) - assert.Exactly(qemuConfig, q.config) } func TestQemuCreateVMMissingParentDirFail(t *testing.T) { @@ -636,3 +749,27 @@ func TestQemuSetConfig(t *testing.T) { assert.Equal(q.config, config) } + +func TestQemuStartSandbox(t *testing.T) { + assert := assert.New(t) + + sandbox, err := createQemuSandboxConfig() + assert.NoError(err) + + network, err := NewNetwork() + assert.NoError(err) + + q := &qemu{ + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, + virtiofsDaemon: &virtiofsdMock{}, + } + + err = q.CreateVM(context.Background(), sandbox.id, network, &sandbox.config.HypervisorConfig) + assert.NoError(err) + + err = q.StartVM(context.Background(), 10) + assert.Error(err) +} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index da5e0ab3f..b0697fd84 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -17,8 +17,6 @@ import ( "os" "os/exec" "path/filepath" - - //"strconv" "strings" "sync" "syscall" @@ -30,16 +28,14 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - cri "github.com/containerd/containerd/pkg/cri/annotations" - crio "github.com/containers/podman/v4/pkg/annotations" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" @@ -138,38 +134,52 @@ type SandboxResourceSizing struct { // SandboxConfig is a Sandbox configuration. type SandboxConfig struct { // Annotations keys must be unique strings and must be name-spaced - // with e.g. reverse domain notation (org.clearlinux.key). - Annotations map[string]string - Hostname string - ID string - HypervisorType HypervisorType + Annotations map[string]string + // Custom SELinux security policy to the container process inside the VM GuestSeLinuxLabel string - // Volumes is a list of shared volumes between the host and the Sandbox. - Volumes []types.Volume + + HypervisorType HypervisorType + + ID string + + Hostname string + // SandboxBindMounts - list of paths to mount into guest SandboxBindMounts []string + // Experimental features enabled Experimental []exp.Feature + // Containers describe the list of containers within a Sandbox. // This list can be empty and populated by adding containers // to the Sandbox a posteriori. - //TODO: this should be a map to avoid duplicated containers - Containers []ContainerConfig - NetworkConfig NetworkConfig - AgentConfig KataAgentConfig + // TODO: this should be a map to avoid duplicated containers + Containers []ContainerConfig + + Volumes []types.Volume + + NetworkConfig NetworkConfig + + AgentConfig KataAgentConfig + HypervisorConfig HypervisorConfig - ShmSize uint64 + + ShmSize uint64 + SandboxResources SandboxResourceSizing - VfioMode config.VFIOModeType + + VfioMode config.VFIOModeType + // StaticResourceMgmt indicates if the shim should rely on statically sizing the sandbox (VM) StaticResourceMgmt bool - // Offload the CRI image management service to the Kata agent. - ServiceOffload bool + // SharePidNs sets all containers to share the same sandbox level pid namespace. SharePidNs bool + // SystemdCgroup enables systemd cgroup support SystemdCgroup bool + // SandboxCgroupOnly enables cgroup only at podlevel in the host SandboxCgroupOnly bool @@ -329,7 +339,6 @@ func (s *Sandbox) Release(ctx context.Context) error { if s.monitor != nil { s.monitor.stop() } - s.fsShare.StopFileEventWatcher(ctx) s.hypervisor.Disconnect(ctx) return s.agent.disconnect(ctx) } @@ -612,17 +621,21 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor return nil, err } - if len(sandboxConfig.Containers) > 0 { - // These values are required by remove hypervisor - for _, a := range []string{cri.SandboxName, crio.SandboxName} { - if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok { - sandboxConfig.HypervisorConfig.SandboxName = value - } - } - - for _, a := range []string{cri.SandboxNamespace, crio.Namespace} { - if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok { - sandboxConfig.HypervisorConfig.SandboxNamespace = value + // If we have a confidential guest we need to cold-plug the PCIe VFIO devices + // until we have TDISP/IDE PCIe support. + coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort) + var devs []config.DeviceInfo + for cnt, containers := range sandboxConfig.Containers { + for dev, device := range containers.DeviceInfos { + if coldPlugVFIO && deviceManager.IsVFIO(device.ContainerPath) { + device.ColdPlug = true + devs = append(devs, device) + // We need to remove the devices marked for cold-plug + // otherwise at the container level the kata-agent + // will try to hot-plug them. + infos := sandboxConfig.Containers[cnt].DeviceInfos + infos = append(infos[:dev], infos[dev+1:]...) + sandboxConfig.Containers[cnt].DeviceInfos = infos } } } @@ -636,6 +649,17 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor return nil, err } + if !coldPlugVFIO { + return s, nil + } + + for _, dev := range devs { + _, err := s.AddDevice(ctx, dev) + if err != nil { + s.Logger().WithError(err).Debug("Cannot cold-plug add device") + return nil, err + } + } return s, nil } @@ -673,6 +697,7 @@ func (s *Sandbox) createResourceController() error { // Determine if device /dev/null and /dev/urandom exist, and add if they don't nullDeviceExist := false urandomDeviceExist := false + ptmxDeviceExist := false for _, device := range resources.Devices { if device.Type == "c" && device.Major == intptr(1) && device.Minor == intptr(3) { nullDeviceExist = true @@ -681,6 +706,10 @@ func (s *Sandbox) createResourceController() error { if device.Type == "c" && device.Major == intptr(1) && device.Minor == intptr(9) { urandomDeviceExist = true } + + if device.Type == "c" && device.Major == intptr(5) && device.Minor == intptr(2) { + ptmxDeviceExist = true + } } if !nullDeviceExist { @@ -696,6 +725,18 @@ func (s *Sandbox) createResourceController() error { }...) } + // If the hypervisor debug console is enabled and + // sandbox_cgroup_only are configured, then the vmm needs access to + // /dev/ptmx. Add this to the device allowlist if it is not + // already present in the config. + if s.config.HypervisorConfig.Debug && s.config.SandboxCgroupOnly && !ptmxDeviceExist { + // "/dev/ptmx" + resources.Devices = append(resources.Devices, []specs.LinuxDeviceCgroup{ + {Type: "c", Major: intptr(5), Minor: intptr(2), Access: rwm, Allow: true}, + }...) + + } + if spec.Linux.Resources.CPU != nil { resources.CPU = &specs.LinuxCPU{ Cpus: spec.Linux.Resources.CPU.Cpus, @@ -1251,13 +1292,6 @@ func (s *Sandbox) startVM(ctx context.Context, prestartHookFunc func(context.Con return err } - // not sure how we know that this callback has been executed - if s.config.HypervisorConfig.ConfidentialGuest && s.config.HypervisorConfig.GuestPreAttestation { - if err := s.hypervisor.AttestVM(ctx); err != nil { - return err - } - } - if prestartHookFunc != nil { hid, err := s.GetHypervisorPid() if err != nil { @@ -2649,8 +2683,3 @@ func (s *Sandbox) resetVCPUsPinning(ctx context.Context, vCPUThreadsMap VcpuThre } return nil } - -// PullImage pulls an image on a sandbox. -func (s *Sandbox) PullImage(ctx context.Context, req *image.PullImageReq) (*image.PullImageResp, error) { - return s.agent.PullImage(ctx, req) -} diff --git a/src/tools/agent-ctl/Makefile b/src/tools/agent-ctl/Makefile index 980e47ccb..fe56fa098 100644 --- a/src/tools/agent-ctl/Makefile +++ b/src/tools/agent-ctl/Makefile @@ -5,6 +5,10 @@ include ../../../utils.mk +ifeq ($(ARCH), ppc64le) + override ARCH = powerpc64le + endif + .DEFAULT_GOAL := default default: build diff --git a/src/tools/agent-ctl/src/main.rs b/src/tools/agent-ctl/src/main.rs index f08c1a884..a56915727 100644 --- a/src/tools/agent-ctl/src/main.rs +++ b/src/tools/agent-ctl/src/main.rs @@ -314,7 +314,7 @@ fn real_main() -> Result<()> { Ok(()) } "connect" => connect(name, args), - _ => return Err(anyhow!(format!("invalid sub-command: {:?}", subcmd))), + _ => Err(anyhow!(format!("invalid sub-command: {:?}", subcmd))), } } diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index 6acf131f0..ed90e1117 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -33,6 +33,46 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-wincon", + "concolor-override", + "concolor-query", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" + +[[package]] +name = "anstyle-parse" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-wincon" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +dependencies = [ + "anstyle", + "windows-sys 0.45.0", +] + [[package]] name = "anyhow" version = "1.0.69" @@ -53,18 +93,7 @@ checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "syn 1.0.107", ] [[package]] @@ -98,7 +127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2" dependencies = [ "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -149,41 +178,60 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "3.2.23" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" dependencies = [ - "atty", - "bitflags", + "clap_builder", "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +dependencies = [ + "anstream", + "anstyle", + "bitflags", "clap_lex", - "indexmap", "once_cell", "strsim", - "termcolor", - "textwrap", ] [[package]] name = "clap_derive" -version = "3.2.18" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" dependencies = [ "heck 0.4.1", - "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] name = "clap_lex" -version = "0.2.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" + +[[package]] +name = "concolor-override" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" + +[[package]] +name = "concolor-query" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" dependencies = [ - "os_str_bytes", + "windows-sys 0.45.0", ] [[package]] @@ -229,7 +277,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -257,6 +305,27 @@ dependencies = [ "libc", ] +[[package]] +name = "errno" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.45.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "fastrand" version = "1.8.0" @@ -358,7 +427,7 @@ checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -437,15 +506,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -455,6 +515,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -587,6 +653,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -602,6 +679,18 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +[[package]] +name = "is-terminal" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.45.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -653,10 +742,12 @@ dependencies = [ "slog-scope", "strum", "strum_macros", + "sys-info", "tempfile", "test-utils", "thiserror", "tokio", + "toml", "url", "vmm-sys-util", ] @@ -694,6 +785,12 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "linux-raw-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" + [[package]] name = "lock_api" version = "0.4.9" @@ -772,7 +869,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -901,7 +998,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -923,12 +1020,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - [[package]] name = "parking_lot" version = "0.11.2" @@ -987,7 +1078,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1018,35 +1109,11 @@ dependencies = [ "nix 0.26.2", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -1089,7 +1156,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1181,9 +1248,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1280,6 +1347,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustix" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b24138615de35e32031d041a09032ef3487a616d901ca4db224e7d557efae2" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.45.0", +] + [[package]] name = "rustls" version = "0.20.8" @@ -1326,7 +1407,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1391,7 +1472,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1436,7 +1517,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1507,9 +1588,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1549,7 +1630,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 1.0.107", ] [[package]] @@ -1563,6 +1644,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "take_mut" version = "0.2.2" @@ -1583,15 +1685,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "test-utils" version = "0.1.0" @@ -1599,12 +1692,6 @@ dependencies = [ "nix 0.24.3", ] -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - [[package]] name = "thiserror" version = "1.0.38" @@ -1622,7 +1709,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1679,31 +1766,30 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ "autocfg", "bytes 1.4.0", "libc", - "memchr", "mio", "num_cpus", "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] @@ -1885,18 +1971,18 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "vmm-sys-util" version = "0.11.1" @@ -1954,7 +2040,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-shared", ] @@ -1988,7 +2074,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2055,15 +2141,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2076,56 +2153,146 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winreg" diff --git a/src/tools/kata-ctl/Cargo.toml b/src/tools/kata-ctl/Cargo.toml index 3de3a532a..141a90a0c 100644 --- a/src/tools/kata-ctl/Cargo.toml +++ b/src/tools/kata-ctl/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" [dependencies] anyhow = "1.0.31" -clap = { version = "3.2.20", features = ["derive", "cargo"] } +clap = { version = "4.1.13", features = ["derive", "cargo"] } serde_json = "1.0.85" thiserror = "1.0.35" privdrop = "0.5.2" @@ -25,6 +25,8 @@ serde = { version = "1.0.149", features = ["derive"] } url = "2.3.1" futures = "0.3.24" base64 = "0.13.0" +toml = "0.5.8" +sys-info = "0.9.1" shim-interface = { path = "../../libs/shim-interface"} kata-types = { path = "../../libs/kata-types" } @@ -37,7 +39,7 @@ libc = "0.2.138" slog = "2.7.0" slog-scope = "4.4.0" hyper = "0.14.20" -tokio = "1.8.0" +tokio = "1.28.1" [target.'cfg(target_arch = "s390x")'.dependencies] reqwest = { version = "0.11", default-features = false, features = ["json", "blocking", "native-tls"] } diff --git a/src/tools/kata-ctl/Cross.toml b/src/tools/kata-ctl/Cross.toml new file mode 100644 index 000000000..88f18e96e --- /dev/null +++ b/src/tools/kata-ctl/Cross.toml @@ -0,0 +1,12 @@ +[target.s390x-unknown-linux-gnu] +pre-build = ["dpkg --add-architecture s390x && apt-get update && apt-get install -y libssl-dev:s390x"] + +[target.aarch64-unknown-linux-musl] +pre-build = ["dpkg --add-architecture arm64 && apt-get update && apt-get install -y libssl-dev:arm64"] + +[target.x86-64-unknown-linux-musl] +pre-build = ["dpkg --add-architecture amd64 && apt-get update && apt-get install -y libssl-dev:amd64"] + +# Powerpc compile seems to be broken, due to `ring` crate not being supported on powerpc. +[target.powerpc64le-unknown-linux-gnu] +pre-build = ["dpkg --add-architecture ppc64le && apt-get update && apt-get install -y libssl-dev:ppc64le"] diff --git a/src/tools/kata-ctl/Makefile b/src/tools/kata-ctl/Makefile index 60abae34b..23ae7ca1e 100644 --- a/src/tools/kata-ctl/Makefile +++ b/src/tools/kata-ctl/Makefile @@ -10,12 +10,13 @@ PROJECT_URL = https://github.com/kata-containers PROJECT_COMPONENT = kata-ctl TARGET = $(PROJECT_COMPONENT) +INSTALL_PATH = $(HOME)/.cargo VERSION_FILE := ./VERSION -VERSION := $(shell grep -v ^\# $(VERSION_FILE)) +export VERSION := $(shell grep -v ^\# $(VERSION_FILE)) COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true) COMMIT_NO_SHORT := $(shell git rev-parse --short HEAD 2>/dev/null || true) -COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO}) +export COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO}) # Exported to allow cargo to see it export KATA_CTL_VERSION := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) @@ -23,7 +24,9 @@ export KATA_CTL_VERSION := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) GENERATED_CODE = src/ops/version.rs GENERATED_REPLACEMENTS= \ - KATA_CTL_VERSION + KATA_CTL_VERSION \ + VERSION \ + COMMIT GENERATED_FILES := $(GENERATED_CODE) @@ -53,7 +56,7 @@ test: @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo test --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) -- --nocapture install: - @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . --root $(INSTALL_PATH) check: standard_rust_check diff --git a/src/tools/kata-ctl/README.md b/src/tools/kata-ctl/README.md index bf908f60d..c37ab9dcd 100644 --- a/src/tools/kata-ctl/README.md +++ b/src/tools/kata-ctl/README.md @@ -27,6 +27,11 @@ $ make $ make install ``` +If you would like to install the tool to a specific directory, then you can provide it through the `INSTALL_PATH` variable. +```bash +$ make install INSTALL_PATH=/path/to/your/custom/install/directory +``` + ## Run the tool ```bash @@ -47,3 +52,34 @@ For a usage statement, run: ```bash $ kata-ctl --help ``` + +## Cross-builds + +For developers that want to build and test the `kata-ctl` tool on various architectures, +the makefile included does have support for that. This would however, require installing +the cross compile toolchain for the target architecture on the host along with required libraries. + +[Cross](https://github.com/cross-rs/cross) is an open source tool that offers zero setup +cross compile and requires no changes to the system installation for cross-compiling +rust binaries. It makes use of docker containers for cross-compilation. + +You can install cross with: +``` +cargo install -f cross +``` + +`cross` relies on `docker` or `podman`. For dependencies take a look at: https://github.com/cross-rs/cross#dependencies + +There is an included `cross` configuration file [Cross.yaml](./Cross.toml) that can be used +to compile `kata-ctl` for various targets. This configuration helps install required +dependencies inside a docker container. + +For example, to compile for target `s390x-unknown-linux-gnu` included in `Cross.yaml` simple run: +``` +cross build --target=s390x-unknown-linux-gnu +``` + +You may also need to add the target on your host system prior to the above step as: +``` +rustup target add s390x-unknown-linux-gnu +``` diff --git a/src/tools/kata-ctl/src/arch/aarch64/mod.rs b/src/tools/kata-ctl/src/arch/aarch64/mod.rs index a6137856f..7eaf3ccb4 100644 --- a/src/tools/kata-ctl/src/arch/aarch64/mod.rs +++ b/src/tools/kata-ctl/src/arch/aarch64/mod.rs @@ -9,6 +9,7 @@ pub use arch_specific::*; mod arch_specific { use crate::check; use crate::types::*; + use crate::utils; use anyhow::Result; use std::path::Path; @@ -37,6 +38,34 @@ mod arch_specific { Ok(()) } + fn normalize_vendor(vendor: &str) -> String { + match vendor { + "0x41" => String::from("ARM Limited"), + _ => String::from("3rd Party Limited"), + } + } + + fn normalize_model(model: &str) -> String { + match model { + "8" => String::from("v8"), + "7" | "7M" | "?(12)" | "?(13)" | "?(14)" | "?(15)" | "?(16)" | "?(17)" => { + String::from("v7") + } + "6" | "6TEJ" => String::from("v6"), + "5" | "5T" | "5TE" | "5TEJ" => String::from("v5"), + "4" | "4T" => String::from("v4"), + "3" => String::from("v3"), + _ => String::from("unknown"), + } + } + + pub fn get_cpu_details() -> Result<(String, String)> { + let (vendor, model) = utils::get_generic_cpu_details(check::PROC_CPUINFO)?; + let norm_vendor = normalize_vendor(&vendor); + let norm_model = normalize_model(&model); + Ok((norm_vendor, norm_model)) + } + pub fn get_checks() -> Option<&'static [CheckItem<'static>]> { Some(CHECK_LIST) } diff --git a/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs b/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs index 8290dbb13..fc849c631 100644 --- a/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs +++ b/src/tools/kata-ctl/src/arch/powerpc64le/mod.rs @@ -9,6 +9,7 @@ pub use arch_specific::*; mod arch_specific { use crate::check; + use crate::utils; use anyhow::Result; pub const ARCH_CPU_VENDOR_FIELD: &str = ""; @@ -24,6 +25,14 @@ mod arch_specific { const PEF_SYS_FIRMWARE_DIR: &str = "/sys/firmware/ultravisor/"; + pub fn get_cpu_details() -> Result<(String, String)> { + utils::get_generic_cpu_details(check::PROC_CPUINFO) + + // TODO: In case of error from get_generic_cpu_details, implement functionality + // to get cpu details specific to powerpc architecture similar + // to the goloang implementation of function getCPUDetails() + } + pub fn available_guest_protection() -> Result { if !Uid::effective().is_root() { return Err(check::ProtectionError::NoPerms); diff --git a/src/tools/kata-ctl/src/arch/s390x/mod.rs b/src/tools/kata-ctl/src/arch/s390x/mod.rs index 7a9940dcf..929e79c92 100644 --- a/src/tools/kata-ctl/src/arch/s390x/mod.rs +++ b/src/tools/kata-ctl/src/arch/s390x/mod.rs @@ -10,6 +10,7 @@ pub use arch_specific::*; mod arch_specific { use crate::check; use crate::types::*; + use crate::utils; use anyhow::{anyhow, Result}; use nix::unistd::Uid; use std::collections::HashMap; @@ -144,6 +145,14 @@ mod arch_specific { Ok(false) } + pub fn get_cpu_details() -> Result<(String, String)> { + utils::get_generic_cpu_details(check::PROC_CPUINFO) + + // TODO: In case of error from get_generic_cpu_details, implement functionality + // to get cpu details specific to s390x architecture similar + // to the goloang implementation of function getS390xCPUDetails() + } + #[allow(dead_code)] // Guest protection is not supported on ARM64. pub fn available_guest_protection() -> Result { diff --git a/src/tools/kata-ctl/src/arch/x86_64/mod.rs b/src/tools/kata-ctl/src/arch/x86_64/mod.rs index 026312624..206c447a9 100644 --- a/src/tools/kata-ctl/src/arch/x86_64/mod.rs +++ b/src/tools/kata-ctl/src/arch/x86_64/mod.rs @@ -12,7 +12,8 @@ mod arch_specific { use crate::check; use crate::check::{GuestProtection, ProtectionError}; use crate::types::*; - use anyhow::{anyhow, Result}; + use crate::utils; + use anyhow::{anyhow, Context, Result}; use nix::unistd::Uid; use std::fs; use std::path::Path; @@ -40,6 +41,12 @@ mod arch_specific { fp: check_kernel_modules, perm: PermissionType::NonPrivileged, }, + CheckItem { + name: CheckType::KvmIsUsable, + descr: "This parameter performs check to see if KVM is usable", + fp: check_kvm_is_usable, + perm: PermissionType::Privileged, + }, ]; static MODULE_LIST: &[KernelModule] = &[ @@ -109,6 +116,19 @@ mod arch_specific { Ok(cpu_flags) } + pub fn get_cpu_details() -> Result<(String, String)> { + utils::get_generic_cpu_details(check::PROC_CPUINFO) + } + + // check if kvm is usable + fn check_kvm_is_usable(_args: &str) -> Result<()> { + println!("INFO: check if kvm is usable: x86_64"); + + let result = check::check_kvm_is_usable_generic(); + + result.context("KVM check failed") + } + pub const TDX_SYS_FIRMWARE_DIR: &str = "/sys/firmware/tdx_seam/"; pub const TDX_CPU_FLAG: &str = "tdx"; pub const SEV_KVM_PARAMETER_PATH: &str = "/sys/module/kvm_amd/parameters/sev"; @@ -225,7 +245,7 @@ mod arch_specific { expected_param_value ); - return Err(anyhow!("{} {}", error_msg, action_msg)); + Err(anyhow!("{} {}", error_msg, action_msg)) } } diff --git a/src/tools/kata-ctl/src/args.rs b/src/tools/kata-ctl/src/args.rs index ff489f82b..cc2dc0513 100644 --- a/src/tools/kata-ctl/src/args.rs +++ b/src/tools/kata-ctl/src/args.rs @@ -23,7 +23,7 @@ pub enum Commands { DirectVolume(DirectVolumeCommand), /// Display settings - Env, + Env(EnvArgument), /// Enter into guest VM by debug console Exec(ExecArguments), @@ -69,6 +69,15 @@ pub enum CheckSubCommand { List, } +#[derive(Debug, Args)] +pub struct EnvArgument { + /// Format output as JSON + #[arg(long)] + pub json: bool, + /// File to write env output to + #[arg(short = 'f', long = "file")] + pub file: Option, +} #[derive(Debug, Args)] pub struct MetricsCommand { #[clap(subcommand)] diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs index dfb9a3b7b..ef1007f86 100644 --- a/src/tools/kata-ctl/src/check.rs +++ b/src/tools/kata-ctl/src/check.rs @@ -6,8 +6,13 @@ // Contains checks that are not architecture-specific use anyhow::{anyhow, Result}; +use nix::fcntl::{open, OFlag}; +use nix::sys::stat::Mode; +use nix::unistd::close; +use nix::{ioctl_write_int_bad, request_code_none}; use reqwest::header::{CONTENT_TYPE, USER_AGENT}; use serde::{Deserialize, Serialize}; +use std::fmt; use thiserror::Error; #[cfg(any(target_arch = "x86_64"))] @@ -46,7 +51,6 @@ pub const GENERIC_CPU_MODEL_FIELD: &str = "model name"; #[allow(dead_code)] pub const PROC_CPUINFO: &str = "/proc/cpuinfo"; -#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))] fn read_file_contents(file_path: &str) -> Result { let contents = std::fs::read_to_string(file_path)?; Ok(contents) @@ -54,7 +58,6 @@ fn read_file_contents(file_path: &str) -> Result { // get_single_cpu_info returns the contents of the first cpu from // the specified cpuinfo file by parsing based on a specified delimiter -#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))] pub fn get_single_cpu_info(cpu_info_file: &str, substring: &str) -> Result { let contents = read_file_contents(cpu_info_file)?; @@ -147,6 +150,19 @@ pub enum GuestProtection { Se, } +impl fmt::Display for GuestProtection { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + GuestProtection::Tdx => write!(f, "tdx"), + GuestProtection::Sev => write!(f, "sev"), + GuestProtection::Snp => write!(f, "snp"), + GuestProtection::Pef => write!(f, "pef"), + GuestProtection::Se => write!(f, "se"), + GuestProtection::NoProtection => write!(f, "none"), + } + } +} + #[allow(dead_code)] #[derive(Error, Debug)] pub enum ProtectionError { @@ -164,6 +180,65 @@ pub fn run_network_checks() -> Result<()> { Ok(()) } +// Set of basic checks for kvm. Architectures should implement more specific checks if needed +#[allow(dead_code)] +pub fn check_kvm_is_usable_generic() -> Result<()> { + // check for root user + if !nix::unistd::Uid::effective().is_root() { + return Err(anyhow!("Will not perform kvm checks as non root user")); + } + + // we do not want to create syscalls to any device besides /dev/kvm + const KVM_DEVICE: &str = "/dev/kvm"; + + // constants specific to kvm ioctls found in kvm.h + const KVM_IOCTL_ID: u8 = 0xAE; + const KVM_CREATE_VM: u8 = 0x01; + const KVM_GET_API_VERSION: u8 = 0x00; + // per kvm api documentation, this number should always be 12 + // https://www.kernel.org/doc/html/latest/virt/kvm/api.html#kvm-get-api-version + const API_VERSION: i32 = 12; + + // open kvm device + // since file is not being created, mode argument is not relevant + let mode = Mode::empty(); + let flags = OFlag::O_RDWR | OFlag::O_CLOEXEC; + let fd = open(KVM_DEVICE, flags, mode)?; + + // check kvm api version + ioctl_write_int_bad!( + kvm_api_version, + request_code_none!(KVM_IOCTL_ID, KVM_GET_API_VERSION) + ); + // 0 is not used but required to produce output + let v = unsafe { kvm_api_version(fd, 0)? }; + if v != API_VERSION { + return Err(anyhow!("KVM API version is not correct")); + } + + // check if you can create vm + ioctl_write_int_bad!( + kvm_create_vm, + request_code_none!(KVM_IOCTL_ID, KVM_CREATE_VM) + ); + // 0 is default machine type + let vmfd = unsafe { kvm_create_vm(fd, 0) }; + let _vmfd = match vmfd { + Ok(vm) => vm, + Err(ref error) if error.to_string() == "EBUSY: Device or resource busy" => { + return Err(anyhow!( + "Another hypervisor is running. KVM_CREATE_VM error: {:?}", + error + )) + } + Err(error) => return Err(anyhow!("Other KVM_CREATE_VM error: {:?}", error)), + }; + + let _ = close(fd); + + Ok(()) +} + fn get_kata_all_releases_by_url(url: &str) -> std::result::Result, reqwest::Error> { let releases: Vec = reqwest::blocking::Client::new() .get(url) @@ -320,6 +395,7 @@ mod tests { use std::fs; use std::io::Write; use tempfile::tempdir; + use test_utils::skip_if_root; #[test] fn test_get_single_cpu_info() { @@ -447,6 +523,16 @@ mod tests { } } + #[test] + fn test_check_kvm_is_usable_generic() { + skip_if_root!(); + #[allow(dead_code)] + let result = check_kvm_is_usable_generic(); + assert!( + result.err().unwrap().to_string() == "Will not perform kvm checks as non root user" + ); + } + #[test] fn test_get_kata_all_releases_by_url() { #[derive(Debug)] diff --git a/src/tools/kata-ctl/src/main.rs b/src/tools/kata-ctl/src/main.rs index 980a94771..fe64f10e1 100644 --- a/src/tools/kata-ctl/src/main.rs +++ b/src/tools/kata-ctl/src/main.rs @@ -17,8 +17,9 @@ use std::process::exit; use args::{Commands, KataCtlCli}; use ops::check_ops::{ - handle_check, handle_env, handle_factory, handle_iptables, handle_metrics, handle_version, + handle_check, handle_factory, handle_iptables, handle_metrics, handle_version, }; +use ops::env_ops::handle_env; use ops::exec_ops::handle_exec; use ops::volume_ops::handle_direct_volume; @@ -29,7 +30,7 @@ fn real_main() -> Result<()> { Commands::Check(args) => handle_check(args), Commands::DirectVolume(args) => handle_direct_volume(args), Commands::Exec(args) => handle_exec(args), - Commands::Env => handle_env(), + Commands::Env(args) => handle_env(args), Commands::Factory => handle_factory(), Commands::Iptables(args) => handle_iptables(args), Commands::Metrics(args) => handle_metrics(args), diff --git a/src/tools/kata-ctl/src/ops.rs b/src/tools/kata-ctl/src/ops.rs index d5d4fe162..f90f55cb2 100644 --- a/src/tools/kata-ctl/src/ops.rs +++ b/src/tools/kata-ctl/src/ops.rs @@ -4,6 +4,7 @@ // pub mod check_ops; +pub mod env_ops; pub mod exec_ops; pub mod version; pub mod volume_ops; diff --git a/src/tools/kata-ctl/src/ops/check_ops.rs b/src/tools/kata-ctl/src/ops/check_ops.rs index f2dbea702..ed418169d 100644 --- a/src/tools/kata-ctl/src/ops/check_ops.rs +++ b/src/tools/kata-ctl/src/ops/check_ops.rs @@ -80,6 +80,9 @@ pub fn handle_check(checkcmd: CheckArgument) -> Result<()> { // run kernel module checks handle_builtin_check(CheckType::KernelModules, "")?; + + // run kvm checks + handle_builtin_check(CheckType::KvmIsUsable, "")?; } CheckSubCommand::NoNetworkChecks => { @@ -107,10 +110,6 @@ pub fn handle_check(checkcmd: CheckArgument) -> Result<()> { Ok(()) } -pub fn handle_env() -> Result<()> { - Ok(()) -} - pub fn handle_factory() -> Result<()> { Ok(()) } diff --git a/src/tools/kata-ctl/src/ops/env_ops.rs b/src/tools/kata-ctl/src/ops/env_ops.rs new file mode 100644 index 000000000..05602e479 --- /dev/null +++ b/src/tools/kata-ctl/src/ops/env_ops.rs @@ -0,0 +1,469 @@ +// Copyright (c) 2022 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// Contains checks that are not architecture-specific + +use crate::arch::arch_specific; +use crate::args::EnvArgument; +use crate::ops::version; +use crate::utils; +use kata_types::config::TomlConfig; + +use anyhow::{anyhow, Context, Result}; +use nix::unistd::Uid; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::{self, Write}; +use std::process::Command; +use sys_info; + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct HostInfo { + #[serde(default)] + available_guest_protection: String, + #[serde(default)] + kernel: String, + #[serde(default)] + architecture: String, + #[serde(default)] + vm_container_capable: bool, + #[serde(default)] + support_vsocks: bool, + #[serde(default)] + distro: DistroInfo, + #[serde(default)] + cpu: CPUInfo, + #[serde(default)] + memory: MemoryInfo, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct DistroInfo { + #[serde(default)] + name: String, + #[serde(default)] + version: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct CPUInfo { + #[serde(default)] + vendor: String, + #[serde(default)] + model: String, + #[serde(default)] + cpus: usize, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct MemoryInfo { + #[serde(default)] + total: u64, + #[serde(default)] + available: u64, + #[serde(default)] + free: u64, +} + +// Semantic version for the output of the command. +// +// XXX: Increment for every change to the output format +// (meaning any change to the EnvInfo type). +const FORMAT_VERSION: &str = "0.0.1-kata-ctl"; + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct MetaInfo { + #[serde(default)] + version: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct VersionInfo { + #[serde(default)] + semver: String, + #[serde(default)] + commit: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct RuntimeConfigInfo { + #[serde(default)] + path: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct RuntimeInfo { + #[serde(default)] + path: String, + #[serde(default)] + guest_selinux_label: String, + #[serde(default)] + pub experimental: Vec, + #[serde(default)] + debug: bool, + #[serde(default)] + trace: bool, + #[serde(default)] + disable_guest_seccomp: bool, + #[serde(default)] + disable_new_net_ns: bool, + #[serde(default)] + sandbox_cgroup_only: bool, + #[serde(default)] + static_sandbox_resource_mgmt: bool, + #[serde(default)] + config: RuntimeConfigInfo, + #[serde(default)] + version: VersionInfo, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct AgentInfo { + #[serde(default)] + debug: bool, + #[serde(default)] + trace: bool, +} +// KernelInfo stores kernel details +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct KernelInfo { + #[serde(default)] + path: String, + #[serde(default)] + parameters: String, +} + +// InitrdInfo stores initrd image details +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct InitrdInfo { + #[serde(default)] + path: String, +} + +// ImageInfo stores root filesystem image details +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct ImageInfo { + #[serde(default)] + path: String, +} + +// HypervisorInfo stores hypervisor details +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct HypervisorInfo { + #[serde(default)] + machine_type: String, + #[serde(default)] + machine_accelerators: String, + #[serde(default)] + version: String, + #[serde(default)] + path: String, + #[serde(default)] + block_device_driver: String, + #[serde(default)] + entropy_source: String, + #[serde(default)] + shared_fs: String, + #[serde(default)] + virtio_fs_daemon: String, + #[serde(default)] + msize_9p: u32, + #[serde(default)] + memory_slots: u32, + #[serde(default)] + pcie_root_port: u32, + #[serde(default)] + hotplug_vfio_on_rootbus: bool, + #[serde(default)] + debug: bool, + #[serde(default)] + enable_iommu: bool, + #[serde(default)] + enable_iommu_platform: bool, + #[serde(default)] + default_vcpus: i32, + #[serde(default)] + cpu_features: String, +} + +// EnvInfo collects all information that will be displayed by the +// env command. +// +// XXX: Any changes must be coupled with a change to formatVersion. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct EnvInfo { + #[serde(default)] + kernel: KernelInfo, + #[serde(default)] + meta: MetaInfo, + #[serde(default)] + image: ImageInfo, + #[serde(default)] + initrd: InitrdInfo, + #[serde(default)] + hypervisor: HypervisorInfo, + #[serde(default)] + runtime: RuntimeInfo, + #[serde(default)] + host: HostInfo, + #[serde(default)] + agent: AgentInfo, +} + +pub fn get_meta_info() -> MetaInfo { + MetaInfo { + version: String::from(FORMAT_VERSION), + } +} + +pub fn get_memory_info() -> Result { + let mem_info = sys_info::mem_info().context("get host memory information")?; + Ok(MemoryInfo { + total: mem_info.total, + available: mem_info.avail, + free: mem_info.free, + }) +} + +fn get_host_info() -> Result { + let host_kernel_version = utils::get_kernel_version(utils::PROC_VERSION_FILE)?; + let (host_distro_name, host_distro_version) = + utils::get_distro_details(utils::OS_RELEASE, utils::OS_RELEASE_CLR)?; + let (cpu_vendor, cpu_model) = arch_specific::get_cpu_details()?; + + let host_distro = DistroInfo { + name: host_distro_name, + version: host_distro_version, + }; + + let cores: usize = std::thread::available_parallelism() + .context("get available parallelism")? + .into(); + + let host_cpu = CPUInfo { + vendor: cpu_vendor, + model: cpu_model, + cpus: cores, + }; + + let memory_info = get_memory_info()?; + + let guest_protection = + arch_specific::available_guest_protection().map_err(|e| anyhow!(e.to_string()))?; + + let guest_protection = guest_protection.to_string(); + + let support_vsocks = utils::supports_vsocks(utils::VHOST_VSOCK_DEVICE)?; + + Ok(HostInfo { + kernel: host_kernel_version, + architecture: String::from(std::env::consts::ARCH), + distro: host_distro, + cpu: host_cpu, + memory: memory_info, + available_guest_protection: guest_protection, + // TODO: See https://github.com/kata-containers/kata-containers/issues/6727 + vm_container_capable: true, + support_vsocks, + }) +} + +pub fn get_runtime_info(toml_config: &TomlConfig) -> Result { + let version = VersionInfo { + semver: String::from(version::VERSION), + commit: String::from(version::COMMIT), + }; + + let config_path = TomlConfig::get_default_config_file(); + let mut toml_path = String::new(); + if config_path.is_ok() { + let p = config_path?; + let path_str = p.to_str(); + toml_path = match path_str { + Some(s) => String::from(s), + None => String::new(), + }; + } + + Ok(RuntimeInfo { + // TODO: Needs to be implemented: https://github.com/kata-containers/kata-containers/issues/6518 + path: String::from("not implemented yet. See: https://github.com/kata-containers/kata-containers/issues/6518"), + version, + experimental: toml_config.runtime.experimental.clone(), + // TODO: See https://github.com/kata-containers/kata-containers/issues/6667 + guest_selinux_label: String::from("not implemented yet: See https://github.com/kata-containers/kata-containers/issues/6667"), + debug: toml_config.runtime.debug, + trace: toml_config.runtime.enable_tracing, + disable_guest_seccomp: toml_config.runtime.disable_guest_seccomp, + disable_new_net_ns: toml_config.runtime.disable_new_netns, + sandbox_cgroup_only: toml_config.runtime.sandbox_cgroup_only, + static_sandbox_resource_mgmt: toml_config.runtime.static_sandbox_resource_mgmt, + config: RuntimeConfigInfo { path: toml_path }, + }) +} + +pub fn get_agent_info(toml_config: &TomlConfig) -> Result { + // Assign the first entry to the agent config, to make this + // work for configs where agent_name is absent. + // This is a workaround for https://github.com/kata-containers/kata-containers/issues/5954 + let key_val = toml_config.agent.iter().next(); + let mut agent_config = match key_val { + Some(x) => Ok(x.1), + None => Err(anyhow!("Missing agent config")), + }?; + + // If the agent_name config is present, use that + if !&toml_config.runtime.agent_name.is_empty() { + agent_config = toml_config + .agent + .get(&toml_config.runtime.agent_name) + .ok_or("could not find agent config in configuration") + .map_err(|e| anyhow!(e))?; + } + + Ok(AgentInfo { + debug: agent_config.debug, + trace: agent_config.enable_tracing, + }) +} + +pub fn get_command_version(cmd: &str) -> Result { + // Path is empty in case of dragonball hypervisor + if cmd.is_empty() { + return Ok("unknown".to_string()); + } + let output = Command::new(cmd) + .arg("--version") + .output() + .map_err(|e| anyhow!(e))?; + + let version = String::from_utf8(output.stdout).map_err(|e| anyhow!(e))?; + + Ok(version) +} + +pub fn get_hypervisor_info( + toml_config: &TomlConfig, +) -> Result<(HypervisorInfo, ImageInfo, KernelInfo, InitrdInfo)> { + // Assign the first entry in the hashmap to the hypervisor config, to make this + // work for configs where hypervisor_name is absent. + // This is a workaround for https://github.com/kata-containers/kata-containers/issues/5954 + let key_val = toml_config.hypervisor.iter().next(); + let mut hypervisor_config = match key_val { + Some(x) => Ok(x.1), + None => Err(anyhow!("Missing hypervisor config")), + }?; + + // If hypervisor_name config is present, use that + if !&toml_config.runtime.hypervisor_name.is_empty() { + hypervisor_config = toml_config + .hypervisor + .get(&toml_config.runtime.hypervisor_name) + .ok_or("could not find hypervisor config in configuration") + .map_err(|e| anyhow!(e))?; + } + + let version = + get_command_version(&hypervisor_config.path).context("error getting hypervisor version")?; + + let hypervisor_info = HypervisorInfo { + machine_type: hypervisor_config.machine_info.machine_type.to_string(), + machine_accelerators: hypervisor_config + .machine_info + .machine_accelerators + .to_string(), + version, + path: hypervisor_config.path.to_string(), + block_device_driver: hypervisor_config + .blockdev_info + .block_device_driver + .to_string(), + entropy_source: hypervisor_config.machine_info.entropy_source.to_string(), + shared_fs: hypervisor_config + .shared_fs + .shared_fs + .clone() + .unwrap_or_else(|| String::from("none")), + virtio_fs_daemon: hypervisor_config.shared_fs.virtio_fs_daemon.to_string(), + msize_9p: hypervisor_config.shared_fs.msize_9p, + memory_slots: hypervisor_config.memory_info.memory_slots, + pcie_root_port: hypervisor_config.device_info.pcie_root_port, + hotplug_vfio_on_rootbus: hypervisor_config.device_info.hotplug_vfio_on_root_bus, + debug: hypervisor_config.debug_info.enable_debug, + enable_iommu: hypervisor_config.device_info.enable_iommu, + enable_iommu_platform: hypervisor_config.device_info.enable_iommu_platform, + default_vcpus: hypervisor_config.cpu_info.default_vcpus, + cpu_features: hypervisor_config.cpu_info.cpu_features.to_string(), + }; + + let image_info = ImageInfo { + path: hypervisor_config.boot_info.image.clone(), + }; + + let kernel_info = KernelInfo { + path: hypervisor_config.boot_info.kernel.to_string(), + parameters: hypervisor_config.boot_info.kernel_params.to_string(), + }; + + let initrd_info = InitrdInfo { + path: hypervisor_config.boot_info.initrd.to_string(), + }; + + Ok((hypervisor_info, image_info, kernel_info, initrd_info)) +} + +pub fn get_env_info(toml_config: &TomlConfig) -> Result { + let metainfo = get_meta_info(); + + let runtime_info = get_runtime_info(toml_config).context("get runtime info")?; + + let agent_info = get_agent_info(toml_config).context("get agent configuration")?; + + let host_info = get_host_info().context("get host information")?; + + let (hypervisor_info, _image_info, kernel_info, initrd_info) = + get_hypervisor_info(toml_config).context("get hypervisor configuration")?; + + let env_info = EnvInfo { + meta: metainfo, + runtime: runtime_info, + kernel: kernel_info, + image: _image_info, + initrd: initrd_info, + hypervisor: hypervisor_info, + host: host_info, + agent: agent_info, + }; + + Ok(env_info) +} + +pub fn handle_env(env_args: EnvArgument) -> Result<()> { + if !Uid::effective().is_root() { + return Err(anyhow!("kata-ctl env command requires root privileges to get host information. Please run as root or use sudo")); + } + + let mut file: Box = if let Some(path) = env_args.file { + Box::new( + File::create(path.as_str()).with_context(|| format!("Error creating file {}", path))?, + ) + } else { + Box::new(io::stdout()) + }; + + let (toml_config, _) = TomlConfig::load_from_default().context("load toml config")?; + + let env_info = get_env_info(&toml_config)?; + + if env_args.json { + let serialized_json = serde_json::to_string_pretty(&env_info)?; + write!(file, "{}", serialized_json)?; + } else { + let toml = toml::to_string(&env_info)?; + write!(file, "{}", toml)?; + } + + Ok(()) +} diff --git a/src/tools/kata-ctl/src/ops/exec_ops.rs b/src/tools/kata-ctl/src/ops/exec_ops.rs index 2a6b970aa..9e1d4549a 100644 --- a/src/tools/kata-ctl/src/ops/exec_ops.rs +++ b/src/tools/kata-ctl/src/ops/exec_ops.rs @@ -326,9 +326,7 @@ fn setup_client(server_url: String, dbg_console_port: u32) -> anyhow::Result { - return Err(anyhow!("invalid URI scheme: {:?}", scheme)); - } + _ => Err(anyhow!("invalid URI scheme: {:?}", scheme)), } } diff --git a/src/tools/kata-ctl/src/ops/version.rs.in b/src/tools/kata-ctl/src/ops/version.rs.in index 052eccf16..e4a5db3c9 100644 --- a/src/tools/kata-ctl/src/ops/version.rs.in +++ b/src/tools/kata-ctl/src/ops/version.rs.in @@ -10,6 +10,8 @@ use clap::crate_version; const KATA_CTL_VERSION: &str = "@KATA_CTL_VERSION@"; +pub const VERSION: &str = "@VERSION@"; +pub const COMMIT: &str = "@COMMIT@"; pub fn get() -> Result { if KATA_CTL_VERSION.trim().is_empty() { diff --git a/src/tools/kata-ctl/src/ops/volume_ops.rs b/src/tools/kata-ctl/src/ops/volume_ops.rs index 60c753f6f..1027dcb72 100644 --- a/src/tools/kata-ctl/src/ops/volume_ops.rs +++ b/src/tools/kata-ctl/src/ops/volume_ops.rs @@ -162,7 +162,7 @@ pub fn get_sandbox_id_for_volume(volume_path: &str) -> Result { return Ok(String::from(file_name)); } - return Err(anyhow!("no sandbox found for {}", volume_path)); + Err(anyhow!("no sandbox found for {}", volume_path)) } #[cfg(test)] diff --git a/src/tools/kata-ctl/src/types.rs b/src/tools/kata-ctl/src/types.rs index 26f5954d8..fcafeb435 100644 --- a/src/tools/kata-ctl/src/types.rs +++ b/src/tools/kata-ctl/src/types.rs @@ -15,6 +15,7 @@ pub enum CheckType { Cpu, Network, KernelModules, + KvmIsUsable, } // PermissionType is used to show whether a check needs to run with elevated (super-user) diff --git a/src/tools/kata-ctl/src/utils.rs b/src/tools/kata-ctl/src/utils.rs index 03c005e9f..a292d3b78 100644 --- a/src/tools/kata-ctl/src/utils.rs +++ b/src/tools/kata-ctl/src/utils.rs @@ -5,9 +5,7 @@ #![allow(dead_code)] -#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))] use crate::arch::arch_specific; -#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))] use crate::check::get_single_cpu_info; use anyhow::{anyhow, Context, Result}; @@ -27,7 +25,7 @@ pub fn drop_privs() -> Result<()> { Ok(()) } -const PROC_VERSION_FILE: &str = "/proc/version"; +pub const PROC_VERSION_FILE: &str = "/proc/version"; pub fn get_kernel_version(proc_version_file: &str) -> Result { let contents = fs::read_to_string(proc_version_file) @@ -43,17 +41,17 @@ pub fn get_kernel_version(proc_version_file: &str) -> Result { Ok(kernel_version) } -const OS_RELEASE: &str = "/etc/os-release"; +pub const OS_RELEASE: &str = "/etc/os-release"; // Clear Linux has a different path (for stateless support) -const OS_RELEASE_CLR: &str = "/usr/lib/os-release"; +pub const OS_RELEASE_CLR: &str = "/usr/lib/os-release"; const UNKNOWN: &str = "unknown"; fn get_field_fn(line: &str, delimiter: &str, file_name: &str) -> Result { let fields: Vec<&str> = line.split(delimiter).collect(); if fields.len() < 2 { - return Err(anyhow!("Unexpected file contents for {}", file_name)); + Err(anyhow!("Unexpected file contents for {}", file_name)) } else { let val = fields[1].trim(); Ok(String::from(val)) @@ -106,11 +104,10 @@ pub fn get_distro_details(os_release: &str, os_release_clr: &str) -> Result<(Str Ok((name, version)) } -#[cfg(any(target_arch = "s390x", target_arch = "x86_64"))] +#[cfg(any(target_arch = "s390x", target_arch = "x86_64", target_arch = "aarch64"))] pub fn get_generic_cpu_details(cpu_info_file: &str) -> Result<(String, String)> { let cpu_info = get_single_cpu_info(cpu_info_file, "\n\n")?; let lines = cpu_info.lines(); - println!("Single cpu info: {}", cpu_info); let mut vendor = String::new(); let mut model = String::new(); @@ -144,9 +141,14 @@ pub fn get_generic_cpu_details(cpu_info_file: &str) -> Result<(String, String)> Ok((vendor, model)) } -const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock"; +pub const VHOST_VSOCK_DEVICE: &str = "/dev/vhost-vsock"; pub fn supports_vsocks(vsock_path: &str) -> Result { - let metadata = fs::metadata(vsock_path)?; + let metadata = fs::metadata(vsock_path).map_err(|err| { + anyhow!( + "Host system does not support vhost-vsock (try running (`sudo modprobe vhost_vsock`) : {}", + err.to_string() + ) + })?; Ok(metadata.is_file()) } diff --git a/src/tools/log-parser-rs/Cargo.lock b/src/tools/log-parser-rs/Cargo.lock new file mode 100644 index 000000000..c1a19ec4e --- /dev/null +++ b/src/tools/log-parser-rs/Cargo.lock @@ -0,0 +1,899 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time 0.1.45", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "clap" +version = "4.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" +dependencies = [ + "bitflags 2.0.2", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "csv" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "cxx" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.5", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.5", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "log-parser" +version = "0.0.1" +dependencies = [ + "chrono", + "clap", + "csv", + "quick-xml", + "ron", + "serde", + "serde_json", + "serde_with", + "serde_yaml", + "slog", + "thiserror", + "toml", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "os_str_bytes" +version = "6.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ron" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300a51053b1cb55c80b7a9fde4120726ddf25ca241a1cbb926626f62fb136bff" +dependencies = [ + "base64", + "bitflags 1.3.2", + "serde", +] + +[[package]] +name = "rustix" +version = "0.36.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "serde" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.20", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_yaml" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c2d1c76a26822187a1fbb5964e3fff108bc208f02e820ab9dac1234f6b388a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.5", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +dependencies = [ + "serde", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" diff --git a/src/tools/log-parser-rs/Cargo.toml b/src/tools/log-parser-rs/Cargo.toml new file mode 100644 index 000000000..bcd98246b --- /dev/null +++ b/src/tools/log-parser-rs/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "log-parser" +version = "0.0.1" +edition = "2021" +authors = ["The Kata Containers community "] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9" +toml = "0.4" +ron = "0.8" +quick-xml = { version = "0.28", features = ["serialize"]} +csv = "1.2" +serde_with = "2.3" +clap = { version = "4.1", features = ["derive", "cargo"] } +thiserror = "1.0" +chrono = { version = "0.4", features = ["serde"]} +slog = "2.7" diff --git a/src/tools/log-parser-rs/Makefile b/src/tools/log-parser-rs/Makefile new file mode 100644 index 000000000..5bca7cc57 --- /dev/null +++ b/src/tools/log-parser-rs/Makefile @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Gabe Venberg +# +# SPDX-License-Identifier: Apache-2.0 + +include ../../../utils.mk + +.DEFAULT_GOAL := default +default: build + +build: + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) + +static-checks-build: + @echo "INFO: static-checks-build do nothing.." + +clean: + cargo clean + +vendor: + cargo vendor + +test: + +install: + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . + +check: standard_rust_check + +.PHONY: \ + build \ + check \ + clean \ + install \ + test \ + vendor diff --git a/src/tools/log-parser-rs/README.md b/src/tools/log-parser-rs/README.md new file mode 100644 index 000000000..a641b0030 --- /dev/null +++ b/src/tools/log-parser-rs/README.md @@ -0,0 +1,67 @@ +# `kata-log-parser` + +## Introduction + +`log-parser-rs` is a tool that combines logfiles generated by the various +system components, sorts them by time stamp, and re-displays the log entries. + +The tool is also able to check the validity of all log records, can re-format the +logs, and output them in a different format. + +For more information on the `kata-log-parser` tool, use the help command: + +``` +$ kata-log-parser --help +``` + +> **Note** this is a rewrite of the go-based `kata-log-parser` tool, and will eventually replace it. + +## Log Format + +Kata's `runtime-rs` logs are JSON objects in the following format: + +```json +{"msg":"message","level":"INFO","ts":"1970-01-01T00:00:00.000000000Z","name":"kata-runtime","version":"0.1.0","pid":"0","source":"source","subsystem":"subsystem"} +``` + +However, if `--ignore-missing-fields` is set, a log missing one or more of the following fields may be omitted: + +- `level` +- `name` +- `version` +- `pid` +- `source` +- `subsystem` + +> **Note** a log entry must be on one single line, and a line must contain only one log entry. + +## Command line opts + +The most valuable command line options are listed below: + +- `-o, --output-file ` File to output to. If not set, sends to stdout. +- `--output-format ` Sets the format of the output. Defaults to `json`, and can be set to `csv`, `json`, `ron`, `text`, `toml`, `xml`, and `yaml`. +- `-q, --quiet` Will not print invalid log entry errors to stderr. +- `-s, --strict` Any invalid log entry will halt the program. + +For a comprehensive (and guaranteed up to date) list, please run `log-parser-rs --help`. + +## Usage + +1. Make sure containerd is in [debug mode](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#enabling-full-containerd-debug) +1. Make sure you are running runtime-rs: + ``` + $ containerd-shim-kata-v2 --version|grep -qi rust && echo rust || echo golang + ``` +1. Collect the logs (alternatively to journal clearing you may consider constraining collected logs by adding `--since=`). + ``` + $ sudo journalctl -q -o cat -a -t kata | grep "^{" > ./kata.log ./kata.log + ``` +1. Ensure the logs are readable: + ``` + $ sudo chown $USER *.log + ``` +1. Process the logs: + ``` + $ log-parser-rs kata.log -o out.log + ``` diff --git a/src/tools/log-parser-rs/src/args.rs b/src/tools/log-parser-rs/src/args.rs new file mode 100644 index 000000000..1980fcc35 --- /dev/null +++ b/src/tools/log-parser-rs/src/args.rs @@ -0,0 +1,58 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use clap::{Parser, ValueEnum}; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(name="kata-log-parser", author, version, about, long_about = None)] // Read from `Cargo.toml` +pub struct Cli { + pub input_file: Vec, + + #[arg(short, long)] + pub output_file: Option, + + #[arg(short, long, help = "check log files and only display output on error")] + pub check_only: bool, + + #[arg(long, help = "error if any files are empty")] + pub error_if_file_empty: bool, + + #[arg(long, help = "error if all logfiles are empty")] + pub error_if_no_records: bool, + + #[arg( + long, + help = "do not make an error for lines with no pid, source, name, or level" + )] + pub ignore_missing_fields: bool, + + #[arg( + short, + long, + help = "suppress warning messages that would otherwise go to stderr." + )] + pub quiet: bool, + + #[arg( + short, + long, + help = "do not tolerate misformed agent messages (may be caused by non-Kata Containers log lines)" + )] + pub strict: bool, + + #[arg(long, value_enum, default_value_t = OutputFormat::Json, help="set the output format")] + pub output_format: OutputFormat, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +pub enum OutputFormat { + Csv, + Json, + Ron, + Text, + Toml, + Xml, + Yaml, +} diff --git a/src/tools/log-parser-rs/src/log_message.rs b/src/tools/log-parser-rs/src/log_message.rs new file mode 100644 index 000000000..9caf849c6 --- /dev/null +++ b/src/tools/log-parser-rs/src/log_message.rs @@ -0,0 +1,467 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{fmt::Debug, fmt::Display, str::FromStr}; + +use chrono::{DateTime, Utc}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_with::{ + serde_as, skip_serializing_none, DeserializeFromStr, DisplayFromStr, SerializeDisplay, +}; +use thiserror::Error; + +pub trait AnyLogMessage: Serialize + DeserializeOwned + Debug { + fn get_timestamp(&self) -> DateTime; +} + +#[serde_as] +#[skip_serializing_none] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default)] +pub struct LogMessage { + pub level: Option, + + #[serde(rename = "msg")] + pub message: String, + + pub name: Option, + + #[serde_as(as = "Option")] + pub pid: Option, + + pub source: Option, + + pub subsystem: Option, + + #[serde_as(as = "DisplayFromStr")] + #[serde(rename = "ts")] + pub timestamp: DateTime, +} + +impl AnyLogMessage for LogMessage { + fn get_timestamp(&self) -> DateTime { + self.timestamp + } +} + +//totally abusing serde to easily display this. +impl Display for LogMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + serde_json::to_string_pretty(&self).map_err(|_| std::fmt::Error)? + ) + } +} + +#[serde_as] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default)] +pub struct StrictLogMessage { + pub level: LogLevel, + + #[serde(rename = "msg")] + pub message: String, + + pub name: String, + + #[serde_as(as = "DisplayFromStr")] + pub pid: usize, + + pub source: String, + + pub subsystem: String, + + #[serde_as(as = "DisplayFromStr")] + #[serde(rename = "ts")] + pub timestamp: DateTime, +} + +impl AnyLogMessage for StrictLogMessage { + fn get_timestamp(&self) -> DateTime { + self.timestamp + } +} + +//totally abusing serde to easily display this. +impl Display for StrictLogMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + serde_json::to_string_pretty(&self).map_err(|_| std::fmt::Error)? + ) + } +} + +// A newtype for slog::Level, as it does not implement Serialize and Deserialize. +#[derive(Debug, SerializeDisplay, DeserializeFromStr, PartialEq, Eq)] +pub struct LogLevel(slog::Level); + +#[derive(Debug, Error)] +pub enum LevelError { + #[error("invalid slog level: {0}")] + InvalidLevel(String), +} + +impl Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}", self.0.as_str())) + } +} + +impl Default for LogLevel { + fn default() -> Self { + LogLevel(slog::Level::Info) + } +} + +impl FromStr for LogLevel { + type Err = LevelError; + fn from_str(s: &str) -> Result { + let level = match s.to_lowercase().as_str() { + //need to accept both the short and long string versions. + "critical" | "crit" => slog::Level::Critical, + "error" | "erro" => slog::Level::Error, + "warning" | "warn" => slog::Level::Warning, + "info" => slog::Level::Info, + "debug" | "debg" => slog::Level::Debug, + "trace" | "trce" => slog::Level::Trace, + _ => return Err(LevelError::InvalidLevel(s.to_string())), + }; + + Ok(LogLevel(level)) + } +} + +//TODO: add tests for serialization. +#[cfg(test)] +mod test { + use super::*; + use crate::log_parser_error::LogParserError; + + #[test] + fn parse_string() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#; + let result = Ok(LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: Some("kata-runtime".to_string()), + pid: Some(3327263), + source: Some("foo".to_string()), + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }); + assert_eq!( + serde_json::from_str(log) + .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + result + ) + } + + #[test] + fn parse_string_with_missing_fields() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","version":"0.1.0","subsystem":"hypervisor","source":"foo"}"#; + let result = Ok(LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: None, + source: Some("foo".to_string()), + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }); + assert_eq!( + serde_json::from_str(log) + .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + result + ) + } + + #[test] + #[should_panic] + fn parse_error() { + let log = "random non-kata log message"; + serde_json::from_str::(log).unwrap(); + } + + #[test] + fn parse_string_strict() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#; + let result = Ok(StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }); + assert_eq!( + serde_json::from_str(log) + .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + result + ) + } + + #[test] + #[should_panic] + fn parse_string_with_missing_fields_strict() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","version":"0.1.0","subsystem":"hypervisor","source":"foo"}"#; + println!( + "{:?}", + serde_json::from_str::(log).unwrap() + ); + } + + #[test] + #[should_panic] + fn parse_error_strict() { + let log = "random non-kata log message"; + serde_json::from_str::(log).unwrap(); + } + + #[test] + fn serialize_json_strict() { + let result = r#"{"level":"DEBUG","msg":"vmm-master thread is uninitialized or has exited.","name":"kata-runtime","pid":"3327263","source":"foo","subsystem":"hypervisor","ts":"2023-03-15 14:17:02.526992506 UTC"}"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_json::to_string(&log).unwrap()); + } + + #[test] + fn serialize_json() { + let result = r#"{"level":"DEBUG","msg":"vmm-master thread is uninitialized or has exited.","pid":"3327263","subsystem":"hypervisor","ts":"2023-03-15 14:17:02.526992506 UTC"}"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_json::to_string(&log).unwrap()); + } + + #[test] + fn serialize_csv_strict() { + let result = r#"level,msg,name,pid,source,subsystem,ts +DEBUG,vmm-master thread is uninitialized or has exited.,kata-runtime,3327263,foo,hypervisor,2023-03-15 14:17:02.526992506 UTC +"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + let mut csv_writer = csv::Writer::from_writer(vec![]); + csv_writer.serialize(&log).unwrap(); + let output = String::from_utf8(csv_writer.into_inner().unwrap()).unwrap(); + assert_eq!(result, output); + } + + #[test] + fn serialize_csv() { + let result = r#"level,msg,pid,subsystem,ts +DEBUG,vmm-master thread is uninitialized or has exited.,3327263,hypervisor,2023-03-15 14:17:02.526992506 UTC +"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + let mut csv_writer = csv::Writer::from_writer(vec![]); + csv_writer.serialize(&log).unwrap(); + let output = String::from_utf8(csv_writer.into_inner().unwrap()).unwrap(); + assert_eq!(result, output); + } + + #[test] + fn serialize_ron_strict() { + let result = r#"(level:"DEBUG",msg:"vmm-master thread is uninitialized or has exited.",name:"kata-runtime",pid:"3327263",source:"foo",subsystem:"hypervisor",ts:"2023-03-15 14:17:02.526992506 UTC")"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, ron::to_string(&log).unwrap()); + } + + #[test] + fn serialize_ron() { + let result = r#"(level:Some("DEBUG"),msg:"vmm-master thread is uninitialized or has exited.",pid:Some("3327263"),subsystem:Some("hypervisor"),ts:"2023-03-15 14:17:02.526992506 UTC")"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, ron::to_string(&log).unwrap()); + } + + #[test] + fn serialize_toml_strict() { + let result = r#"level = "DEBUG" +msg = "vmm-master thread is uninitialized or has exited." +name = "kata-runtime" +pid = "3327263" +source = "foo" +subsystem = "hypervisor" +ts = "2023-03-15 14:17:02.526992506 UTC" +"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, toml::to_string(&log).unwrap()); + } + + #[test] + fn serialize_toml() { + let result = r#"level = "DEBUG" +msg = "vmm-master thread is uninitialized or has exited." +pid = "3327263" +subsystem = "hypervisor" +ts = "2023-03-15 14:17:02.526992506 UTC" +"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, toml::to_string(&log).unwrap()); + } + + #[test] + fn serialize_xml_strict() { + let result = r#"DEBUGvmm-master thread is uninitialized or has exited.kata-runtime3327263foohypervisor2023-03-15 14:17:02.526992506 UTC"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, quick_xml::se::to_string(&log).unwrap()); + } + + #[test] + fn serialize_xml() { + let result = r#"DEBUGvmm-master thread is uninitialized or has exited.3327263hypervisor2023-03-15 14:17:02.526992506 UTC"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, quick_xml::se::to_string(&log).unwrap()); + } + + #[test] + fn serialize_yaml_strict() { + let result = r#"level: DEBUG +msg: vmm-master thread is uninitialized or has exited. +name: kata-runtime +pid: '3327263' +source: foo +subsystem: hypervisor +ts: 2023-03-15 14:17:02.526992506 UTC +"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_yaml::to_string(&log).unwrap()); + } + + #[test] + fn serialize_yaml() { + let result = r#"level: DEBUG +msg: vmm-master thread is uninitialized or has exited. +pid: '3327263' +subsystem: hypervisor +ts: 2023-03-15 14:17:02.526992506 UTC +"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_yaml::to_string(&log).unwrap()); + } +} diff --git a/src/tools/log-parser-rs/src/log_parser_error.rs b/src/tools/log-parser-rs/src/log_parser_error.rs new file mode 100644 index 000000000..4af197d27 --- /dev/null +++ b/src/tools/log-parser-rs/src/log_parser_error.rs @@ -0,0 +1,58 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{error::Error, path::PathBuf}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum LogParserError { + #[error("Unknown Error")] + Unknown(Box), + + #[error("Input file '{0}' cannot be found")] + InputFileNotFound(PathBuf), + + #[error("Input file '{0}' does not contain any valid logs")] + FileEmpty(PathBuf), + + #[error("No permission to open '{0}'")] + InputFilePermissionError(PathBuf), + + #[error("No permission to write to '{0}'")] + OutputFilePermissionError(PathBuf), + + #[error("Log parsing error: {0} with string {1}")] + ParsingError(serde_json::Error, String), + + #[error("Error serializing {0}: {1}")] + SerializationError(String, Box), + + #[error("No logs in any file")] + NoRecordsError(), +} + +impl PartialEq for LogParserError { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Unknown(l0), Self::Unknown(r0)) => l0.to_string() == r0.to_string(), + (Self::InputFileNotFound(l0), Self::InputFileNotFound(r0)) => l0 == r0, + (Self::FileEmpty(l0), Self::FileEmpty(r0)) => l0 == r0, + (Self::InputFilePermissionError(l0), Self::InputFilePermissionError(r0)) => l0 == r0, + (Self::OutputFilePermissionError(l0), Self::OutputFilePermissionError(r0)) => l0 == r0, + //serde_json::Error does not impl partialeq, but for testing cases a quick and dirty + //string comparison works well enough. + (Self::ParsingError(l0, l1), Self::ParsingError(r0, r1)) => { + l0.to_string() == r0.to_string() && l1 == r1 + } + //this catch all returns whether the two enums are the same variant type. eg, + //core::mem:discriminant(LogParserError::Unkown)==core:mem:discriminant(LogParserError::Unkown) + //is true, but it would not be true, for say, Unkown and InputFilePermissionError. + //Note that it only compares the variants, not the contents of the variants, hence the + //need for the above branches. + _ => core::mem::discriminant(self) == core::mem::discriminant(other), + } + } +} + +impl Eq for LogParserError {} diff --git a/src/tools/log-parser-rs/src/main.rs b/src/tools/log-parser-rs/src/main.rs new file mode 100644 index 000000000..72c406a59 --- /dev/null +++ b/src/tools/log-parser-rs/src/main.rs @@ -0,0 +1,68 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +#![warn(unused_crate_dependencies)] +#![warn(missing_debug_implementations)] + +mod args; +mod log_message; +mod log_parser_error; +mod output_file; +mod parse_file; +mod process_logs; + +use crate::args::Cli; +use crate::log_message::StrictLogMessage; +use crate::log_parser_error::LogParserError; +use crate::output_file::*; +use crate::parse_file::*; +use crate::process_logs::*; +use clap::Parser; +use log_message::AnyLogMessage; +use log_message::LogMessage; +use std::process::exit; + +fn handle_logs(cli: Cli) -> Result<(), LogParserError> { + let mut logs = Vec::new(); + + for file in &cli.input_file { + let in_file = open_file_into_memory(file)?; + let file_logs = filter_errors(parse_log::(in_file), &cli)?; + + if cli.error_if_file_empty && file_logs.is_empty() { + return Err(LogParserError::FileEmpty(file.to_path_buf())); + } + + logs.extend(file_logs) + } + + if cli.error_if_no_records && logs.is_empty() { + return Err(LogParserError::NoRecordsError()); + } + if cli.check_only { + return Ok(()); + } + + sort_logs(&mut logs); + output_file(logs, &cli)?; + Ok(()) +} + +//needed another layer of function call in order to genericize over both LogMessage and +//StrictLogMessage. +fn real_main() -> std::result::Result<(), LogParserError> { + let cli = Cli::parse(); + if cli.ignore_missing_fields { + handle_logs::(cli) + } else { + handle_logs::(cli) + } +} + +fn main() { + if let Err(e) = real_main() { + eprintln!("ERROR: {:#}", e); + exit(1); + } +} diff --git a/src/tools/log-parser-rs/src/output_file.rs b/src/tools/log-parser-rs/src/output_file.rs new file mode 100644 index 000000000..eb6ddf146 --- /dev/null +++ b/src/tools/log-parser-rs/src/output_file.rs @@ -0,0 +1,149 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs::File, + io::{ErrorKind, Write}, + path::Path, +}; + +use crate::{args::Cli, log_message::AnyLogMessage, log_parser_error::LogParserError}; + +/// a simple dispatcher method that outputs the deserialized result of the parsed logs according to +/// the CLI arguments. +/// +/// # Errors +/// +/// If outputting to a file, may return a LogParserError having to do with opening and writing to +/// the output file. +pub(crate) fn output_file( + contents: Vec, + options: &Cli, +) -> Result<(), LogParserError> { + let serializer = choose_formatting(options); + if let Some(out_file) = &options.output_file { + write_logs_to_file(out_file, contents, serializer)?; + } else { + print_logs(contents, serializer)?; + }; + Ok(()) +} + +fn choose_formatting( + options: &Cli, +) -> fn(Vec) -> Result { + match options.output_format { + crate::args::OutputFormat::Csv => serialize_csv, + crate::args::OutputFormat::Json => serialize_json, + crate::args::OutputFormat::Ron => serialize_ron, + crate::args::OutputFormat::Text => serialize_text, + crate::args::OutputFormat::Toml => serialize_toml, + crate::args::OutputFormat::Xml => serialize_xml, + crate::args::OutputFormat::Yaml => serialize_yaml, + } +} + +fn serialize_text(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| Ok(format!("{:?}", r))) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_json(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + serde_json::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_toml(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + toml::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_yaml(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + serde_yaml::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_xml(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + quick_xml::se::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_ron(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + ron::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_csv(input: Vec) -> Result { + let mut csv_writer = csv::Writer::from_writer(vec![]); + for record in input { + csv_writer.serialize(&record).map_err(|e| { + LogParserError::SerializationError(format!("{:?}", record), Box::new(e)) + })?; + } + String::from_utf8( + csv_writer + .into_inner() + .map_err(|e| LogParserError::Unknown(Box::new(e)))?, + ) + .map_err(|e| LogParserError::Unknown(Box::new(e))) +} + +fn write_logs_to_file( + out_file: &Path, + contents: Vec, + serializer: fn(Vec) -> Result, +) -> Result<(), LogParserError> { + let mut file = File::create(out_file).map_err(|e| map_out_io_err(e, out_file))?; + file.write_all(serializer(contents)?.as_bytes()) + .map_err(|e| LogParserError::Unknown(Box::new(e)))?; + Ok(()) +} + +fn print_logs( + contents: Vec, + serializer: fn(Vec) -> Result, +) -> Result<(), LogParserError> { + print!("{}", serializer(contents)?); + Ok(()) +} + +fn map_out_io_err(e: std::io::Error, out_file: &Path) -> LogParserError { + match e.kind() { + ErrorKind::PermissionDenied => LogParserError::OutputFilePermissionError(out_file.into()), + _ => LogParserError::Unknown(Box::new(e)), + } +} diff --git a/src/tools/log-parser-rs/src/parse_file.rs b/src/tools/log-parser-rs/src/parse_file.rs new file mode 100644 index 000000000..3304aebe2 --- /dev/null +++ b/src/tools/log-parser-rs/src/parse_file.rs @@ -0,0 +1,85 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use crate::{log_message::AnyLogMessage, log_parser_error::LogParserError}; +use std::{fs, io::ErrorKind, path::Path}; + +/// Reads the entire file into memory as a string. +/// +/// # Errors +/// +/// This function will return an error if opening the file returns an error. +/// FileNotFound and FilePermissionError *should* be the only error types that can happen, and they +/// are wrapped in the appropriate LogParserError types. Any other error will panic. +/// +/// # Panic +/// +/// Will panic if File::open returns an error other than NotFound or PermissionDenied. +/// This *should* not happen. +/// +/// # Gochas +/// +/// Memory use can be unexpectedly high, as entire file is read into memory. +pub(crate) fn open_file_into_memory(inputfile: &Path) -> Result { + match fs::read_to_string(inputfile) { + Ok(s) => Ok(s), + Err(e) => match e.kind() { + ErrorKind::NotFound => Err(LogParserError::InputFileNotFound(inputfile.to_path_buf())), + ErrorKind::PermissionDenied => Err(LogParserError::InputFilePermissionError( + inputfile.to_path_buf(), + )), + _ => Err(LogParserError::Unknown(Box::new(e))), + }, + } +} + +/// Parses a series of logs from a string, returning an array of log messages and parsing errors. +/// +/// # Errors +/// +/// Will pass any errors from serde in to the output array +pub(crate) fn parse_log(in_file: String) -> Vec> +where + O: AnyLogMessage, +{ + let mut output = Vec::new(); + for line in in_file.lines() { + let logentry = serde_json::from_str::(line) + .map_err(|e| LogParserError::ParsingError(e, line.to_string())); + output.push(logentry); + } + output +} + +#[cfg(test)] +mod test { + use super::*; + use crate::log_message::LogMessage; + + #[test] + fn parse_strings() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"} +{"msg":"resource clean up","level":"INFO","ts":"2023-03-15T14:17:02.527047136Z","subsystem":"virt-container","name":"kata-runtime","pid":"3327263","version":"0.1.0","source":"foo"}"#; + let result = vec![ + serde_json::from_str(r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + serde_json::from_str(r#"{"msg":"resource clean up","level":"INFO","ts":"2023-03-15T14:17:02.527047136Z","subsystem":"virt-container","name":"kata-runtime","pid":"3327263","version":"0.1.0","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + ]; + assert_eq!(parse_log::(log.into()), result) + } + #[test] + fn parse_mixed() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"} +Random Kernel Message"#; + let result = vec![ + serde_json::from_str(r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + + Err(LogParserError::ParsingError( + serde_json::from_str::("Random Kernel Message") + .err() + .unwrap() + , "Random Kernel Message".to_string())), + ]; + assert_eq!(parse_log::(log.into()), result) + } +} diff --git a/src/tools/log-parser-rs/src/process_logs.rs b/src/tools/log-parser-rs/src/process_logs.rs new file mode 100644 index 000000000..966124fa0 --- /dev/null +++ b/src/tools/log-parser-rs/src/process_logs.rs @@ -0,0 +1,90 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use crate::{args::Cli, log_message::AnyLogMessage, log_parser_error::LogParserError}; + +/// Calls functions to either check for errors, or to filter them out, discarding them or printing +/// them to stderr. +/// +/// # Errors +/// +/// If cli.strict is true, will return the first error it finds in input (if any) +pub(crate) fn filter_errors( + input: Vec>, + cli: &Cli, +) -> Result, LogParserError> { + if cli.strict { + find_errors(input) + } else if cli.quiet { + Ok(filter_errors_quiet(input)) + } else { + Ok(filter_errors_to_stderr(input)) + } +} + +/// checks if any errors are in the passed log vector. If there are none, returns a vec of O +/// (stripping the containing Result). If there are some, returns the first found error. +fn find_errors( + input: Vec>, +) -> Result, LogParserError> { + // yes, this is only the one line, but the behavior of collect() here is non-obvious. In short, + // as Result implements IntoIter, you can go from a `Vec` to a `Result, E>` + // with just collect() + input.into_iter().collect() +} + +/// removes all LogParserErrors, returning a vec of just O +fn filter_errors_quiet(input: Vec>) -> Vec { + input.into_iter().filter_map(|l| l.ok()).collect() +} + +/// removes all LogParserErrors, returning a vec of just O, and prints any errors to stderr. +fn filter_errors_to_stderr(input: Vec>) -> Vec { + input + .into_iter() + .filter_map(|l| match l { + Ok(log) => Some(log), + Err(e) => { + eprintln!("{}", e); + None + } + }) + .collect() +} + +/// does what it says on the tin. Sorts a vec of logs in place by their timestamp. +pub(crate) fn sort_logs(input: &mut [O]) { + input.sort_by_key(|l| l.get_timestamp()); +} + +#[cfg(test)] +mod test { + use super::*; + use crate::log_message::LogMessage; + #[test] + fn error_filter() { + let unclean_logs = vec![ + Ok(LogMessage::default()), + Err(LogParserError::SerializationError( + "test error".to_string(), + Box::new(slog::Error::Fmt(std::fmt::Error)), + )), + ]; + let logs = vec![LogMessage::default()]; + assert_eq!(filter_errors_quiet(unclean_logs), logs) + } + + #[test] + fn error_filter_to_stderr() { + let unclean_logs = vec![ + Ok(LogMessage::default()), + Err(LogParserError::SerializationError( + "test error".to_string(), + Box::new(slog::Error::Fmt(std::fmt::Error)), + )), + ]; + let logs = vec![LogMessage::default()]; + assert_eq!(filter_errors_to_stderr(unclean_logs), logs) + } +} diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock index f9ebce67b..b280b2a27 100644 --- a/src/tools/runk/Cargo.lock +++ b/src/tools/runk/Cargo.lock @@ -102,7 +102,7 @@ checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -119,7 +119,7 @@ checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -164,7 +164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2" dependencies = [ "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -285,7 +285,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -362,7 +362,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 1.0.91", ] [[package]] @@ -373,7 +373,7 @@ checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a" dependencies = [ "darling_core", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -384,7 +384,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -395,7 +395,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -416,7 +416,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -426,7 +426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73" dependencies = [ "derive_builder_core", - "syn", + "syn 1.0.91", ] [[package]] @@ -473,7 +473,7 @@ checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -616,7 +616,7 @@ checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -833,9 +833,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.127" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libcontainer" @@ -949,25 +949,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.2" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -1013,15 +1002,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - [[package]] name = "num-integer" version = "0.1.44" @@ -1208,7 +1188,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.91", "version_check", ] @@ -1225,11 +1205,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -1285,7 +1265,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -1303,9 +1283,16 @@ name = "protobuf" version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + +[[package]] +name = "protobuf" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55bad9126f378a853655831eb7363b7b01b81d19f8cb1218861086ca4a1a61e" dependencies = [ - "serde", - "serde_derive", + "once_cell", + "protobuf-support", + "thiserror", ] [[package]] @@ -1314,17 +1301,47 @@ version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" dependencies = [ - "protobuf", + "protobuf 2.27.1", ] [[package]] -name = "protobuf-codegen-pure" -version = "2.27.1" +name = "protobuf-codegen" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" +checksum = "0dd418ac3c91caa4032d37cb80ff0d44e2ebe637b2fb243b6234bf89cdac4901" dependencies = [ - "protobuf", - "protobuf-codegen", + "anyhow", + "once_cell", + "protobuf 3.2.0", + "protobuf-parse", + "regex", + "tempfile", + "thiserror", +] + +[[package]] +name = "protobuf-parse" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d39b14605eaa1f6a340aec7f320b34064feb26c93aec35d6a9a2272a8ddfa49" +dependencies = [ + "anyhow", + "indexmap", + "log", + "protobuf 3.2.0", + "protobuf-support", + "tempfile", + "thiserror", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d4d7b8601c814cfb36bcebb79f0e61e45e1e93640cf778837833bbed05c372" +dependencies = [ + "thiserror", ] [[package]] @@ -1332,16 +1349,16 @@ name = "protocols" version = "0.1.0" dependencies = [ "oci", - "protobuf", + "protobuf 3.2.0", "ttrpc", "ttrpc-codegen", ] [[package]] name = "quote" -version = "1.0.18" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ "proc-macro2", ] @@ -1530,7 +1547,7 @@ dependencies = [ "nix 0.24.2", "oci", "path-absolutize", - "protobuf", + "protobuf 3.2.0", "protocols", "regex", "rlimit", @@ -1584,7 +1601,7 @@ checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -1606,7 +1623,7 @@ checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -1694,9 +1711,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1735,6 +1752,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syn" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tabwriter" version = "1.2.1" @@ -1803,7 +1831,7 @@ checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -1839,33 +1867,32 @@ dependencies = [ [[package]] name = "tokio" -version = "1.17.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ + "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", - "once_cell", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.7.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.16", ] [[package]] @@ -1897,7 +1924,7 @@ checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] [[package]] @@ -1911,43 +1938,43 @@ dependencies = [ [[package]] name = "ttrpc" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8" +checksum = "a35f22a2964bea14afee161665bb260b83cb48e665e0260ca06ec0e775c8b06c" dependencies = [ "byteorder", "libc", "log", "nix 0.23.1", - "protobuf", - "protobuf-codegen-pure", + "protobuf 3.2.0", + "protobuf-codegen 3.2.0", "thiserror", ] [[package]] name = "ttrpc-codegen" -version = "0.2.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408" +checksum = "94d7f7631d7a9ebed715a47cd4cb6072cbc7ae1d4ec01598971bbec0024340c2" dependencies = [ - "protobuf", - "protobuf-codegen", - "protobuf-codegen-pure", + "protobuf 2.27.1", + "protobuf-codegen 3.2.0", + "protobuf-support", "ttrpc-compiler", ] [[package]] name = "ttrpc-compiler" -version = "0.4.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15" +checksum = "ec3cb5dbf1f0865a34fe3f722290fe776cacb16f50428610b779467b76ddf647" dependencies = [ "derive-new", "prost", "prost-build", "prost-types", - "protobuf", - "protobuf-codegen", + "protobuf 2.27.1", + "protobuf-codegen 2.27.1", "tempfile", ] @@ -1961,6 +1988,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + [[package]] name = "unicode-segmentation" version = "1.9.0" @@ -2096,6 +2129,66 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.34.0" @@ -2108,6 +2201,18 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.34.0" @@ -2120,6 +2225,18 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.34.0" @@ -2132,6 +2249,18 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.34.0" @@ -2144,6 +2273,30 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.34.0" @@ -2156,6 +2309,18 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "xattr" version = "0.2.3" @@ -2215,7 +2380,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 1.0.91", ] [[package]] @@ -2252,5 +2417,5 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.91", ] diff --git a/src/tools/runk/Cargo.toml b/src/tools/runk/Cargo.toml index 19632883d..2379ddb94 100644 --- a/src/tools/runk/Cargo.toml +++ b/src/tools/runk/Cargo.toml @@ -19,7 +19,7 @@ anyhow = "1.0.52" slog = "2.7.0" chrono = { version = "0.4.19", features = ["serde"] } slog-async = "2.7.0" -tokio = { version = "1.15.0", features = ["full"] } +tokio = { version = "1.28.1", features = ["full"] } serde = { version = "1.0.133", features = ["derive"] } serde_json = "1.0.74" users = "0.11.0" diff --git a/src/tools/runk/Makefile b/src/tools/runk/Makefile index d025b6a89..cd9a24a8b 100644 --- a/src/tools/runk/Makefile +++ b/src/tools/runk/Makefile @@ -8,6 +8,10 @@ LIBC ?= gnu include ../../../utils.mk +ifeq ($(ARCH), ppc64le) + override ARCH = powerpc64le + endif + TARGET = runk TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) AGENT_SOURCE_PATH = ../../agent diff --git a/src/tools/runk/libcontainer/src/cgroup.rs b/src/tools/runk/libcontainer/src/cgroup.rs index 586c6e894..0f4673da1 100644 --- a/src/tools/runk/libcontainer/src/cgroup.rs +++ b/src/tools/runk/libcontainer/src/cgroup.rs @@ -27,7 +27,7 @@ pub fn remove_cgroup_dir(cgroup: &cgroups::Cgroup) -> Result<()> { retries -= 1; } - return Err(anyhow!("failed to remove cgroups paths")); + Err(anyhow!("failed to remove cgroups paths")) } // Make sure we get a stable freezer state, so retry if the cgroup is still undergoing freezing. diff --git a/src/tools/runk/src/main.rs b/src/tools/runk/src/main.rs index c79746678..2e6e5220e 100644 --- a/src/tools/runk/src/main.rs +++ b/src/tools/runk/src/main.rs @@ -66,9 +66,7 @@ async fn cmd_run(subcmd: SubCommand, root_path: &Path, logger: &Logger) -> Resul CommonCmd::Ps(ps) => commands::ps::run(ps, root_path, logger), CommonCmd::Pause(pause) => commands::pause::run(pause, root_path, logger), CommonCmd::Resume(resume) => commands::resume::run(resume, root_path, logger), - _ => { - return Err(anyhow!("command is not implemented yet")); - } + _ => Err(anyhow!("command is not implemented yet")), }, _ => unreachable!(), } diff --git a/src/tools/trace-forwarder/Makefile b/src/tools/trace-forwarder/Makefile index 5a529e211..3bdd5d53a 100644 --- a/src/tools/trace-forwarder/Makefile +++ b/src/tools/trace-forwarder/Makefile @@ -5,6 +5,10 @@ include ../../../utils.mk +ifeq ($(ARCH), ppc64le) + override ARCH = powerpc64le + endif + .DEFAULT_GOAL := default default: build diff --git a/tests/integration/kubernetes/k8s-cpu-ns.bats b/tests/integration/kubernetes/k8s-cpu-ns.bats index 4d5f2e883..f3c69a2f6 100644 --- a/tests/integration/kubernetes/k8s-cpu-ns.bats +++ b/tests/integration/kubernetes/k8s-cpu-ns.bats @@ -11,7 +11,9 @@ load "${BATS_TEST_DIRNAME}/tests_common.sh" setup() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "dragonball" ] && skip "test not working see: ${dragonball_limitations}" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "TEEs do not support memory / CPU hotplug" + ( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || [ "${KATA_HYPERVISOR}" == "qemu-sev" ] ) \ + && skip "TEEs do not support memory / CPU hotplug" + pod_name="constraints-cpu-test" container_name="first-cpu-container" @@ -28,7 +30,9 @@ setup() { @test "Check CPU constraints" { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "dragonball" ] && skip "test not working see: ${dragonball_limitations}" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "TEEs do not support memory / CPU hotplug" + ( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || [ "${KATA_HYPERVISOR}" == "qemu-sev" ] ) \ + && skip "TEEs do not support memory / CPU hotplug" + # Create the pod kubectl create -f "${pod_config_dir}/pod-cpu.yaml" @@ -73,7 +77,8 @@ setup() { teardown() { [ "${KATA_HYPERVISOR}" == "firecracker" ] && skip "test not working see: ${fc_limitations}" [ "${KATA_HYPERVISOR}" == "dragonball" ] && skip "test not working see: ${dragonball_limitations}" - [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] && skip "TEEs do not support memory / CPU hotplug" + ( [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || [ "${KATA_HYPERVISOR}" == "qemu-sev" ] ) \ + && skip "TEEs do not support memory / CPU hotplug" # Debugging information kubectl describe "pod/$pod_name" diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh old mode 100755 new mode 100644 diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index b4062ce8c..e0c900034 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -214,11 +214,11 @@ docker_extra_args() args+=" -v ${gentoo_local_portage_dir}:/usr/portage/packages" args+=" --volumes-from ${gentoo_portage_container}" ;; - debian | ubuntu | suse) + debian | ubuntu | suse) source /etc/os-release case "$ID" in - fedora | centos | rhel) + fedora | centos | rhel) # Depending on the podman version, we'll face issues when passing # `--security-opt apparmor=unconfined` on a system where not apparmor is not installed. # Because of this, let's just avoid adding this option when the host OS comes from Red Hat. diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index 27bd02c57..26c32178d 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -26,6 +26,8 @@ final_initrd_name="kata-containers-initrd" image_initrd_extension=".img" arch_target="$(uname -m)" +final_initrd_name="kata-containers-initrd" +image_initrd_extension=".img" build_initrd() { info "Build initrd" @@ -93,7 +95,7 @@ Options: --imagetype=${image_type} --prefix=${prefix} --destdir=${destdir} - --image_initrd_sufix=${image_initrd_suffix} + --image_initrd_suffix=${image_initrd_suffix} EOF exit "${return_code}" @@ -126,7 +128,12 @@ main() { ;; image_initrd_suffix=*) image_initrd_suffix=${OPTARG#*=} - if [ -n "${image_initrd_suffix}" ]; then + if [ "${image_initrd_suffix}" == "sev" ]; then + initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name") + initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version") + initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}" + final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" + elif [ -n "${image_initrd_suffix}" ]; then img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name") img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version") image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}" @@ -184,4 +191,4 @@ main() { popd } -main $* +main $* \ No newline at end of file diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index 64cb204c8..bb579631b 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -24,6 +24,7 @@ apt-get update && \ apt-get install -y --no-install-recommends kubectl && \ apt-get clean && rm -rf /var/lib/apt/lists/ && \ mkdir -p ${DESTINATION} && \ -tar xvf ${KATA_ARTIFACTS} -C ${DESTINATION} +tar xvf ${WORKDIR}/${KATA_ARTIFACTS} -C ${DESTINATION} && \ +rm -f ${WORKDIR}/${KATA_ARTIFACTS} COPY scripts ${DESTINATION}/scripts diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md index 5a8b9cc1a..9e546e4f5 100644 --- a/tools/packaging/kata-deploy/README.md +++ b/tools/packaging/kata-deploy/README.md @@ -16,13 +16,13 @@ be utilized to install Kata Containers on a running Kubernetes cluster. For your [k3s](https://k3s.io/) cluster, run: ```sh -$ git clone github.com/kata-containers/kata-containers +$ git clone https://github.com/kata-containers/kata-containers.git ``` Check and switch to the stable branch of your choice, if wanted, and then run: ```bash -$ cd kata-containers/kata-containers/tools/packaging/kata-deploy +$ cd kata-containers/tools/packaging/kata-deploy $ kubectl apply -f kata-rbac/base/kata-rbac.yaml $ kubectl apply -k kata-deploy/overlays/k3s ``` @@ -32,13 +32,13 @@ $ kubectl apply -k kata-deploy/overlays/k3s For your [RKE2](https://docs.rke2.io/) cluster, run: ```sh -$ git clone github.com/kata-containers/kata-containers +$ git clone https://github.com/kata-containers/kata-containers.git ``` Check and switch to the stable branch of your choice, if wanted, and then run: ```bash -$ cd kata-containers/kata-containers/tools/packaging/kata-deploy +$ cd kata-containers/tools/packaging/kata-deploy $ kubectl apply -f kata-rbac/base/kata-rbac.yaml $ kubectl apply -k kata-deploy/overlays/rke2 ``` diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index e83b87ead..bd177834f 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -21,9 +21,6 @@ spec: image: quay.io/kata-containers/kata-deploy-cc:v0 imagePullPolicy: Always command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] - readinessProbe: - exec: - command: [ "bash", "-c", "[ -f /opt/kata/kata-deployed ]", "&&", "bash", "-c", "[ $? == 1 ]" ] env: - name: NODE_NAME valueFrom: diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index c371a4a8a..b850d604f 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -18,9 +18,6 @@ spec: - name: kube-kata image: quay.io/kata-containers/kata-deploy-cc:v0 imagePullPolicy: Always - readinessProbe: - exec: - command: [ "bash", "-c", "[ -f /opt/kata/kata-deployed ]", "&&", "bash", "-c", "[ $? == 0 ]" ] lifecycle: preStop: exec: diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 74192d536..7d63560d5 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -37,13 +37,16 @@ all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh all: serial-targets \ firecracker-tarball \ - kernel-tarball \ kernel-dragonball-experimental-tarball \ + kernel-nvidia-gpu-tarball \ + kernel-nvidia-gpu-snp-tarball \ + kernel-nvidia-gpu-tdx-experimental-tarball \ + kernel-tarball \ kernel-tdx-experimental-tarball \ - kernel-gpu \ - kernel-gpu-snp-tarball \ - kernel-gpu-tdx-experimental-tarball \ nydus-tarball \ + ovmf-sev-tarball \ + ovmf-tarball \ + qemu-snp-experimental-tarball \ qemu-tarball \ qemu-tdx-experimental-tarball \ shim-v2-tarball \ @@ -53,6 +56,7 @@ all: serial-targets \ serial-targets: ${MAKE} -f $(MK_PATH) -j 1 V= \ rootfs-image-tarball \ + rootfs-initrd-sev-tarball \ rootfs-initrd-tarball \ cloud-hypervisor-tarball @@ -65,30 +69,42 @@ cloud-hypervisor-tarball: firecracker-tarball: ${MAKE} $@-build -kernel-tarball: - ${MAKE} $@-build - kernel-dragonball-experimental-tarball: ${MAKE} $@-build -kernel-gpu-tarball: +kernel-experimental-tarball: ${MAKE} $@-build -kernel-gpu-snp-tarball: +kernel-nvidia-gpu-tarball: ${MAKE} $@-build -kernel-gpu-tdx-experimental-tarball: +kernel-nvidia-gpu-snp-tarball: + ${MAKE} $@-build + +kernel-nvidia-gpu-tdx-experimental-tarball: ${MAKE} $@-build -kernel-experimental-tarball: +kernel-tarball: ${MAKE} $@-build kernel-tdx-experimental-tarball: ${MAKE} $@-build +kernel-sev-tarball: + ${MAKE} $@-build + nydus-tarball: ${MAKE} $@-build +ovmf-sev-tarball: + ${MAKE} $@-build + +ovmf-tarball: + ${MAKE} $@-build + +qemu-snp-experimental-tarball: + ${MAKE} $@-build + qemu-tarball: ${MAKE} $@-build @@ -98,6 +114,9 @@ qemu-tdx-experimental-tarball: rootfs-image-tarball: ${MAKE} $@-build +rootfs-initrd-sev-tarball: kernel-sev-tarball + ${MAKE} $@-build + rootfs-initrd-tarball: ${MAKE} $@-build diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 0e8eff1e6..d9f1401a1 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -16,6 +16,8 @@ kata_dir=$(realpath "${script_dir}/../../../../") kata_deploy_create="${script_dir}/kata-deploy-binaries.sh" uid=$(id -u ${USER}) gid=$(id -g ${USER}) +http_proxy="${http_proxy:-}" +https_proxy="${https_proxy:-}" if [ "${script_dir}" != "${PWD}" ]; then ln -sf "${script_dir}/build" "${PWD}/build" @@ -42,6 +44,8 @@ docker build -q -t build-kata-deploy \ --build-arg IMG_USER="${USER}" \ --build-arg UID=${uid} \ --build-arg GID=${gid} \ + --build-arg http_proxy="${http_proxy}" \ + --build-arg https_proxy="${https_proxy}" \ --build-arg HOST_DOCKER_GID=${docker_gid} \ "${script_dir}/dockerbuild/" diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 7f7d791b8..45cf1220c 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -89,15 +89,20 @@ options: kernel kernel-dragonball-experimental kernel-experimental + kernel-nvidia-gpu + kernel-nvidia-gpu-snp + kernel-nvidia-gpu-tdx-experimental + kernel-sev-tarball kernel-tdx-experimental - kernel-gpu - kernel-gpu-snp - kernel-gpu-tdx-experimental nydus + ovmf + ovmf-sev qemu + qemu-snp-experimental qemu-tdx-experimental rootfs-image rootfs-initrd + rootfs-initrd-sev shim-v2 tdvf virtiofsd @@ -588,8 +593,10 @@ install_image() { #Install guest initrd install_initrd() { - local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-initrd-$(uname -m)/${cached_artifacts_path}" - local component="rootfs-initrd" + local initrd_type="${1:-"initrd"}" + local initrd_suffix="${2:-""}" + local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}" + local component="rootfs-${initrd_type}" local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" @@ -602,14 +609,52 @@ install_initrd() { install_cached_component \ "${component}" \ "${jenkins}" \ - "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-initrd" \ + "${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${initrd_type}" \ "" \ "${final_tarball_name}" \ "${final_tarball_path}" \ && return 0 info "Create initrd" - "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" + "${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}" +} + +#Install guest initrd for sev +install_initrd_sev() { + install_initrd "initrd-sev" "sev" +} + +#Install kernel component helper +install_cached_kernel_tarball_component() { + local kernel_name=${1} + + install_cached_tarball_component \ + "${kernel_name}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "${final_tarball_name}" \ + "${final_tarball_path}" \ + || return 1 + + if [[ "${kernel_name}" != "kernel-sev" ]]; then + return 0 + fi + + # SEV specific code path + install_cached_tarball_component \ + "${kernel_name}" \ + "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ + "${kernel_version}-${kernel_kata_config_version}" \ + "$(get_kernel_image_name)" \ + "kata-static-kernel-sev-modules.tar.xz" \ + "${workdir}/kata-static-kernel-sev-modules.tar.xz" \ + || return 1 + + mkdir -p "${module_dir}" + tar xvf "${workdir}/kata-static-kernel-sev-modules.tar.xz" -C "${module_dir}" && return 0 + + return 1 } install_cc_initrd() { @@ -624,16 +669,16 @@ install_kernel_helper() { local extra_cmd=${3} export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})" - local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" + local module_dir="" - install_cached_tarball_component \ - "${kernel_name}" \ - "${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \ - "${kernel_version}-${kernel_kata_config_version}" \ - "$(get_kernel_image_name)" \ - "${final_tarball_name}" \ - "${final_tarball_path}" \ - && return 0 + if [[ "${kernel_name}" == "kernel-sev" ]]; then + kernel_version="$(get_from_kata_deps assets.kernel.sev.version)" + default_patches_dir="${repo_root_dir}/tools/packaging/kernel/patches" + module_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kernel-sev/builddir/kata-linux-${kernel_version#v}-${kernel_kata_config_version}/lib/modules/${kernel_version#v}" + fi + + install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0 info "build ${kernel_name}" info "Kernel version ${kernel_version}" @@ -656,32 +701,32 @@ install_kernel_dragonball_experimental() { } #Install GPU enabled kernel asset -install_kernel_gpu() { +install_kernel_nvidia_gpu() { local kernel_url="$(get_from_kata_deps assets.kernel.url)" install_kernel_helper \ "assets.kernel.version" \ - "kernel-gpu" \ + "kernel-nvidia-gpu" \ "-g nvidia -u ${kernel_url} -H deb" } #Install GPU and SNP enabled kernel asset -install_kernel_gpu_snp() { +install_kernel_nvidia_gpu_snp() { local kernel_url="$(get_from_kata_deps assets.kernel.snp.url)" install_kernel_helper \ "assets.kernel.snp.version" \ - "kernel-gpu-snp" \ + "kernel-nvidia-gpu-snp" \ "-x snp -g nvidia -u ${kernel_url} -H deb" } #Install GPU and TDX experimental enabled kernel asset -install_kernel_gpu_tdx_experimental() { +install_kernel_nvidia_gpu_tdx_experimental() { local kernel_url="$(get_from_kata_deps assets.kernel-tdx-experimental.url)" install_kernel_helper \ "assets.kernel-tdx-experimental.version" \ - "kernel-gpu-tdx" \ + "kernel-nvidia-gpu-tdx-experimental" \ "-x tdx -g nvidia -u ${kernel_url} -H deb" } @@ -703,6 +748,17 @@ install_kernel_tdx_experimental() { "-x tdx -u ${kernel_url}" } +#Install sev kernel asset +install_kernel_sev() { + info "build sev kernel" + local kernel_url="$(get_from_kata_deps assets.kernel.sev.url)" + + install_kernel_helper \ + "assets.kernel.sev.version" \ + "kernel-sev" \ + "-x sev -u ${kernel_url}" +} + install_qemu_helper() { local qemu_repo_yaml_path="${1}" local qemu_version_yaml_path="${2}" @@ -747,12 +803,25 @@ install_qemu_tdx_experimental() { "${qemu_experimental_builder}" } +install_qemu_snp_experimental() { + export qemu_suffix="snp-experimental" + export qemu_tarball_name="kata-static-qemu-${qemu_suffix}.tar.gz" + + install_qemu_helper \ + "assets.hypervisor.qemu-${qemu_suffix}.url" \ + "assets.hypervisor.qemu-${qemu_suffix}.tag" \ + "qemu-${qemu_suffix}" \ + "${qemu_experimental_builder}" +} + # Install static firecracker asset install_firecracker() { - install_cached_component \ + local firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version") + + install_cached_tarball_component \ "firecracker" \ "${jenkins_url}/job/kata-containers-main-firecracker-$(uname -m)/${cached_artifacts_path}" \ - "$(get_from_kata_deps "assets.hypervisor.firecracker.version")" \ + "${firecracker_version}" \ "" \ "${final_tarball_name}" \ "${final_tarball_path}" \ @@ -762,8 +831,8 @@ install_firecracker() { "${firecracker_builder}" info "Install static firecracker" mkdir -p "${destdir}/opt/kata/bin/" - sudo install -D --owner root --group root --mode 0744 firecracker/firecracker-static "${destdir}/opt/kata/bin/firecracker" - sudo install -D --owner root --group root --mode 0744 firecracker/jailer-static "${destdir}/opt/kata/bin/jailer" + sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/firecracker-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/firecracker" + sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/jailer-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/jailer" } # Install static cloud-hypervisor asset @@ -808,7 +877,9 @@ install_virtiofsd() { # Install static nydus asset install_nydus() { - install_cached_component \ + [ "${ARCH}" == "aarch64" ] && ARCH=arm64 + + install_cached_tarball_component \ "nydus" \ "${jenkins_url}/job/kata-containers-main-nydus-$(uname -m)/${cached_artifacts_path}" \ "$(get_from_kata_deps "externals.nydus.version")" \ @@ -851,7 +922,7 @@ install_shimv2() { install_ovmf() { ovmf_type="${1:-x86_64}" - tarball_name="${2:-edk2.tar.xz}" + tarball_name="${2:-edk2-x86_64.tar.gz}" local component_name="ovmf" local component_version="$(get_from_kata_deps "externals.ovmf.${ovmf_type}.version")" @@ -874,6 +945,11 @@ install_tdvf() { install_ovmf "tdx" "edk2-tdx.tar.gz" } +# Install OVMF SEV +install_ovmf_sev() { + install_ovmf "sev" "edk2-sev.tar.gz" +} + get_kata_version() { local v v=$(cat "${version_file}") @@ -895,11 +971,15 @@ handle_build() { install_firecracker install_image install_initrd + install_initrd_sev install_kernel install_kernel_dragonball_experimental install_kernel_tdx_experimental install_nydus + install_ovmf + install_ovmf_sev install_qemu + install_qemu_snp_experimental install_qemu_tdx_experimental install_shimv2 install_tdvf @@ -958,28 +1038,38 @@ handle_build() { kernel) install_kernel ;; - nydus) install_nydus ;; - kernel-dragonball-experimental) install_kernel_dragonball_experimental ;; kernel-experimental) install_kernel_experimental ;; + kernel-nvidia-gpu) install_kernel_nvidia_gpu ;; + + kernel-nvidia-gpu-snp) install_kernel_nvidia_gpu_snp;; + + kernel-nvidia-gpu-tdx-experimental) install_kernel_nvidia_gpu_tdx_experimental;; + kernel-tdx-experimental) install_kernel_tdx_experimental ;; - kernel-gpu) install_kernel_gpu ;; + kernel-sev) install_kernel_sev ;; - kernel-gpu-snp) install_kernel_gpu_snp;; + nydus) install_nydus ;; - kernel-gpu-tdx-experimental) install_kernel_gpu_tdx_experimental;; + ovmf) install_ovmf ;; + + ovmf-sev) install_ovmf_sev ;; qemu) install_qemu ;; + qemu-snp-experimental) install_qemu_snp_experimental ;; + qemu-tdx-experimental) install_qemu_tdx_experimental ;; rootfs-image) install_image ;; rootfs-initrd) install_initrd ;; + rootfs-initrd-sev) install_initrd_sev ;; + shim-v2) install_shimv2 ;; tdvf) install_tdvf ;; diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh index 394dc4ba0..729e9a241 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh @@ -20,8 +20,8 @@ mkdir "${tarball_content_dir}" for c in kata-static-*.tar.xz do - echo "untarring tarball "${c}" into ${tarball_content_dir}" - tar -xvf "${c}" -C "${tarball_content_dir}" + echo "untarring tarball "${c}" into ${tarball_content_dir}" + tar -xvf "${c}" -C "${tarball_content_dir}" done echo "create ${tar_path}" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml index f65ec6b0a..b55096226 100644 --- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml @@ -14,6 +14,19 @@ scheduling: --- kind: RuntimeClass apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-sev +handler: kata-qemu-sev +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 metadata: name: kata-qemu-tdx handler: kata-qemu-tdx @@ -27,6 +40,19 @@ scheduling: --- kind: RuntimeClass apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu-snp +handler: kata-qemu-snp +overhead: + podFixed: + memory: "2048Mi" + cpu: "1.0" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 metadata: name: kata-clh handler: kata-clh @@ -67,8 +93,8 @@ scheduling: kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: - name: kata-qemu-gpu -handler: kata-qemu-gpu + name: kata-qemu-nvidia-gpu +handler: kata-qemu-nvidia-gpu overhead: podFixed: memory: "160Mi" diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 5ae15802a..6bb660198 100644 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -16,8 +16,10 @@ containerd_conf_file_backup="${containerd_conf_file}.bak" shims=( "fc" "qemu" + "qemu-nvidia-gpu" "qemu-tdx" - "qemu-gpu" + "qemu-sev" + "qemu-snp" "clh" "dragonball" ) @@ -90,6 +92,20 @@ function configure_cri_runtime() { wait_till_node_is_ready } +function backup_shim() { + local shim_file="$1" + local shim_backup="${shim_file}.bak" + + if [ -f "${shim_file}" ]; then + echo "warning: ${shim_file} already exists" >&2 + if [ ! -f "${shim_backup}" ]; then + mv "${shim_file}" "${shim_backup}" + else + rm "${shim_file}" + fi + fi +} + function configure_different_shims_base() { # Currently containerd has an assumption on the location of the shimv2 implementation # This forces kata-deploy to create files in a well-defined location that's part of @@ -98,21 +114,15 @@ function configure_different_shims_base() { # https://github.com/containerd/containerd/issues/3073 # https://github.com/containerd/containerd/issues/5006 + local default_shim_file="/usr/local/bin/containerd-shim-kata-v2" + mkdir -p /usr/local/bin for shim in "${shims[@]}"; do local shim_binary="containerd-shim-kata-${shim}-v2" local shim_file="/usr/local/bin/${shim_binary}" - local shim_backup="/usr/local/bin/${shim_binary}.bak" - if [ -f "${shim_file}" ]; then - echo "warning: ${shim_binary} already exists" >&2 - if [ ! -f "${shim_backup}" ]; then - mv "${shim_file}" "${shim_backup}" - else - rm "${shim_file}" - fi - fi + backup_shim "${shim_file}" if [[ "${shim}" == "dragonball" ]]; then ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}" @@ -122,26 +132,37 @@ function configure_different_shims_base() { chmod +x "$shim_file" if [ "${shim}" == "${default_shim}" ]; then + backup_shim "${default_shim_file}" + echo "Creating the default shim-v2 binary" - ln -sf "${shim_file}" /usr/local/bin/containerd-shim-kata-v2 + ln -sf "${shim_file}" "${default_shim_file}" fi done } +function restore_shim() { + local shim_file="$1" + local shim_backup="${shim_file}.bak" + + if [ -f "${shim_backup}" ]; then + mv "$shim_backup" "$shim_file" + fi +} + function cleanup_different_shims_base() { + local default_shim_file="/usr/local/bin/containerd-shim-kata-v2" + for shim in "${shims[@]}"; do local shim_binary="containerd-shim-kata-${shim}-v2" local shim_file="/usr/local/bin/${shim_binary}" - local shim_backup="/usr/local/bin/${shim_binary}.bak" rm "${shim_file}" || true - if [ -f "${shim_backup}" ]; then - mv "$shim_backup" "$shim_file" - fi + restore_shim "${shim_file}" done - rm /usr/local/bin/containerd-shim-kata-v2 + rm "${default_shim_file}" || true + restore_shim "${default_shim_file}" } function configure_crio_runtime() { @@ -324,13 +345,11 @@ function main() { install_artifacts configure_cri_runtime "$runtime" kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=true - touch /opt/kata/kata-deployed ;; cleanup) cleanup_cri_runtime "$runtime" kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup remove_artifacts - rm /opt/kata/kata-deployed ;; reset) reset_runtime $runtime diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md index d9b78480a..5c8580a68 100644 --- a/tools/packaging/kernel/README.md +++ b/tools/packaging/kernel/README.md @@ -70,7 +70,7 @@ $ ./build-kernel.sh -v 5.10.25 -g nvidia -f -d setup ## Setup kernel source code ```bash -$ git clone github.com/kata-containers/kata-containers +$ git clone https://github.com/kata-containers/kata-containers.git $ cd kata-containers/tools/packaging/kernel $ ./build-kernel.sh setup ``` diff --git a/tools/packaging/scripts/configure-hypervisor.sh b/tools/packaging/scripts/configure-hypervisor.sh index 495251e4c..8386690a4 100755 --- a/tools/packaging/scripts/configure-hypervisor.sh +++ b/tools/packaging/scripts/configure-hypervisor.sh @@ -340,7 +340,7 @@ generate_qemu_options() { qemu_options+=(functionality:--disable-virtiofsd) ;; ppc64le) - qemu_options+=(functionality:--enable-virtiofsd) + qemu_options+=(functionality:--disable-virtiofsd) ;; s390x) qemu_options+=(functionality:--disable-virtiofsd) diff --git a/tools/packaging/static-build/cache_components_main.sh b/tools/packaging/static-build/cache_components_main.sh index 0e8a0120f..50b2ba83e 100755 --- a/tools/packaging/static-build/cache_components_main.sh +++ b/tools/packaging/static-build/cache_components_main.sh @@ -12,9 +12,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../scripts/lib.sh" -KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental +KERNEL_FLAVOUR="${KERNEL_FLAVOUR:-kernel}" # kernel | kernel-nvidia-gpu | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental | kernel-nvidia-gpu-tdx-experimental | kernel-nvidia-gpu-snp OVMF_FLAVOUR="${OVMF_FLAVOUR:-x86_64}" # x86_64 | tdx -QEMU_FLAVOUR="${QEMU_FLAVOUR:-qemu}" # qemu | qemu-tdx-experimental +QEMU_FLAVOUR="${QEMU_FLAVOUR:-qemu}" # qemu | qemu-tdx-experimental | qemu-snp-experimental ROOTFS_IMAGE_TYPE="${ROOTFS_IMAGE_TYPE:-image}" # image | initrd cache_clh_artifacts() { @@ -33,8 +33,40 @@ cache_kernel_artifacts() { local kernel_tarball_name="kata-static-${KERNEL_FLAVOUR}.tar.xz" local current_kernel_image="$(get_kernel_image_name)" local current_kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" - local current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")-${current_kernel_kata_config_version}" - create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}" "${current_kernel_image}" + local kernel_modules_tarball_path="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kata-static-kernel-sev-modules.tar.xz" + + # The ${vendor}-gpu kernels are based on an already existing entry, and does not require + # adding a new entry to the versions.yaml. + # + # With this in mind, let's just make sure we get the version from correct entry in the + # versions.yaml file. + case ${KERNEL_FLAVOUR} in + *"nvidia-gpu"*) + KERNEL_FLAVOUR=${KERNEL_FLAVOUR//"-nvidia-gpu"/} + ;; + *) + ;; + esac + + case ${KERNEL_FLAVOUR} in + "kernel-sev"|"kernel-snp") + # In these cases, like "kernel-foo", it must be set to "kernel.foo" when looking at + # the versions.yaml file + current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR/-/.}.version")" + ;; + *) + current_kernel_version="$(get_from_kata_deps "assets.${KERNEL_FLAVOUR}.version")" + ;; + esac + + create_cache_asset "${kernel_tarball_name}" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}" + if [[ "${KERNEL_FLAVOUR}" == "kernel-sev" ]]; then + module_dir="${repo_root_dir}/tools/packaging/kata-deploy/local-build/build/kernel-sev/builddir/kata-linux-${current_kernel_version#v}-${current_kernel_kata_config_version}/lib/modules/${current_kernel_version#v}" + if [ ! -f "${kernel_modules_tarball_path}" ]; then + tar cvfJ "${kernel_modules_tarball_path}" "${module_dir}/kernel/drivers/virt/coco/efi_secret/" + fi + create_cache_asset "kata-static-kernel-sev-modules.tar.xz" "${current_kernel_version}-${current_kernel_kata_config_version}" "${current_kernel_image}" + fi } cache_nydus_artifacts() { @@ -45,8 +77,18 @@ cache_nydus_artifacts() { cache_ovmf_artifacts() { local current_ovmf_version="$(get_from_kata_deps "externals.ovmf.${OVMF_FLAVOUR}.version")" - [ "${OVMF_FLAVOUR}" == "tdx" ] && OVMF_FLAVOUR="tdvf" - local ovmf_tarball_name="kata-static-${OVMF_FLAVOUR}.tar.xz" + case ${OVMF_FLAVOUR} in + "tdx") + ovmf_tarball_name="kata-static-tdvf.tar.xz" + ;; + "x86_64") + ovmf_tarball_name="kata-static-ovmf.tar.xz" + ;; + *) + ovmf_tarball_name="kata-static-ovmf-${OVMF_FLAVOUR}.tar.xz" + ;; + esac + local current_ovmf_image="$(get_ovmf_image_name)" create_cache_asset "${ovmf_tarball_name}" "${current_ovmf_version}" "${current_ovmf_image}" } @@ -116,11 +158,11 @@ Usage: $0 "[options]" -c Cloud hypervisor cache -F Firecracker cache -k Kernel cache - * Export KERNEL_FLAVOUR="kernel | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental" for a specific build + * Export KERNEL_FLAVOUR="kernel | kernel-nvidia-gpu | kernel-experimental | kernel-arm-experimental | kernel-dragonball-experimental | kernel-tdx-experimental | kernel-nvidia-gpu-tdx-experimental | kernel-nvidia-gpu-snp" for a specific build The default KERNEL_FLAVOUR value is "kernel" -n Nydus cache -q QEMU cache - * Export QEMU_FLAVOUR="qemu | qemu-tdx-experimental" for a specific build + * Export QEMU_FLAVOUR="qemu | qemu-tdx-experimental | qemu-snp-experimental" for a specific build The default QEMU_FLAVOUR value is "qemu" -r RootFS cache * Export ROOTFS_IMAGE_TYPE="image|initrd" for one of those two types diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 0bee1ea04..975a517a1 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -44,6 +44,9 @@ pull_clh_released_binary() { info "Download cloud-hypervisor version: ${cloud_hypervisor_version}" cloud_hypervisor_binary="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${cloud_hypervisor_version}/cloud-hypervisor-static" + [ "${ARCH}" == "aarch64" ] && \ + cloud_hypervisor_binary="${cloud_hypervisor_binary}-aarch64" + curl --fail -L ${cloud_hypervisor_binary} -o cloud-hypervisor-static || return 1 mkdir -p cloud-hypervisor mv -f cloud-hypervisor-static cloud-hypervisor/cloud-hypervisor @@ -82,11 +85,6 @@ build_clh_from_source() { popd } -if [ "${ARCH}" == "aarch64" ]; then - info "aarch64 binaries are not distributed as part of the Cloud Hypervisor releases, forcing to build from source" - force_build_from_source="true" -fi - if [ -n "${features}" ]; then info "As an extra build argument has been passed to the script, forcing to build from source" force_build_from_source="true" diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh index 159575410..95cba1389 100755 --- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh +++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh @@ -14,30 +14,31 @@ source "${script_dir}/../../scripts/lib.sh" config_dir="${script_dir}/../../scripts/" -firecracker_repo="${firecracker_repo:-}" +firecracker_url="${firecracker_url:-}" firecracker_dir="firecracker" firecracker_version="${firecracker_version:-}" arch=$(uname -m) -if [ -z "$firecracker_repo" ]; then - info "Get firecracker information from runtime versions.yaml" - firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url") - [ -n "$firecracker_url" ] || die "failed to get firecracker url" - firecracker_repo="${firecracker_url}.git" -fi -[ -n "$firecracker_repo" ] || die "failed to get firecracker repo" +[ -n "$firecracker_url" ] ||firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url") +[ -n "$firecracker_url" ] || die "failed to get firecracker url" [ -n "$firecracker_version" ] || firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version") [ -n "$firecracker_version" ] || die "failed to get firecracker version" -info "Build ${firecracker_repo} version: ${firecracker_version}" +firecracker_tarball_url="${firecracker_url}/releases/download" -[ -d "${firecracker_dir}" ] || git clone ${firecracker_repo} -cd "${firecracker_dir}" -git fetch -git checkout ${firecracker_version} -sudo ./tools/devtool --unattended build --release +file_name="firecracker-${firecracker_version}-${arch}.tgz" +download_url="${firecracker_tarball_url}/${firecracker_version}/${file_name}" -ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/firecracker ./firecracker-static -ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/jailer ./jailer-static +info "Download firecracker version: ${firecracker_version} from ${download_url}" +curl -o ${file_name} -L $download_url + +sha256sum="${file_name}.sha256.txt" +sha256sum_url="${firecracker_tarball_url}/${firecracker_version}/${sha256sum}" + +info "Download firecracker ${sha256sum} from ${sha256sum_url}" +curl -o ${sha256sum} -L $sha256sum_url + +sha256sum -c ${sha256sum} +tar zxvf ${file_name} diff --git a/utils.mk b/utils.mk index f58a37ca1..b1f4c1f85 100644 --- a/utils.mk +++ b/utils.mk @@ -146,7 +146,6 @@ ifneq ($(LIBC),musl) endif ifeq ($(ARCH), ppc64le) - override ARCH = powerpc64le override LIBC = gnu $(warning "WARNING: powerpc64le-unknown-linux-musl target is unavailable") endif diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh index c51fb516c..249a510fa 100755 --- a/utils/kata-manager.sh +++ b/utils/kata-manager.sh @@ -262,7 +262,7 @@ pre_checks() command -v "${kata_shim_v2}" &>/dev/null \ && die "Please remove existing $kata_project installation" - [skip_containerd = "false" ] && return 0 + [ "$skip_containerd" = 'true' ] && return 0 local ret diff --git a/versions.yaml b/versions.yaml index ebac758eb..0fc854ab2 100644 --- a/versions.yaml +++ b/versions.yaml @@ -102,16 +102,23 @@ assets: description: "VMM that uses KVM and supports TDX" url: "https://github.com/kata-containers/qemu" tag: "TDX-v3.1" - snp: - description: "VMM that uses KVM and supports AMD SEV-SNP" - url: "https://github.com/AMDESE/qemu" - tag: "3b6a2b6b7466f6dea53243900b7516c3f29027b7" qemu-experimental: description: "QEMU with virtiofs support" url: "https://github.com/qemu/qemu" version: "7a800cf9496fddddf71b21a00991e0ec757a170a" + qemu-tdx-experimental: + # yamllint disable-line rule:line-length + description: "QEMU with TDX support - based on https://github.com/intel/tdx-tools/releases/tag/2023ww01" + url: "https://github.com/kata-containers/qemu" + tag: "ad4c7f529a279685da84297773b4ec8080153c2d-plus-TDX-v1.3" + + qemu-snp-experimental: + description: "QEMU with experimental SNP support (no UPM)" + url: "https://github.com/AMDESE/qemu" + tag: "3b6a2b6b7466f6dea53243900b7516c3f29027b7" + image: description: | Root filesystem disk image used to boot the guest virtual @@ -151,6 +158,9 @@ assets: x86_64: name: *default-initrd-name version: *default-initrd-version + sev: + name: *glibc-initrd-name + version: *glibc-initrd-version kernel: description: "Linux kernel optimised for virtual machines" @@ -161,7 +171,7 @@ assets: url: "https://github.com/kata-containers/linux/archive/refs/tags" tag: "5.15-plus-TDX" sev: - description: "Linux kernel that supports SEV" + description: "Linux kernel that supports SEV and SNP" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" version: "v5.19.2" snp: @@ -277,7 +287,7 @@ externals: nydus: description: "Nydus image acceleration service" url: "https://github.com/dragonflyoss/image-service" - version: "v2.2.0" + version: "v2.2.1" nydus-snapshotter: description: "Snapshotter for Nydus image acceleration service" @@ -294,10 +304,7 @@ externals: package_output_dir: "OvmfX64" sev: description: "AmdSev build needed for SEV measured direct boot." - # CCv0 switching to this fork until new release of edk2 has fix - # per #5893, will need to be reverted to upstream - url: "https://github.com/kata-containers/edk2" - version: "edk2-202208-sev-alloc-runtime-secret-area" + version: "edk2-stable202302" package: "OvmfPkg/AmdSev/AmdSevX64.dsc" package_output_dir: "AmdSev" tdx: @@ -316,14 +323,14 @@ externals: virtiofsd: description: "vhost-user virtio-fs device backend written in Rust" url: "https://gitlab.com/virtio-fs/virtiofsd" - version: "v1.3.0" - toolchain: "1.62.0" + version: "v1.6.1" + toolchain: "1.66.0" meta: - # From https://gitlab.com/virtio-fs/virtiofsd/-/releases/v1.3.0, - # this is the link labelled virtiofsd-v1.3.0.zip + # From https://gitlab.com/virtio-fs/virtiofsd/-/releases/v1.6.1, + # this is the link labelled virtiofsd-v1.6.1.zip # # yamllint disable-line rule:line-length - binary: "https://gitlab.com/virtio-fs/virtiofsd/uploads/9a4f2261fcb1701f1e709694b5c5d980/virtiofsd-v1.3.0.zip" + binary: "https://gitlab.com/virtio-fs/virtiofsd/uploads/14c1e8a7acc82d515cec6608727a1e4a/virtiofsd-v1.6.1.zip" languages: description: | @@ -343,12 +350,12 @@ languages: rust: description: "Rust language" notes: "'version' is the default minimum version used by this project." - version: "1.66.0" + version: "1.68.0" meta: description: | 'newest-version' is the latest version known to work when building Kata - newest-version: "1.66.0" + newest-version: "1.68.0" golangci-lint: description: "golangci-lint"